From 57535c039c938f7c179693d9db8b052912019823 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 1 Jul 2024 11:23:31 +0300 Subject: [PATCH 001/194] tests: remove a leftover 'running' flag (#8216) The 'running' boolean was replaced with a semaphore in commit f0e2bb79b2, but this initialization was missed. Remove it so that if a test tries to access it, you get an error rather than always claiming that the endpoint is not running. Spotted by Arseny at https://github.com/neondatabase/neon/pull/7288#discussion_r1660068657 --- test_runner/fixtures/neon_fixtures.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 4911917bf452..a1cb1b51953c 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -3491,7 +3491,6 @@ def __init__( ): super().__init__(host="localhost", port=pg_port, user="cloud_admin", dbname="postgres") self.env = env - self.running = False self.branch_name: Optional[str] = None # dubious self.endpoint_id: Optional[str] = None # dubious, see asserts below self.pgdata_dir: Optional[str] = None # Path to computenode PGDATA From 75c84c846a2517cbbe414ae5f3e0649f4a359036 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 1 Jul 2024 12:58:08 +0300 Subject: [PATCH 002/194] tests: Make neon_xlogflush() flush all WAL, if you omit the LSN arg This makes it much more convenient to use in the common case that you want to flush all the WAL. (Passing pg_current_wal_insert_lsn() as the argument doesn't work for the same reasons as explained in the comments: we need to be back off to the beginning of a page if the previous record ended at page boundary.) I plan to use this to fix the issue that Arseny Sher called out at https://github.com/neondatabase/neon/pull/7288#discussion_r1660063852 --- pgxn/neon_test_utils/Makefile | 2 +- ...tils--1.1.sql => neon_test_utils--1.2.sql} | 2 +- pgxn/neon_test_utils/neon_test_utils.control | 2 +- pgxn/neon_test_utils/neontest.c | 38 ++++++++++++++++++- 4 files changed, 40 insertions(+), 4 deletions(-) rename pgxn/neon_test_utils/{neon_test_utils--1.1.sql => neon_test_utils--1.2.sql} (96%) diff --git a/pgxn/neon_test_utils/Makefile b/pgxn/neon_test_utils/Makefile index 1ee87357e5e2..13712724399d 100644 --- a/pgxn/neon_test_utils/Makefile +++ b/pgxn/neon_test_utils/Makefile @@ -7,7 +7,7 @@ OBJS = \ neontest.o EXTENSION = neon_test_utils -DATA = neon_test_utils--1.1.sql +DATA = neon_test_utils--1.2.sql PGFILEDESC = "neon_test_utils - helpers for neon testing and debugging" PG_CONFIG = pg_config diff --git a/pgxn/neon_test_utils/neon_test_utils--1.1.sql b/pgxn/neon_test_utils/neon_test_utils--1.2.sql similarity index 96% rename from pgxn/neon_test_utils/neon_test_utils--1.1.sql rename to pgxn/neon_test_utils/neon_test_utils--1.2.sql index 534784f31912..f84a24ec8d48 100644 --- a/pgxn/neon_test_utils/neon_test_utils--1.1.sql +++ b/pgxn/neon_test_utils/neon_test_utils--1.2.sql @@ -41,7 +41,7 @@ RETURNS bytea AS 'MODULE_PATHNAME', 'get_raw_page_at_lsn_ex' LANGUAGE C PARALLEL UNSAFE; -CREATE FUNCTION neon_xlogflush(lsn pg_lsn) +CREATE FUNCTION neon_xlogflush(lsn pg_lsn DEFAULT NULL) RETURNS VOID AS 'MODULE_PATHNAME', 'neon_xlogflush' LANGUAGE C PARALLEL UNSAFE; diff --git a/pgxn/neon_test_utils/neon_test_utils.control b/pgxn/neon_test_utils/neon_test_utils.control index 5f6d64083591..c7b9191ddc12 100644 --- a/pgxn/neon_test_utils/neon_test_utils.control +++ b/pgxn/neon_test_utils/neon_test_utils.control @@ -1,6 +1,6 @@ # neon_test_utils extension comment = 'helpers for neon testing and debugging' -default_version = '1.1' +default_version = '1.2' module_pathname = '$libdir/neon_test_utils' relocatable = true trusted = true diff --git a/pgxn/neon_test_utils/neontest.c b/pgxn/neon_test_utils/neontest.c index 47f245fbf1af..944936d39517 100644 --- a/pgxn/neon_test_utils/neontest.c +++ b/pgxn/neon_test_utils/neontest.c @@ -15,6 +15,7 @@ #include "access/relation.h" #include "access/xact.h" #include "access/xlog.h" +#include "access/xlog_internal.h" #include "catalog/namespace.h" #include "fmgr.h" #include "funcapi.h" @@ -444,11 +445,46 @@ get_raw_page_at_lsn_ex(PG_FUNCTION_ARGS) /* * Directly calls XLogFlush(lsn) to flush WAL buffers. + * + * If 'lsn' is not specified (is NULL), flush all generated WAL. */ Datum neon_xlogflush(PG_FUNCTION_ARGS) { - XLogRecPtr lsn = PG_GETARG_LSN(0); + XLogRecPtr lsn; + + if (RecoveryInProgress()) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("recovery is in progress"), + errhint("cannot flush WAL during recovery."))); + + if (!PG_ARGISNULL(0)) + lsn = PG_GETARG_LSN(0); + else + { + lsn = GetXLogInsertRecPtr(); + + /*--- + * The LSN returned by GetXLogInsertRecPtr() is the position where the + * next inserted record would begin. If the last record ended just at + * the page boundary, the next record will begin after the page header + * on the next page, and that's what GetXLogInsertRecPtr().returns, + * but the page header has not been written yet. If we tried to flush + * it, XLogFlush() would throw an error: + * + * ERROR : xlog flush request %X/%X is not satisfied --- flushed only to %X/%X + * + * To avoid that, if the insert position points to just after the page + * header, back off to page boundary. + */ + if (lsn % XLOG_BLCKSZ == SizeOfXLogShortPHD && + XLogSegmentOffset(lsn, wal_segment_size) > XLOG_BLCKSZ) + lsn -= SizeOfXLogShortPHD; + else if (lsn % XLOG_BLCKSZ == SizeOfXLogLongPHD && + XLogSegmentOffset(lsn, wal_segment_size) < XLOG_BLCKSZ) + lsn -= SizeOfXLogLongPHD; + } XLogFlush(lsn); PG_RETURN_VOID(); From 9ce193082a26714400a788f96e0c0cf95c7879df Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 1 Jul 2024 12:58:12 +0300 Subject: [PATCH 003/194] Restore running xacts from CLOG on replica startup (#7288) We have one pretty serious MVCC visibility bug with hot standby replicas. We incorrectly treat any transactions that are in progress in the primary, when the standby is started, as aborted. That can break MVCC for queries running concurrently in the standby. It can also lead to hint bits being set incorrectly, and that damage can last until the replica is restarted. The fundamental bug was that we treated any replica start as starting from a shut down server. The fix for that is straightforward: we need to set 'wasShutdown = false' in InitWalRecovery() (see changes in the postgres repo). However, that introduces a new problem: with wasShutdown = false, the standby will not open up for queries until it receives a running-xacts WAL record from the primary. That's correct, and that's how Postgres hot standby always works. But it's a problem for Neon, because: * It changes the historical behavior for existing users. Currently, the standby immediately opens up for queries, so if they now need to wait, we can breka existing use cases that were working fine (assuming you don't hit the MVCC issues). * The problem is much worse for Neon than it is for standalone PostgreSQL, because in Neon, we can start a replica from an arbitrary LSN. In standalone PostgreSQL, the replica always starts WAL replay from a checkpoint record, and the primary arranges things so that there is always a running-xacts record soon after each checkpoint record. You can still hit this issue with PostgreSQL if you have a transaction with lots of subtransactions running in the primary, but it's pretty rare in practice. To mitigate that, we introduce another way to collect the running-xacts information at startup, without waiting for the running-xacts WAL record: We can the CLOG for XIDs that haven't been marked as committed or aborted. It has limitations with subtransactions too, but should mitigate the problem for most users. See https://github.com/neondatabase/neon/issues/7236. Co-authored-by: Konstantin Knizhnik --- pageserver/src/walingest.rs | 40 +- pgxn/neon/neon.c | 293 ++++++++ test_runner/fixtures/neon_fixtures.py | 4 +- test_runner/fixtures/pageserver/utils.py | 2 +- test_runner/regress/test_replica_start.py | 646 ++++++++++++++++++ test_runner/regress/test_replication_start.py | 32 - vendor/postgres-v14 | 2 +- vendor/postgres-v15 | 2 +- vendor/postgres-v16 | 2 +- vendor/revisions.json | 6 +- 10 files changed, 981 insertions(+), 48 deletions(-) create mode 100644 test_runner/regress/test_replica_start.py delete mode 100644 test_runner/regress/test_replication_start.py diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index fb10bca5a6ba..07c90385e654 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -343,7 +343,33 @@ impl WalIngest { xlog_checkpoint.oldestActiveXid, self.checkpoint.oldestActiveXid ); - self.checkpoint.oldestActiveXid = xlog_checkpoint.oldestActiveXid; + + // A shutdown checkpoint has `oldestActiveXid == InvalidTransactionid`, + // because at shutdown, all in-progress transactions will implicitly + // end. Postgres startup code knows that, and allows hot standby to start + // immediately from a shutdown checkpoint. + // + // In Neon, Postgres hot standby startup always behaves as if starting from + // an online checkpoint. It needs a valid `oldestActiveXid` value, so + // instead of overwriting self.checkpoint.oldestActiveXid with + // InvalidTransactionid from the checkpoint WAL record, update it to a + // proper value, knowing that there are no in-progress transactions at this + // point, except for prepared transactions. + // + // See also the neon code changes in the InitWalRecovery() function. + if xlog_checkpoint.oldestActiveXid == pg_constants::INVALID_TRANSACTION_ID + && info == pg_constants::XLOG_CHECKPOINT_SHUTDOWN + { + let mut oldest_active_xid = self.checkpoint.nextXid.value as u32; + for xid in modification.tline.list_twophase_files(lsn, ctx).await? { + if (xid.wrapping_sub(oldest_active_xid) as i32) < 0 { + oldest_active_xid = xid; + } + } + self.checkpoint.oldestActiveXid = oldest_active_xid; + } else { + self.checkpoint.oldestActiveXid = xlog_checkpoint.oldestActiveXid; + } // Write a new checkpoint key-value pair on every checkpoint record, even // if nothing really changed. Not strictly required, but it seems nice to @@ -375,6 +401,7 @@ impl WalIngest { if info == pg_constants::XLOG_RUNNING_XACTS { let xlrec = crate::walrecord::XlRunningXacts::decode(&mut buf); self.checkpoint.oldestActiveXid = xlrec.oldest_running_xid; + self.checkpoint_modified = true; } } pg_constants::RM_REPLORIGIN_ID => { @@ -1277,13 +1304,10 @@ impl WalIngest { xlrec.pageno, xlrec.oldest_xid, xlrec.oldest_xid_db ); - // Here we treat oldestXid and oldestXidDB - // differently from postgres redo routines. - // In postgres checkpoint.oldestXid lags behind xlrec.oldest_xid - // until checkpoint happens and updates the value. - // Here we can use the most recent value. - // It's just an optimization, though and can be deleted. - // TODO Figure out if there will be any issues with replica. + // In Postgres, oldestXid and oldestXidDB are updated in memory when the CLOG is + // truncated, but a checkpoint record with the updated values isn't written until + // later. In Neon, a server can start at any LSN, not just on a checkpoint record, + // so we keep the oldestXid and oldestXidDB up-to-date. self.checkpoint.oldestXid = xlrec.oldest_xid; self.checkpoint.oldestXidDB = xlrec.oldest_xid_db; self.checkpoint_modified = true; diff --git a/pgxn/neon/neon.c b/pgxn/neon/neon.c index b6b2db7e71ad..e4968bdf8991 100644 --- a/pgxn/neon/neon.c +++ b/pgxn/neon/neon.c @@ -12,6 +12,8 @@ #include "fmgr.h" #include "miscadmin.h" +#include "access/subtrans.h" +#include "access/twophase.h" #include "access/xact.h" #include "access/xlog.h" #include "storage/buf_internals.h" @@ -22,10 +24,12 @@ #include "replication/logical.h" #include "replication/slot.h" #include "replication/walsender.h" +#include "storage/proc.h" #include "storage/procsignal.h" #include "tcop/tcopprot.h" #include "funcapi.h" #include "access/htup_details.h" +#include "utils/builtins.h" #include "utils/pg_lsn.h" #include "utils/guc.h" #include "utils/wait_event.h" @@ -266,6 +270,293 @@ LogicalSlotsMonitorMain(Datum main_arg) } } +/* + * XXX: These private to procarray.c, but we need them here. + */ +#define PROCARRAY_MAXPROCS (MaxBackends + max_prepared_xacts) +#define TOTAL_MAX_CACHED_SUBXIDS \ + ((PGPROC_MAX_CACHED_SUBXIDS + 1) * PROCARRAY_MAXPROCS) + +/* + * Restore running-xact information by scanning the CLOG at startup. + * + * In PostgreSQL, a standby always has to wait for a running-xacts WAL record + * to arrive before it can start accepting queries. Furthermore, if there are + * transactions with too many subxids (> 64) open to fit in the in-memory + * subxids cache, the running-xacts record will be marked as "suboverflowed", + * and the standby will need to also wait for the currently in-progress + * transactions to finish. + * + * That's not great in PostgreSQL, because a hot standby does not necessary + * open up for queries immediately as you might expect. But it's worse in + * Neon: A standby in Neon doesn't need to start WAL replay from a checkpoint + * record; it can start at any LSN. Postgres arranges things so that there is + * a running-xacts record soon after every checkpoint record, but when you + * start from an arbitrary LSN, that doesn't help. If the primary is idle, or + * not running at all, it might never write a new running-xacts record, + * leaving the replica in a limbo where it can never start accepting queries. + * + * To mitigate that, we have an additional mechanism to find the running-xacts + * information: we scan the CLOG, making note of any XIDs not marked as + * committed or aborted. They are added to the Postgres known-assigned XIDs + * array by calling ProcArrayApplyRecoveryInfo() in the caller of this + * function. + * + * There is one big limitation with that mechanism: The size of the + * known-assigned XIDs is limited, so if there are a lot of in-progress XIDs, + * we have to give up. Furthermore, we don't know how many of the in-progress + * XIDs are subtransactions, and if we use up all the space in the + * known-assigned XIDs array for subtransactions, we might run out of space in + * the array later during WAL replay, causing the replica to shut down with + * "ERROR: too many KnownAssignedXids". The safe # of XIDs that we can add to + * the known-assigned array without risking that error later is very low, + * merely PGPROC_MAX_CACHED_SUBXIDS == 64, so we take our chances and use up + * to half of the known-assigned XIDs array for the subtransactions, even + * though that risks getting the error later. + * + * Note: It's OK if the recovered list of XIDs includes some transactions that + * have crashed in the primary, and hence will never commit. They will be seen + * as in-progress, until we see a new next running-acts record with an + * oldestActiveXid that invalidates them. That's how the known-assigned XIDs + * array always works. + * + * If scraping the CLOG doesn't succeed for some reason, like the subxid + * overflow, Postgres will fall back to waiting for a running-xacts record + * like usual. + * + * Returns true if a complete list of in-progress XIDs was scraped. + */ +static bool +RestoreRunningXactsFromClog(CheckPoint *checkpoint, TransactionId **xids, int *nxids) +{ + TransactionId from; + TransactionId till; + int max_xcnt; + TransactionId *prepared_xids = NULL; + int n_prepared_xids; + TransactionId *restored_xids = NULL; + int n_restored_xids; + int next_prepared_idx; + + Assert(*xids == NULL); + + /* + * If the checkpoint doesn't have a valid oldestActiveXid, bail out. We + * don't know where to start the scan. + * + * This shouldn't happen, because the pageserver always maintains a valid + * oldestActiveXid nowadays. Except when starting at an old point in time + * that was ingested before the pageserver was taught to do that. + */ + if (!TransactionIdIsValid(checkpoint->oldestActiveXid)) + { + elog(LOG, "cannot restore running-xacts from CLOG because oldestActiveXid is not set"); + goto fail; + } + + /* + * We will scan the CLOG starting from the oldest active XID. + * + * In some corner cases, the oldestActiveXid from the last checkpoint + * might already have been truncated from the CLOG. That is, + * oldestActiveXid might be older than oldestXid. That's possible because + * oldestActiveXid is only updated at checkpoints. After the last + * checkpoint, the oldest transaction might have committed, and the CLOG + * might also have been already truncated. So if oldestActiveXid is older + * than oldestXid, start at oldestXid instead. (Otherwise we'd try to + * access CLOG segments that have already been truncated away.) + */ + from = TransactionIdPrecedes(checkpoint->oldestXid, checkpoint->oldestActiveXid) + ? checkpoint->oldestActiveXid : checkpoint->oldestXid; + till = XidFromFullTransactionId(checkpoint->nextXid); + + /* + * To avoid "too many KnownAssignedXids" error later during replay, we + * limit number of collected transactions. This is a tradeoff: if we are + * willing to consume more of the KnownAssignedXids space for the XIDs + * now, that allows us to start up, but we might run out of space later. + * + * The size of the KnownAssignedXids array is TOTAL_MAX_CACHED_SUBXIDS, + * which is (PGPROC_MAX_CACHED_SUBXIDS + 1) * PROCARRAY_MAXPROCS). In + * PostgreSQL, that's always enough because the primary will always write + * an XLOG_XACT_ASSIGNMENT record if a transaction has more than + * PGPROC_MAX_CACHED_SUBXIDS subtransactions. Seeing that record allows + * the standby to mark the XIDs in pg_subtrans and removing them from the + * KnowingAssignedXids array. + * + * Here, we don't know which XIDs belong to subtransactions that have + * already been WAL-logged with an XLOG_XACT_ASSIGNMENT record. If we + * wanted to be totally safe and avoid the possibility of getting a "too + * many KnownAssignedXids" error later, we would have to limit ourselves + * to PGPROC_MAX_CACHED_SUBXIDS, which is not much. And that includes top + * transaction IDs too, because we cannot distinguish between top + * transaction IDs and subtransactions here. + * + * Somewhat arbitrarily, we use up to half of KnownAssignedXids. That + * strikes a sensible balance between being useful, and risking a "too + * many KnownAssignedXids" error later. + */ + max_xcnt = TOTAL_MAX_CACHED_SUBXIDS / 2; + + /* + * Collect XIDs of prepared transactions in an array. This includes only + * their top-level XIDs. We assume that StandbyRecoverPreparedTransactions + * has already been called, so we can find all the sub-transactions in + * pg_subtrans. + */ + PrescanPreparedTransactions(&prepared_xids, &n_prepared_xids); + qsort(prepared_xids, n_prepared_xids, sizeof(TransactionId), xidLogicalComparator); + + /* + * Scan the CLOG, collecting in-progress XIDs into 'restored_xids'. + */ + elog(DEBUG1, "scanning CLOG between %u and %u for in-progress XIDs", from, till); + restored_xids = (TransactionId *) palloc(max_xcnt * sizeof(TransactionId)); + n_restored_xids = 0; + next_prepared_idx = 0; + for (TransactionId xid = from; xid != till;) + { + XLogRecPtr xidlsn; + XidStatus xidstatus; + + xidstatus = TransactionIdGetStatus(xid, &xidlsn); + + /* + * "Merge" the prepared transactions into the restored_xids array as + * we go. The prepared transactions array is sorted. This is mostly + * a sanity check to ensure that all the prpeared transactions are + * seen as in-progress. (There is a check after the loop that we didn't + * miss any.) + */ + if (next_prepared_idx < n_prepared_xids && xid == prepared_xids[next_prepared_idx]) + { + /* + * This is a top-level transaction ID of a prepared transaction. + * Include it in the array. + */ + + /* sanity check */ + if (xidstatus != TRANSACTION_STATUS_IN_PROGRESS) + { + elog(LOG, "prepared transaction %u has unexpected status %X, cannot restore running-xacts from CLOG", + xid, xidstatus); + Assert(false); + goto fail; + } + + elog(DEBUG1, "XID %u: was next prepared xact (%d / %d)", xid, next_prepared_idx, n_prepared_xids); + next_prepared_idx++; + } + else if (xidstatus == TRANSACTION_STATUS_COMMITTED) + { + elog(DEBUG1, "XID %u: was committed", xid); + goto skip; + } + else if (xidstatus == TRANSACTION_STATUS_ABORTED) + { + elog(DEBUG1, "XID %u: was aborted", xid); + goto skip; + } + else if (xidstatus == TRANSACTION_STATUS_IN_PROGRESS) + { + /* + * In-progress transactions are included in the array. + * + * Except subtransactions of the prepared transactions. They are + * already set in pg_subtrans, and hence don't need to be tracked + * in the known-assigned XIDs array. + */ + if (n_prepared_xids > 0) + { + TransactionId parent = SubTransGetParent(xid); + + if (TransactionIdIsValid(parent)) + { + /* + * This is a subtransaction belonging to a prepared + * transaction. + * + * Sanity check that it is in the prepared XIDs array. It + * should be, because StandbyRecoverPreparedTransactions + * populated pg_subtrans, and no other XID should be set + * in it yet. (This also relies on the fact that + * StandbyRecoverPreparedTransactions sets the parent of + * each subxid to point directly to the top-level XID, + * rather than restoring the original subtransaction + * hierarchy.) + */ + if (bsearch(&parent, prepared_xids, next_prepared_idx, + sizeof(TransactionId), xidLogicalComparator) == NULL) + { + elog(LOG, "sub-XID %u has unexpected parent %u, cannot restore running-xacts from CLOG", + xid, parent); + Assert(false); + goto fail; + } + elog(DEBUG1, "XID %u: was a subtransaction of prepared xid %u", xid, parent); + goto skip; + } + } + + /* include it in the array */ + elog(DEBUG1, "XID %u: is in progress", xid); + } + else + { + /* + * SUB_COMMITTED is a transient state used at commit. We don't + * expect to see that here. + */ + elog(LOG, "XID %u has unexpected status %X in pg_xact, cannot restore running-xacts from CLOG", + xid, xidstatus); + Assert(false); + goto fail; + } + + if (n_restored_xids >= max_xcnt) + { + /* + * Overflowed. We won't be able to install the RunningTransactions + * snapshot. + */ + elog(LOG, "too many running xacts to restore from the CLOG; oldestXid=%u oldestActiveXid=%u nextXid %u", + checkpoint->oldestXid, checkpoint->oldestActiveXid, + XidFromFullTransactionId(checkpoint->nextXid)); + goto fail; + } + + restored_xids[n_restored_xids++] = xid; + + skip: + TransactionIdAdvance(xid); + continue; + } + + /* sanity check */ + if (next_prepared_idx != n_prepared_xids) + { + elog(LOG, "prepared transaction ID %u was not visited in the CLOG scan, cannot restore running-xacts from CLOG", + prepared_xids[next_prepared_idx]); + Assert(false); + goto fail; + } + + elog(LOG, "restored %d running xacts by scanning the CLOG; oldestXid=%u oldestActiveXid=%u nextXid %u", + n_restored_xids, checkpoint->oldestXid, checkpoint->oldestActiveXid, XidFromFullTransactionId(checkpoint->nextXid)); + *nxids = n_restored_xids; + *xids = restored_xids; + return true; + + fail: + *nxids = 0; + *xids = NULL; + if (restored_xids) + pfree(restored_xids); + if (prepared_xids) + pfree(prepared_xids); + return false; +} + void _PG_init(void) { @@ -288,6 +579,8 @@ _PG_init(void) pg_init_extension_server(); + restore_running_xacts_callback = RestoreRunningXactsFromClog; + /* * Important: This must happen after other parts of the extension are * loaded, otherwise any settings to GUCs that were set before the diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index a1cb1b51953c..e1c851435142 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -3856,7 +3856,9 @@ def stop_all(self) -> "EndpointFactory": return self - def new_replica(self, origin: Endpoint, endpoint_id: str, config_lines: Optional[List[str]]): + def new_replica( + self, origin: Endpoint, endpoint_id: str, config_lines: Optional[List[str]] = None + ): branch_name = origin.branch_name assert origin in self.endpoints assert branch_name is not None diff --git a/test_runner/fixtures/pageserver/utils.py b/test_runner/fixtures/pageserver/utils.py index 60535b759261..b75a480a637e 100644 --- a/test_runner/fixtures/pageserver/utils.py +++ b/test_runner/fixtures/pageserver/utils.py @@ -198,7 +198,7 @@ def wait_for_last_record_lsn( lsn: Lsn, ) -> Lsn: """waits for pageserver to catch up to a certain lsn, returns the last observed lsn.""" - for i in range(100): + for i in range(1000): current_lsn = last_record_lsn(pageserver_http, tenant, timeline) if current_lsn >= lsn: return current_lsn diff --git a/test_runner/regress/test_replica_start.py b/test_runner/regress/test_replica_start.py new file mode 100644 index 000000000000..17d476a8a690 --- /dev/null +++ b/test_runner/regress/test_replica_start.py @@ -0,0 +1,646 @@ +""" +In PostgreSQL, a standby always has to wait for a running-xacts WAL record to +arrive before it can start accepting queries. Furthermore, if there are +transactions with too many subxids (> 64) open to fit in the in-memory subxids +cache, the running-xacts record will be marked as "suboverflowed", and the +standby will need to also wait for the currently in-progress transactions to +finish. + +In Neon, we have an additional mechanism that scans the CLOG at server startup +to determine the list of running transactions, so that the standby can start up +immediately without waiting for the running-xacts record, but that mechanism +only works if the # of active (sub-)transactions is reasonably small. Otherwise +it falls back to waiting. Furthermore, it's somewhat optimistic in using up the +known-assigned XIDs array: if too many transactions with subxids are started in +the primary later, the replay in the replica will crash with "too many +KnownAssignedXids" error. + +This module contains tests for those various cases at standby startup: starting +from shutdown checkpoint, using the CLOG scanning mechanism, waiting for +running-xacts record and for in-progress transactions to finish etc. +""" + +import threading +from contextlib import closing + +import psycopg2 +import pytest +from fixtures.log_helper import log +from fixtures.neon_fixtures import NeonEnv, wait_for_last_flush_lsn, wait_replica_caughtup +from fixtures.pg_version import PgVersion +from fixtures.utils import query_scalar, wait_until + +CREATE_SUBXACTS_FUNC = """ +create or replace function create_subxacts(n integer) returns void as $$ +declare + i integer; +begin + for i in 1..n loop + begin + insert into t (payload) values (0); + exception + when others then + raise exception 'caught something: %', sqlerrm; + end; + end loop; +end; $$ language plpgsql +""" + + +def test_replica_start_scan_clog(neon_simple_env: NeonEnv): + """ + Test the CLOG-scanning mechanism at hot standby startup. There is one + transaction active in the primary when the standby is started. The primary + is killed before it has a chance to write a running-xacts record. The + CLOG-scanning at neon startup allows the standby to start up anyway. + + See the module docstring for background. + """ + + # Initialize the primary, a test table, and a helper function to create lots + # of subtransactions. + env = neon_simple_env + primary = env.endpoints.create_start(branch_name="main", endpoint_id="primary") + primary_conn = primary.connect() + primary_cur = primary_conn.cursor() + primary_cur.execute("CREATE EXTENSION neon_test_utils") + primary_cur.execute("create table t(pk serial primary key, payload integer)") + primary_cur.execute(CREATE_SUBXACTS_FUNC) + primary_cur.execute("select pg_switch_wal()") + + # Start a transaction in the primary. Leave the transaction open. + # + # The transaction has some subtransactions, but not too many to cause the + # CLOG-scanning mechanism to give up. + primary_cur.execute("begin") + primary_cur.execute("select create_subxacts(50)") + + # Wait for the WAL to be flushed, but then immediately kill the primary, + # before it has a chance to generate a running-xacts record. + primary_cur.execute("select neon_xlogflush()") + wait_for_last_flush_lsn(env, primary, env.initial_tenant, env.initial_timeline) + primary.stop(mode="immediate") + + # Create a replica. It should start up normally, thanks to the CLOG-scanning + # mechanism. + secondary = env.endpoints.new_replica_start(origin=primary, endpoint_id="secondary") + + # The transaction did not commit, so it should not be visible in the secondary + secondary_conn = secondary.connect() + secondary_cur = secondary_conn.cursor() + secondary_cur.execute("select count(*) from t") + assert secondary_cur.fetchone() == (0,) + + +def test_replica_start_scan_clog_crashed_xids(neon_simple_env: NeonEnv): + """ + Test the CLOG-scanning mechanism at hot standby startup, after + leaving behind crashed transactions. + + See the module docstring for background. + """ + + # Initialize the primary, a test table, and a helper function to create lots + # of subtransactions. + env = neon_simple_env + primary = env.endpoints.create_start(branch_name="main", endpoint_id="primary") + primary_conn = primary.connect() + primary_cur = primary_conn.cursor() + primary_cur.execute("create table t(pk serial primary key, payload integer)") + primary_cur.execute(CREATE_SUBXACTS_FUNC) + primary_cur.execute("select pg_switch_wal()") + + # Consume a lot of XIDs, then kill Postgres without giving it a + # chance to write abort records for them. + primary_cur.execute("begin") + primary_cur.execute("select create_subxacts(100000)") + primary.stop(mode="immediate") + + # Restart the primary. Do some light work, and shut it down cleanly + primary.start() + primary_conn = primary.connect() + primary_cur = primary_conn.cursor() + primary_cur.execute("insert into t (payload) values (0)") + primary.stop(mode="fast") + + # Create a replica. It should start up normally, thanks to the CLOG-scanning + # mechanism. (Restarting the primary writes a checkpoint and/or running-xacts + # record, which allows the standby to know that the crashed XIDs are aborted) + secondary = env.endpoints.new_replica_start(origin=primary, endpoint_id="secondary") + + secondary_conn = secondary.connect() + secondary_cur = secondary_conn.cursor() + secondary_cur.execute("select count(*) from t") + assert secondary_cur.fetchone() == (1,) + + +def test_replica_start_at_running_xacts(neon_simple_env: NeonEnv, pg_version): + """ + Test that starting a replica works right after the primary has + created a running-xacts record. This may seem like a trivial case, + but during development, we had a bug that was triggered by having + oldestActiveXid == nextXid. Starting right after a running-xacts + record is one way to test that case. + + See the module docstring for background. + """ + env = neon_simple_env + + if env.pg_version == PgVersion.V14 or env.pg_version == PgVersion.V15: + pytest.skip("pg_log_standby_snapshot() function is available only in PG16") + + primary = env.endpoints.create_start(branch_name="main", endpoint_id="primary") + primary_conn = primary.connect() + primary_cur = primary_conn.cursor() + + primary_cur.execute("CREATE EXTENSION neon_test_utils") + primary_cur.execute("select pg_log_standby_snapshot()") + primary_cur.execute("select neon_xlogflush()") + wait_for_last_flush_lsn(env, primary, env.initial_tenant, env.initial_timeline) + + secondary = env.endpoints.new_replica_start(origin=primary, endpoint_id="secondary") + + secondary_conn = secondary.connect() + secondary_cur = secondary_conn.cursor() + secondary_cur.execute("select 123") + assert secondary_cur.fetchone() == (123,) + + +def test_replica_start_wait_subxids_finish(neon_simple_env: NeonEnv): + """ + Test replica startup when there are a lot of (sub)transactions active in the + primary. That's too many for the CLOG-scanning mechanism to handle, so the + replica has to wait for the large transaction to finish before it starts to + accept queries. + + After replica startup, test MVCC with transactions that were in-progress + when the replica was started. + + See the module docstring for background. + """ + + # Initialize the primary, a test table, and a helper function to create + # lots of subtransactions. + env = neon_simple_env + primary = env.endpoints.create_start(branch_name="main", endpoint_id="primary") + primary_conn = primary.connect() + primary_cur = primary_conn.cursor() + primary_cur.execute("create table t(pk serial primary key, payload integer)") + primary_cur.execute(CREATE_SUBXACTS_FUNC) + + # Start a transaction with 100000 subtransactions, and leave it open. That's + # too many to fit in the "known-assigned XIDs array" in the replica, and + # also too many to fit in the subxid caches so the running-xacts record will + # also overflow. + primary_cur.execute("begin") + primary_cur.execute("select create_subxacts(100000)") + + # Start another, smaller transaction in the primary. We'll come back to this + # later. + primary_conn2 = primary.connect() + primary_cur2 = primary_conn2.cursor() + primary_cur2.execute("begin") + primary_cur2.execute("insert into t (payload) values (0)") + + # Create a replica. but before that, wait for the wal to be flushed to + # safekeepers, so that the replica is started at a point where the large + # transaction is already active. (The whole transaction might not be flushed + # yet, but that's OK.) + # + # Start it in a separate thread, so that we can do other stuff while it's + # blocked waiting for the startup to finish. + wait_for_last_flush_lsn(env, primary, env.initial_tenant, env.initial_timeline) + secondary = env.endpoints.new_replica(origin=primary, endpoint_id="secondary") + start_secondary_thread = threading.Thread(target=secondary.start) + start_secondary_thread.start() + + # Verify that the replica has otherwise started up, but cannot start + # accepting queries yet. + log.info("Waiting 5 s to verify that the secondary does not start") + start_secondary_thread.join(5) + assert secondary.log_contains("consistent recovery state reached") + assert secondary.log_contains("started streaming WAL from primary") + # The "redo starts" message is printed when the first WAL record is + # received. It might or might not be present in the log depending on how + # far exactly the WAL was flushed when the replica was started, and whether + # background activity caused any more WAL records to be flushed on the + # primary afterwards. + # + # assert secondary.log_contains("redo # starts") + + # should not be open for connections yet + assert start_secondary_thread.is_alive() + assert not secondary.is_running() + assert not secondary.log_contains("database system is ready to accept read-only connections") + + # Commit the large transaction in the primary. + # + # Within the next 15 s, the primary should write a new running-xacts record + # to the WAL which shows the transaction as completed. Once the replica + # replays that record, it will start accepting queries. + primary_cur.execute("commit") + start_secondary_thread.join() + + # Verify that the large transaction is correctly visible in the secondary + # (but not the second, small transaction, which is still in-progress!) + secondary_conn = secondary.connect() + secondary_cur = secondary_conn.cursor() + secondary_cur.execute("select count(*) from t") + assert secondary_cur.fetchone() == (100000,) + + # Perform some more MVCC testing using the second transaction that was + # started in the primary before the replica was created + primary_cur2.execute("select create_subxacts(10000)") + + # The second transaction still hasn't committed + wait_replica_caughtup(primary, secondary) + secondary_cur.execute("BEGIN ISOLATION LEVEL REPEATABLE READ") + secondary_cur.execute("select count(*) from t") + assert secondary_cur.fetchone() == (100000,) + + # Commit the second transaction in the primary + primary_cur2.execute("commit") + + # Should still be invisible to the old snapshot + wait_replica_caughtup(primary, secondary) + secondary_cur.execute("select count(*) from t") + assert secondary_cur.fetchone() == (100000,) + + # Commit the REPEATABLE READ transaction in the replica. Both + # primary transactions should now be visible to a new snapshot. + secondary_cur.execute("commit") + secondary_cur.execute("select count(*) from t") + assert secondary_cur.fetchone() == (110001,) + + +def test_replica_too_many_known_assigned_xids(neon_simple_env: NeonEnv): + """ + The CLOG-scanning mechanism fills the known-assigned XIDs array + optimistically at standby startup, betting that it can still fit + upcoming transactions replayed later from the WAL in the + array. This test tests what happens when that bet fails and the + known-assigned XID array fills up after the standby has already + been started. The WAL redo will fail with an error: + + FATAL: too many KnownAssignedXids + CONTEXT: WAL redo at 0/1895CB0 for neon/INSERT: off: 25, flags: 0x08; blkref #0: rel 1663/5/16385, blk 64 + + which causes the standby to shut down. + + See the module docstring for background. + """ + + # Initialize the primary, a test table, and a helper function to create lots + # of subtransactions. + env = neon_simple_env + primary = env.endpoints.create_start(branch_name="main", endpoint_id="primary") + primary_conn = primary.connect() + primary_cur = primary_conn.cursor() + primary_cur.execute("CREATE EXTENSION neon_test_utils") + primary_cur.execute("create table t(pk serial primary key, payload integer)") + primary_cur.execute(CREATE_SUBXACTS_FUNC) + + # Determine how many connections we can use + primary_cur.execute("show max_connections") + max_connections = int(primary_cur.fetchall()[0][0]) + primary_cur.execute("show superuser_reserved_connections") + superuser_reserved_connections = int(primary_cur.fetchall()[0][0]) + n_connections = max_connections - superuser_reserved_connections + n_subxids = 200 + + # Start one top transaction in primary, with lots of subtransactions. This + # uses up much of the known-assigned XIDs space in the standby, but doesn't + # cause it to overflow. + large_p_conn = primary.connect() + large_p_cur = large_p_conn.cursor() + large_p_cur.execute("begin") + large_p_cur.execute(f"select create_subxacts({max_connections} * 30)") + + with closing(primary.connect()) as small_p_conn: + with small_p_conn.cursor() as small_p_cur: + small_p_cur.execute("select create_subxacts(1)") + + # Create a replica at this LSN + primary_cur.execute("select neon_xlogflush()") + wait_for_last_flush_lsn(env, primary, env.initial_tenant, env.initial_timeline) + secondary = env.endpoints.new_replica_start(origin=primary, endpoint_id="secondary") + secondary_conn = secondary.connect() + secondary_cur = secondary_conn.cursor() + + # The transaction in primary has not committed yet. + wait_replica_caughtup(primary, secondary) + secondary_cur.execute("select count(*) from t") + assert secondary_cur.fetchone() == (1,) + + # Start max number of top transactions in primary, with a lot of + # subtransactions each. We add the subtransactions to each top transaction + # in a round-robin fashion, instead of adding a lot of subtransactions to + # one top transaction at a time. This way, we will have the max number of + # subtransactions in the in-memory subxid cache of each top transaction, + # until they all overflow. + # + # Currently, PGPROC_MAX_CACHED_SUBXIDS == 64, so this will overflow the all + # the subxid caches after creating 64 subxids in each top transaction. The + # point just before the caches have overflowed is the most interesting point + # in time, but we'll keep going beyond that, to ensure that this test is + # robust even if PGPROC_MAX_CACHED_SUBXIDS changes. + p_curs = [] + for _ in range(0, n_connections): + p_cur = primary.connect().cursor() + p_cur.execute("begin") + p_curs.append(p_cur) + + for _subxid in range(0, n_subxids): + for i in range(0, n_connections): + p_curs[i].execute("select create_subxacts(1)") + + # Commit all the transactions in the primary + for i in range(0, n_connections): + p_curs[i].execute("commit") + large_p_cur.execute("commit") + + # Wait until the replica crashes with "too many KnownAssignedXids" error. + def check_replica_crashed(): + try: + secondary.connect() + except psycopg2.Error: + # Once the connection fails, return success + return None + raise RuntimeError("connection succeeded") + + wait_until(20, 0.5, check_replica_crashed) + assert secondary.log_contains("too many KnownAssignedXids") + + # Replica is crashed, so ignore stop result + secondary.check_stop_result = False + + +def test_replica_start_repro_visibility_bug(neon_simple_env: NeonEnv): + """ + Before PR #7288, a hot standby in neon incorrectly started up + immediately, before it had received a running-xacts record. That + led to visibility bugs if there were active transactions in the + primary. This test reproduces the incorrect query results and + incorrectly set hint bits, before that was fixed. + """ + env = neon_simple_env + + primary = env.endpoints.create_start(branch_name="main", endpoint_id="primary") + p_cur = primary.connect().cursor() + + p_cur.execute("begin") + p_cur.execute("create table t(pk integer primary key, payload integer)") + p_cur.execute("insert into t values (generate_series(1,100000), 0)") + + secondary = env.endpoints.new_replica_start(origin=primary, endpoint_id="secondary") + wait_replica_caughtup(primary, secondary) + s_cur = secondary.connect().cursor() + + # Set hint bits for pg_class tuples. If primary's transaction is + # not marked as in-progress in MVCC snapshot, then XMIN_INVALID + # hint bit will be set for table's 't' tuple, making it invisible + # even after the commit record is replayed later. + s_cur.execute("select * from pg_class") + + p_cur.execute("commit") + wait_replica_caughtup(primary, secondary) + s_cur.execute("select * from t where pk = 1") + assert s_cur.fetchone() == (1, 0) + + +@pytest.mark.parametrize("shutdown", [True, False]) +def test_replica_start_with_prepared_xacts(neon_simple_env: NeonEnv, shutdown: bool): + """ + Test the CLOG-scanning mechanism at hot standby startup in the presence of + prepared transactions. + + This test is run in two variants: one where the primary server is shut down + before starting the secondary, or not. + """ + + # Initialize the primary, a test table, and a helper function to create lots + # of subtransactions. + env = neon_simple_env + primary = env.endpoints.create_start( + branch_name="main", endpoint_id="primary", config_lines=["max_prepared_transactions=5"] + ) + primary_conn = primary.connect() + primary_cur = primary_conn.cursor() + primary_cur.execute("CREATE EXTENSION neon_test_utils") + primary_cur.execute("create table t(pk serial primary key, payload integer)") + primary_cur.execute("create table t1(pk integer primary key)") + primary_cur.execute("create table t2(pk integer primary key)") + primary_cur.execute(CREATE_SUBXACTS_FUNC) + + # Prepare a transaction for two-phase commit + primary_cur.execute("begin") + primary_cur.execute("insert into t1 values (1)") + primary_cur.execute("prepare transaction 't1'") + + # Prepare another transaction for two-phase commit, with a subtransaction + primary_cur.execute("begin") + primary_cur.execute("insert into t2 values (2)") + primary_cur.execute("savepoint sp") + primary_cur.execute("insert into t2 values (3)") + primary_cur.execute("prepare transaction 't2'") + + # Start a transaction in the primary. Leave the transaction open. + # + # The transaction has some subtransactions, but not too many to cause the + # CLOG-scanning mechanism to give up. + primary_cur.execute("begin") + primary_cur.execute("select create_subxacts(50)") + + # Wait for the WAL to be flushed + primary_cur.execute("select neon_xlogflush()") + wait_for_last_flush_lsn(env, primary, env.initial_tenant, env.initial_timeline) + + if shutdown: + primary.stop(mode="fast") + + # Create a replica. It should start up normally, thanks to the CLOG-scanning + # mechanism. + secondary = env.endpoints.new_replica_start( + origin=primary, endpoint_id="secondary", config_lines=["max_prepared_transactions=5"] + ) + + # The transaction did not commit, so it should not be visible in the secondary + secondary_conn = secondary.connect() + secondary_cur = secondary_conn.cursor() + secondary_cur.execute("select count(*) from t") + assert secondary_cur.fetchone() == (0,) + secondary_cur.execute("select count(*) from t1") + assert secondary_cur.fetchone() == (0,) + secondary_cur.execute("select count(*) from t2") + assert secondary_cur.fetchone() == (0,) + + if shutdown: + primary.start() + primary_conn = primary.connect() + primary_cur = primary_conn.cursor() + else: + primary_cur.execute("commit") + primary_cur.execute("commit prepared 't1'") + primary_cur.execute("commit prepared 't2'") + + wait_replica_caughtup(primary, secondary) + + secondary_cur.execute("select count(*) from t") + if shutdown: + assert secondary_cur.fetchone() == (0,) + else: + assert secondary_cur.fetchone() == (50,) + secondary_cur.execute("select * from t1") + assert secondary_cur.fetchall() == [(1,)] + secondary_cur.execute("select * from t2") + assert secondary_cur.fetchall() == [(2,), (3,)] + + +def test_replica_start_with_prepared_xacts_with_subxacts(neon_simple_env: NeonEnv): + """ + Test the CLOG-scanning mechanism at hot standby startup in the presence of + prepared transactions, with subtransactions. + """ + + # Initialize the primary, a test table, and a helper function to create lots + # of subtransactions. + env = neon_simple_env + primary = env.endpoints.create_start( + branch_name="main", endpoint_id="primary", config_lines=["max_prepared_transactions=5"] + ) + primary_conn = primary.connect() + primary_cur = primary_conn.cursor() + + # Install extension containing function needed for test + primary_cur.execute("CREATE EXTENSION neon_test_utils") + + primary_cur.execute("create table t(pk serial primary key, payload integer)") + primary_cur.execute(CREATE_SUBXACTS_FUNC) + + # Advance nextXid close to the beginning of the next pg_subtrans segment (2^16 XIDs) + # + # This is interesting, because it tests that pg_subtrans is initialized correctly + # at standby startup. (We had a bug where it didn't at one point during development.) + while True: + xid = int(query_scalar(primary_cur, "SELECT txid_current()")) + log.info(f"xid now {xid}") + # Consume 500 transactions at a time until we get close + if xid < 65535 - 600: + primary_cur.execute("select test_consume_xids(500);") + else: + break + primary_cur.execute("checkpoint") + + # Prepare a transaction for two-phase commit + primary_cur.execute("begin") + primary_cur.execute("select create_subxacts(1000)") + primary_cur.execute("prepare transaction 't1'") + + # Wait for the WAL to be flushed, and stop the primary + wait_for_last_flush_lsn(env, primary, env.initial_tenant, env.initial_timeline) + primary.stop(mode="fast") + + # Create a replica. It should start up normally, thanks to the CLOG-scanning + # mechanism. + secondary = env.endpoints.new_replica_start( + origin=primary, endpoint_id="secondary", config_lines=["max_prepared_transactions=5"] + ) + + # The transaction did not commit, so it should not be visible in the secondary + secondary_conn = secondary.connect() + secondary_cur = secondary_conn.cursor() + secondary_cur.execute("select count(*) from t") + assert secondary_cur.fetchone() == (0,) + + primary.start() + + # Open a lot of subtransactions in the primary, causing the subxids cache to overflow + primary_conn = primary.connect() + primary_cur = primary_conn.cursor() + primary_cur.execute("select create_subxacts(100000)") + + wait_replica_caughtup(primary, secondary) + + secondary_cur.execute("select count(*) from t") + assert secondary_cur.fetchone() == (100000,) + + primary_cur.execute("commit prepared 't1'") + + wait_replica_caughtup(primary, secondary) + secondary_cur.execute("select count(*) from t") + assert secondary_cur.fetchone() == (101000,) + + +def test_replica_start_with_prepared_xacts_with_many_subxacts(neon_simple_env: NeonEnv): + """ + Test the CLOG-scanning mechanism at hot standby startup in the presence of + prepared transactions, with lots of subtransactions. + + Like test_replica_start_with_prepared_xacts_with_subxacts, but with more + subxacts, to test that the prepared transaction's subxids don't consume + space in the known-assigned XIDs array. (They are set in pg_subtrans + instead) + """ + + # Initialize the primary, a test table, and a helper function to create lots + # of subtransactions. + env = neon_simple_env + primary = env.endpoints.create_start( + branch_name="main", endpoint_id="primary", config_lines=["max_prepared_transactions=5"] + ) + primary_conn = primary.connect() + primary_cur = primary_conn.cursor() + + # Install extension containing function needed for test + primary_cur.execute("CREATE EXTENSION neon_test_utils") + + primary_cur.execute("create table t(pk serial primary key, payload integer)") + primary_cur.execute(CREATE_SUBXACTS_FUNC) + + # Prepare a transaction for two-phase commit, with lots of subxids + primary_cur.execute("begin") + primary_cur.execute("select create_subxacts(50000)") + + # to make things a bit more varied, intersperse a few other XIDs in between + # the prepared transaction's sub-XIDs + with primary.connect().cursor() as primary_cur2: + primary_cur2.execute("insert into t (payload) values (123)") + primary_cur2.execute("begin; insert into t (payload) values (-1); rollback") + + primary_cur.execute("select create_subxacts(50000)") + primary_cur.execute("prepare transaction 't1'") + + # Wait for the WAL to be flushed + wait_for_last_flush_lsn(env, primary, env.initial_tenant, env.initial_timeline) + + primary.stop(mode="fast") + + # Create a replica. It should start up normally, thanks to the CLOG-scanning + # mechanism. + secondary = env.endpoints.new_replica_start( + origin=primary, endpoint_id="secondary", config_lines=["max_prepared_transactions=5"] + ) + + # The transaction did not commit, so it should not be visible in the secondary + secondary_conn = secondary.connect() + secondary_cur = secondary_conn.cursor() + secondary_cur.execute("select count(*) from t") + assert secondary_cur.fetchone() == (1,) + + primary.start() + + # Open a lot of subtransactions in the primary, causing the subxids cache to overflow + primary_conn = primary.connect() + primary_cur = primary_conn.cursor() + primary_cur.execute("select create_subxacts(100000)") + + wait_replica_caughtup(primary, secondary) + + secondary_cur.execute("select count(*) from t") + assert secondary_cur.fetchone() == (100001,) + + primary_cur.execute("commit prepared 't1'") + + wait_replica_caughtup(primary, secondary) + secondary_cur.execute("select count(*) from t") + assert secondary_cur.fetchone() == (200001,) diff --git a/test_runner/regress/test_replication_start.py b/test_runner/regress/test_replication_start.py deleted file mode 100644 index 236074599021..000000000000 --- a/test_runner/regress/test_replication_start.py +++ /dev/null @@ -1,32 +0,0 @@ -import pytest -from fixtures.log_helper import log -from fixtures.neon_fixtures import NeonEnv, wait_replica_caughtup - - -@pytest.mark.xfail -def test_replication_start(neon_simple_env: NeonEnv): - env = neon_simple_env - - with env.endpoints.create_start(branch_name="main", endpoint_id="primary") as primary: - with primary.connect() as p_con: - with p_con.cursor() as p_cur: - p_cur.execute("begin") - p_cur.execute("create table t(pk integer primary key, payload integer)") - p_cur.execute("insert into t values (generate_series(1,100000), 0)") - p_cur.execute("select txid_current()") - xid = p_cur.fetchall()[0][0] - log.info(f"Master transaction {xid}") - with env.endpoints.new_replica_start( - origin=primary, endpoint_id="secondary" - ) as secondary: - wait_replica_caughtup(primary, secondary) - with secondary.connect() as s_con: - with s_con.cursor() as s_cur: - # Enforce setting hint bits for pg_class tuples. - # If master's transaction is not marked as in-progress in MVCC snapshot, - # then XMIN_INVALID hint bit will be set for table's 't' tuple makeing it invisible. - s_cur.execute("select * from pg_class") - p_cur.execute("commit") - wait_replica_caughtup(primary, secondary) - s_cur.execute("select * from t where pk = 1") - assert s_cur.fetchone() == (1, 0) diff --git a/vendor/postgres-v14 b/vendor/postgres-v14 index 223dd925959f..ad73770c446e 160000 --- a/vendor/postgres-v14 +++ b/vendor/postgres-v14 @@ -1 +1 @@ -Subproject commit 223dd925959f8124711dd3d867dc8ba6629d52c0 +Subproject commit ad73770c446ea361f43e4f0404798b7e5e7a62d8 diff --git a/vendor/postgres-v15 b/vendor/postgres-v15 index f54d7373eb0d..4874c8e52ed3 160000 --- a/vendor/postgres-v15 +++ b/vendor/postgres-v15 @@ -1 +1 @@ -Subproject commit f54d7373eb0de5a54bce2becdb1c801026c7edff +Subproject commit 4874c8e52ed349a9f8290bbdcd91eb92677a5d24 diff --git a/vendor/postgres-v16 b/vendor/postgres-v16 index e06bebc75306..b810fdfcbb59 160000 --- a/vendor/postgres-v16 +++ b/vendor/postgres-v16 @@ -1 +1 @@ -Subproject commit e06bebc75306b583e758b52c95946d41109239b2 +Subproject commit b810fdfcbb59afea7ea7bbe0cf94eaccb55a2ea2 diff --git a/vendor/revisions.json b/vendor/revisions.json index 574e3719340e..da49ff19c3ec 100644 --- a/vendor/revisions.json +++ b/vendor/revisions.json @@ -1,5 +1,5 @@ { - "v16": ["16.3", "e06bebc75306b583e758b52c95946d41109239b2"], - "v15": ["15.7", "f54d7373eb0de5a54bce2becdb1c801026c7edff"], - "v14": ["14.12", "223dd925959f8124711dd3d867dc8ba6629d52c0"] + "v16": ["16.3", "b810fdfcbb59afea7ea7bbe0cf94eaccb55a2ea2"], + "v15": ["15.7", "4874c8e52ed349a9f8290bbdcd91eb92677a5d24"], + "v14": ["14.12", "ad73770c446ea361f43e4f0404798b7e5e7a62d8"] } From aea5cfe21e62b4df285c0c55c12f79df8fbde1a4 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 1 Jul 2024 12:48:20 +0100 Subject: [PATCH 004/194] pageserver: add metric `pageserver_secondary_resident_physical_size` (#8204) ## Problem We lack visibility of how much local disk space is used by secondary tenant locations Close: https://github.com/neondatabase/neon/issues/8181 ## Summary of changes - Add `pageserver_secondary_resident_physical_size`, tagged by tenant - Register & de-register label sets from SecondaryTenant - Add+use wrappers in SecondaryDetail that update metrics when adding+removing layers/timelines --- pageserver/src/metrics.rs | 11 +- pageserver/src/tenant/secondary.rs | 37 +++- pageserver/src/tenant/secondary/downloader.rs | 173 ++++++++++++++---- 3 files changed, 171 insertions(+), 50 deletions(-) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index f5aca6dfb36c..9cd7ffa0426c 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -476,7 +476,7 @@ static STANDBY_HORIZON: Lazy = Lazy::new(|| { static RESIDENT_PHYSICAL_SIZE: Lazy = Lazy::new(|| { register_uint_gauge_vec!( "pageserver_resident_physical_size", - "The size of the layer files present in the pageserver's filesystem.", + "The size of the layer files present in the pageserver's filesystem, for attached locations.", &["tenant_id", "shard_id", "timeline_id"] ) .expect("failed to define a metric") @@ -1691,6 +1691,15 @@ pub(crate) static SECONDARY_MODE: Lazy = Lazy::new(|| { } }); +pub(crate) static SECONDARY_RESIDENT_PHYSICAL_SIZE: Lazy = Lazy::new(|| { + register_uint_gauge_vec!( + "pageserver_secondary_resident_physical_size", + "The size of the layer files present in the pageserver's filesystem, for secondary locations.", + &["tenant_id", "shard_id"] + ) + .expect("failed to define a metric") +}); + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum RemoteOpKind { Upload, diff --git a/pageserver/src/tenant/secondary.rs b/pageserver/src/tenant/secondary.rs index af6840f525ae..a233d11c4a11 100644 --- a/pageserver/src/tenant/secondary.rs +++ b/pageserver/src/tenant/secondary.rs @@ -23,6 +23,8 @@ use super::{ storage_layer::LayerName, }; +use crate::metrics::SECONDARY_RESIDENT_PHYSICAL_SIZE; +use metrics::UIntGauge; use pageserver_api::{ models, shard::{ShardIdentity, TenantShardId}, @@ -99,6 +101,17 @@ pub(crate) struct SecondaryTenant { // Public state indicating overall progress of downloads relative to the last heatmap seen pub(crate) progress: std::sync::Mutex, + + // Sum of layer sizes on local disk + pub(super) resident_size_metric: UIntGauge, +} + +impl Drop for SecondaryTenant { + fn drop(&mut self) { + let tenant_id = self.tenant_shard_id.tenant_id.to_string(); + let shard_id = format!("{}", self.tenant_shard_id.shard_slug()); + let _ = SECONDARY_RESIDENT_PHYSICAL_SIZE.remove_label_values(&[&tenant_id, &shard_id]); + } } impl SecondaryTenant { @@ -108,6 +121,12 @@ impl SecondaryTenant { tenant_conf: TenantConfOpt, config: &SecondaryLocationConfig, ) -> Arc { + let tenant_id = tenant_shard_id.tenant_id.to_string(); + let shard_id = format!("{}", tenant_shard_id.shard_slug()); + let resident_size_metric = SECONDARY_RESIDENT_PHYSICAL_SIZE + .get_metric_with_label_values(&[&tenant_id, &shard_id]) + .unwrap(); + Arc::new(Self { tenant_shard_id, // todo: shall we make this a descendent of the @@ -123,6 +142,8 @@ impl SecondaryTenant { detail: std::sync::Mutex::new(SecondaryDetail::new(config.clone())), progress: std::sync::Mutex::default(), + + resident_size_metric, }) } @@ -211,16 +232,12 @@ impl SecondaryTenant { // have to 100% match what is on disk, because it's a best-effort warming // of the cache. let mut detail = this.detail.lock().unwrap(); - if let Some(timeline_detail) = detail.timelines.get_mut(&timeline_id) { - let removed = timeline_detail.on_disk_layers.remove(&name); - - // We might race with removal of the same layer during downloads, if it was removed - // from the heatmap. If we see that the OnDiskState is gone, then no need to - // do a physical deletion or store in evicted_at. - if let Some(removed) = removed { - removed.remove_blocking(); - timeline_detail.evicted_at.insert(name, now); - } + if let Some(removed) = + detail.evict_layer(name, &timeline_id, now, &this.resident_size_metric) + { + // We might race with removal of the same layer during downloads, so finding the layer we + // were trying to remove is optional. Only issue the disk I/O to remove it if we found it. + removed.remove_blocking(); } }) .await diff --git a/pageserver/src/tenant/secondary/downloader.rs b/pageserver/src/tenant/secondary/downloader.rs index f6f30641dbbb..27439d4f030d 100644 --- a/pageserver/src/tenant/secondary/downloader.rs +++ b/pageserver/src/tenant/secondary/downloader.rs @@ -46,6 +46,7 @@ use crate::tenant::{ use camino::Utf8PathBuf; use chrono::format::{DelayedFormat, StrftimeItems}; use futures::Future; +use metrics::UIntGauge; use pageserver_api::models::SecondaryProgress; use pageserver_api::shard::TenantShardId; use remote_storage::{DownloadError, Etag, GenericRemoteStorage}; @@ -131,16 +132,66 @@ impl OnDiskState { .or_else(fs_ext::ignore_not_found) .fatal_err("Deleting secondary layer") } + + pub(crate) fn file_size(&self) -> u64 { + self.metadata.file_size + } } #[derive(Debug, Clone, Default)] pub(super) struct SecondaryDetailTimeline { - pub(super) on_disk_layers: HashMap, + on_disk_layers: HashMap, /// We remember when layers were evicted, to prevent re-downloading them. pub(super) evicted_at: HashMap, } +impl SecondaryDetailTimeline { + pub(super) fn remove_layer( + &mut self, + name: &LayerName, + resident_metric: &UIntGauge, + ) -> Option { + let removed = self.on_disk_layers.remove(name); + if let Some(removed) = &removed { + resident_metric.sub(removed.file_size()); + } + removed + } + + /// `local_path` + fn touch_layer( + &mut self, + conf: &'static PageServerConf, + tenant_shard_id: &TenantShardId, + timeline_id: &TimelineId, + touched: &HeatMapLayer, + resident_metric: &UIntGauge, + local_path: F, + ) where + F: FnOnce() -> Utf8PathBuf, + { + use std::collections::hash_map::Entry; + match self.on_disk_layers.entry(touched.name.clone()) { + Entry::Occupied(mut v) => { + v.get_mut().access_time = touched.access_time; + } + Entry::Vacant(e) => { + e.insert(OnDiskState::new( + conf, + tenant_shard_id, + timeline_id, + touched.name.clone(), + touched.metadata.clone(), + touched.access_time, + local_path(), + )); + resident_metric.add(touched.metadata.file_size); + } + } + } +} + // Aspects of a heatmap that we remember after downloading it #[derive(Clone, Debug)] struct DownloadSummary { @@ -158,7 +209,7 @@ pub(super) struct SecondaryDetail { last_download: Option, next_download: Option, - pub(super) timelines: HashMap, + timelines: HashMap, } /// Helper for logging SystemTime @@ -191,6 +242,38 @@ impl SecondaryDetail { } } + pub(super) fn evict_layer( + &mut self, + name: LayerName, + timeline_id: &TimelineId, + now: SystemTime, + resident_metric: &UIntGauge, + ) -> Option { + let timeline = self.timelines.get_mut(timeline_id)?; + let removed = timeline.remove_layer(&name, resident_metric); + if removed.is_some() { + timeline.evicted_at.insert(name, now); + } + removed + } + + pub(super) fn remove_timeline( + &mut self, + timeline_id: &TimelineId, + resident_metric: &UIntGauge, + ) { + let removed = self.timelines.remove(timeline_id); + if let Some(removed) = removed { + resident_metric.sub( + removed + .on_disk_layers + .values() + .map(|l| l.metadata.file_size) + .sum(), + ); + } + } + /// Additionally returns the total number of layers, used for more stable relative access time /// based eviction. pub(super) fn get_layers_for_eviction( @@ -601,8 +684,13 @@ impl<'a> TenantDownloader<'a> { Some(t) => t, None => { // We have no existing state: need to scan local disk for layers first. - let timeline_state = - init_timeline_state(self.conf, tenant_shard_id, timeline).await; + let timeline_state = init_timeline_state( + self.conf, + tenant_shard_id, + timeline, + &self.secondary_state.resident_size_metric, + ) + .await; // Re-acquire detail lock now that we're done with async load from local FS self.secondary_state @@ -671,6 +759,25 @@ impl<'a> TenantDownloader<'a> { .await?; } + // Metrics consistency check in testing builds + if cfg!(feature = "testing") { + let detail = self.secondary_state.detail.lock().unwrap(); + let resident_size = detail + .timelines + .values() + .map(|tl| { + tl.on_disk_layers + .values() + .map(|v| v.metadata.file_size) + .sum::() + }) + .sum::(); + assert_eq!( + resident_size, + self.secondary_state.resident_size_metric.get() + ); + } + // Only update last_etag after a full successful download: this way will not skip // the next download, even if the heatmap's actual etag is unchanged. self.secondary_state.detail.lock().unwrap().last_download = Some(DownloadSummary { @@ -783,7 +890,7 @@ impl<'a> TenantDownloader<'a> { for delete_timeline in &delete_timelines { // We haven't removed from disk yet, but optimistically remove from in-memory state: if removal // from disk fails that will be a fatal error. - detail.timelines.remove(delete_timeline); + detail.remove_timeline(delete_timeline, &self.secondary_state.resident_size_metric); } } @@ -801,7 +908,7 @@ impl<'a> TenantDownloader<'a> { let Some(timeline_state) = detail.timelines.get_mut(&timeline_id) else { continue; }; - timeline_state.on_disk_layers.remove(&layer_name); + timeline_state.remove_layer(&layer_name, &self.secondary_state.resident_size_metric); } for timeline_id in delete_timelines { @@ -1000,33 +1107,24 @@ impl<'a> TenantDownloader<'a> { let timeline_detail = detail.timelines.entry(timeline_id).or_default(); tracing::info!("Wrote timeline_detail for {} touched layers", touched.len()); - - for t in touched { - use std::collections::hash_map::Entry; - match timeline_detail.on_disk_layers.entry(t.name.clone()) { - Entry::Occupied(mut v) => { - v.get_mut().access_time = t.access_time; - } - Entry::Vacant(e) => { - let local_path = local_layer_path( + touched.into_iter().for_each(|t| { + timeline_detail.touch_layer( + self.conf, + tenant_shard_id, + &timeline_id, + &t, + &self.secondary_state.resident_size_metric, + || { + local_layer_path( self.conf, tenant_shard_id, &timeline_id, &t.name, &t.metadata.generation, - ); - e.insert(OnDiskState::new( - self.conf, - tenant_shard_id, - &timeline_id, - t.name, - t.metadata.clone(), - t.access_time, - local_path, - )); - } - } - } + ) + }, + ) + }); } result @@ -1135,6 +1233,7 @@ async fn init_timeline_state( conf: &'static PageServerConf, tenant_shard_id: &TenantShardId, heatmap: &HeatMapTimeline, + resident_metric: &UIntGauge, ) -> SecondaryDetailTimeline { let timeline_path = conf.timeline_path(tenant_shard_id, &heatmap.timeline_id); let mut detail = SecondaryDetailTimeline::default(); @@ -1210,17 +1309,13 @@ async fn init_timeline_state( } else { // We expect the access time to be initialized immediately afterwards, when // the latest heatmap is applied to the state. - detail.on_disk_layers.insert( - name.clone(), - OnDiskState::new( - conf, - tenant_shard_id, - &heatmap.timeline_id, - name, - remote_meta.metadata.clone(), - remote_meta.access_time, - file_path, - ), + detail.touch_layer( + conf, + tenant_shard_id, + &heatmap.timeline_id, + remote_meta, + resident_metric, + || file_path, ); } } From e823b9294714d0c5048942907c06b678c4a6c4a0 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Mon, 1 Jul 2024 13:11:55 +0100 Subject: [PATCH 005/194] CI(build-tools): Remove libpq from build image (#8206) ## Problem We use `build-tools` image as a base image to build other images, and it has a pretty old `libpq-dev` installed (v13; it wasn't that old until I removed system Postgres 14 from `build-tools` image in https://github.com/neondatabase/neon/pull/6540) ## Summary of changes - Remove `libpq-dev` from `build-tools` image - Set `LD_LIBRARY_PATH` for tests (for different Postgres binaries that we use, like psql and pgbench) - Set `PQ_LIB_DIR` to build Storage Controller - Set `LD_LIBRARY_PATH`/`DYLD_LIBRARY_PATH` in the Storage Controller where it calls Postgres binaries --- .../actions/run-python-test-set/action.yml | 1 + .github/workflows/benchmarking.yml | 4 +++ .github/workflows/build-build-tools-image.yml | 1 + .github/workflows/build_and_test.yml | 7 ++++ .github/workflows/neon_extra_builds.yml | 7 ++++ Dockerfile | 3 +- Dockerfile.build-tools | 1 - control_plane/src/local_env.rs | 9 +++-- control_plane/src/storage_controller.rs | 34 +++++++++++++++---- 9 files changed, 56 insertions(+), 11 deletions(-) diff --git a/.github/actions/run-python-test-set/action.yml b/.github/actions/run-python-test-set/action.yml index c6ea52ba8812..a2aae0772b15 100644 --- a/.github/actions/run-python-test-set/action.yml +++ b/.github/actions/run-python-test-set/action.yml @@ -114,6 +114,7 @@ runs: export PLATFORM=${PLATFORM:-github-actions-selfhosted} export POSTGRES_DISTRIB_DIR=${POSTGRES_DISTRIB_DIR:-/tmp/neon/pg_install} export DEFAULT_PG_VERSION=${PG_VERSION#v} + export LD_LIBRARY_PATH=${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/lib if [ "${BUILD_TYPE}" = "remote" ]; then export REMOTE_ENV=1 diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index db4209500ff5..0e748adeb69e 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -379,6 +379,10 @@ jobs: - name: Add Postgres binaries to PATH run: | + LD_LIBRARY_PATH="${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/lib" + export LD_LIBRARY_PATH + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV + ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/pgbench --version echo "${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin" >> $GITHUB_PATH diff --git a/.github/workflows/build-build-tools-image.yml b/.github/workflows/build-build-tools-image.yml index 5a94dd8e6f2d..f1c39e7e4f5b 100644 --- a/.github/workflows/build-build-tools-image.yml +++ b/.github/workflows/build-build-tools-image.yml @@ -82,6 +82,7 @@ jobs: tags: neondatabase/build-tools:${{ inputs.image-tag }}-${{ matrix.arch }} - name: Remove custom docker config directory + if: always() run: | rm -rf /tmp/.docker-custom diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 9cea9f41485d..24ad26205b60 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -335,6 +335,8 @@ jobs: - name: Run cargo build run: | + PQ_LIB_DIR=$(pwd)/pg_install/v16/lib + export PQ_LIB_DIR ${cov_prefix} mold -run cargo build $CARGO_FLAGS $CARGO_FEATURES --bins --tests # Do install *before* running rust tests because they might recompile the @@ -383,6 +385,11 @@ jobs: env: NEXTEST_RETRIES: 3 run: | + PQ_LIB_DIR=$(pwd)/pg_install/v16/lib + export PQ_LIB_DIR + LD_LIBRARY_PATH=$(pwd)/pg_install/v16/lib + export LD_LIBRARY_PATH + #nextest does not yet support running doctests cargo test --doc $CARGO_FLAGS $CARGO_FEATURES diff --git a/.github/workflows/neon_extra_builds.yml b/.github/workflows/neon_extra_builds.yml index 7d2187e59cd5..330d858c0eab 100644 --- a/.github/workflows/neon_extra_builds.yml +++ b/.github/workflows/neon_extra_builds.yml @@ -232,12 +232,19 @@ jobs: - name: Run cargo build run: | + PQ_LIB_DIR=$(pwd)/pg_install/v16/lib + export PQ_LIB_DIR mold -run cargo build --locked $CARGO_FLAGS $CARGO_FEATURES --bins --tests -j$(nproc) - name: Run cargo test env: NEXTEST_RETRIES: 3 run: | + PQ_LIB_DIR=$(pwd)/pg_install/v16/lib + export PQ_LIB_DIR + LD_LIBRARY_PATH=$(pwd)/pg_install/v16/lib + export LD_LIBRARY_PATH + cargo nextest run $CARGO_FEATURES -j$(nproc) # Run separate tests for real S3 diff --git a/Dockerfile b/Dockerfile index b4900d4a94a1..f0197758e48b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,12 +42,13 @@ ARG CACHEPOT_BUCKET=neon-github-dev COPY --from=pg-build /home/nonroot/pg_install/v14/include/postgresql/server pg_install/v14/include/postgresql/server COPY --from=pg-build /home/nonroot/pg_install/v15/include/postgresql/server pg_install/v15/include/postgresql/server COPY --from=pg-build /home/nonroot/pg_install/v16/include/postgresql/server pg_install/v16/include/postgresql/server +COPY --from=pg-build /home/nonroot/pg_install/v16/lib pg_install/v16/lib COPY --chown=nonroot . . # Show build caching stats to check if it was used in the end. # Has to be the part of the same RUN since cachepot daemon is killed in the end of this RUN, losing the compilation stats. RUN set -e \ - && RUSTFLAGS="-Clinker=clang -Clink-arg=-fuse-ld=mold -Clink-arg=-Wl,--no-rosegment" cargo build \ + && PQ_LIB_DIR=$(pwd)/pg_install/v16/lib RUSTFLAGS="-Clinker=clang -Clink-arg=-fuse-ld=mold -Clink-arg=-Wl,--no-rosegment" cargo build \ --bin pg_sni_router \ --bin pageserver \ --bin pagectl \ diff --git a/Dockerfile.build-tools b/Dockerfile.build-tools index f85706ef6a44..30314376efdb 100644 --- a/Dockerfile.build-tools +++ b/Dockerfile.build-tools @@ -26,7 +26,6 @@ RUN set -e \ liblzma-dev \ libncurses5-dev \ libncursesw5-dev \ - libpq-dev \ libreadline-dev \ libseccomp-dev \ libsqlite3-dev \ diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index 6634274d2a55..3ac3ce21df8f 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -325,11 +325,16 @@ impl LocalEnv { } } + pub fn pg_dir(&self, pg_version: u32, dir_name: &str) -> anyhow::Result { + Ok(self.pg_distrib_dir(pg_version)?.join(dir_name)) + } + pub fn pg_bin_dir(&self, pg_version: u32) -> anyhow::Result { - Ok(self.pg_distrib_dir(pg_version)?.join("bin")) + self.pg_dir(pg_version, "bin") } + pub fn pg_lib_dir(&self, pg_version: u32) -> anyhow::Result { - Ok(self.pg_distrib_dir(pg_version)?.join("lib")) + self.pg_dir(pg_version, "lib") } pub fn pageserver_bin(&self) -> PathBuf { diff --git a/control_plane/src/storage_controller.rs b/control_plane/src/storage_controller.rs index 5ca1b13b2a35..47103a2e0ac5 100644 --- a/control_plane/src/storage_controller.rs +++ b/control_plane/src/storage_controller.rs @@ -155,16 +155,16 @@ impl StorageController { .expect("non-Unicode path") } - /// Find the directory containing postgres binaries, such as `initdb` and `pg_ctl` + /// Find the directory containing postgres subdirectories, such `bin` and `lib` /// /// This usually uses STORAGE_CONTROLLER_POSTGRES_VERSION of postgres, but will fall back /// to other versions if that one isn't found. Some automated tests create circumstances /// where only one version is available in pg_distrib_dir, such as `test_remote_extensions`. - pub async fn get_pg_bin_dir(&self) -> anyhow::Result { + async fn get_pg_dir(&self, dir_name: &str) -> anyhow::Result { let prefer_versions = [STORAGE_CONTROLLER_POSTGRES_VERSION, 15, 14]; for v in prefer_versions { - let path = Utf8PathBuf::from_path_buf(self.env.pg_bin_dir(v)?).unwrap(); + let path = Utf8PathBuf::from_path_buf(self.env.pg_dir(v, dir_name)?).unwrap(); if tokio::fs::try_exists(&path).await? { return Ok(path); } @@ -172,11 +172,20 @@ impl StorageController { // Fall through anyhow::bail!( - "Postgres binaries not found in {}", - self.env.pg_distrib_dir.display() + "Postgres directory '{}' not found in {}", + dir_name, + self.env.pg_distrib_dir.display(), ); } + pub async fn get_pg_bin_dir(&self) -> anyhow::Result { + self.get_pg_dir("bin").await + } + + pub async fn get_pg_lib_dir(&self) -> anyhow::Result { + self.get_pg_dir("lib").await + } + /// Readiness check for our postgres process async fn pg_isready(&self, pg_bin_dir: &Utf8Path) -> anyhow::Result { let bin_path = pg_bin_dir.join("pg_isready"); @@ -229,12 +238,17 @@ impl StorageController { .unwrap() .join("storage_controller_db"); let pg_bin_dir = self.get_pg_bin_dir().await?; + let pg_lib_dir = self.get_pg_lib_dir().await?; let pg_log_path = pg_data_path.join("postgres.log"); if !tokio::fs::try_exists(&pg_data_path).await? { // Initialize empty database let initdb_path = pg_bin_dir.join("initdb"); let mut child = Command::new(&initdb_path) + .envs(vec![ + ("LD_LIBRARY_PATH".to_owned(), pg_lib_dir.to_string()), + ("DYLD_LIBRARY_PATH".to_owned(), pg_lib_dir.to_string()), + ]) .args(["-D", pg_data_path.as_ref()]) .spawn() .expect("Failed to spawn initdb"); @@ -269,7 +283,10 @@ impl StorageController { &self.env.base_data_dir, pg_bin_dir.join("pg_ctl").as_std_path(), db_start_args, - [], + vec![ + ("LD_LIBRARY_PATH".to_owned(), pg_lib_dir.to_string()), + ("DYLD_LIBRARY_PATH".to_owned(), pg_lib_dir.to_string()), + ], background_process::InitialPidFile::Create(self.postgres_pid_file()), retry_timeout, || self.pg_isready(&pg_bin_dir), @@ -324,7 +341,10 @@ impl StorageController { &self.env.base_data_dir, &self.env.storage_controller_bin(), args, - [], + vec![ + ("LD_LIBRARY_PATH".to_owned(), pg_lib_dir.to_string()), + ("DYLD_LIBRARY_PATH".to_owned(), pg_lib_dir.to_string()), + ], background_process::InitialPidFile::Create(self.pid_file()), retry_timeout, || async { From b02aafdfda4d410a33f11bd8d5f785c7cdccd740 Mon Sep 17 00:00:00 2001 From: Alex Chi Z Date: Mon, 1 Jul 2024 10:36:49 -0400 Subject: [PATCH 006/194] fix(pageserver): include aux file in basebackup only once (#8207) Extracted from https://github.com/neondatabase/neon/pull/6560, currently we include multiple copies of aux files in the basebackup. ## Summary of changes Fix the loop. Signed-off-by: Alex Chi Z Co-authored-by: Konstantin Knizhnik --- pageserver/src/basebackup.rs | 53 ++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/pageserver/src/basebackup.rs b/pageserver/src/basebackup.rs index 0f057a43683c..207f781e1b27 100644 --- a/pageserver/src/basebackup.rs +++ b/pageserver/src/basebackup.rs @@ -348,35 +348,36 @@ where self.add_rel(rel, rel).await?; } } + } - for (path, content) in self - .timeline - .list_aux_files(self.lsn, self.ctx) - .await - .map_err(|e| BasebackupError::Server(e.into()))? - { - if path.starts_with("pg_replslot") { - let offs = pg_constants::REPL_SLOT_ON_DISK_OFFSETOF_RESTART_LSN; - let restart_lsn = Lsn(u64::from_le_bytes( - content[offs..offs + 8].try_into().unwrap(), - )); - info!("Replication slot {} restart LSN={}", path, restart_lsn); - min_restart_lsn = Lsn::min(min_restart_lsn, restart_lsn); - } else if path == "pg_logical/replorigin_checkpoint" { - // replorigin_checkoint is written only on compute shutdown, so it contains - // deteriorated values. So we generate our own version of this file for the particular LSN - // based on information about replorigins extracted from transaction commit records. - // In future we will not generate AUX record for "pg_logical/replorigin_checkpoint" at all, - // but now we should handle (skip) it for backward compatibility. - continue; - } - let header = new_tar_header(&path, content.len() as u64)?; - self.ar - .append(&header, &*content) - .await - .context("could not add aux file to basebackup tarball")?; + for (path, content) in self + .timeline + .list_aux_files(self.lsn, self.ctx) + .await + .map_err(|e| BasebackupError::Server(e.into()))? + { + if path.starts_with("pg_replslot") { + let offs = pg_constants::REPL_SLOT_ON_DISK_OFFSETOF_RESTART_LSN; + let restart_lsn = Lsn(u64::from_le_bytes( + content[offs..offs + 8].try_into().unwrap(), + )); + info!("Replication slot {} restart LSN={}", path, restart_lsn); + min_restart_lsn = Lsn::min(min_restart_lsn, restart_lsn); + } else if path == "pg_logical/replorigin_checkpoint" { + // replorigin_checkoint is written only on compute shutdown, so it contains + // deteriorated values. So we generate our own version of this file for the particular LSN + // based on information about replorigins extracted from transaction commit records. + // In future we will not generate AUX record for "pg_logical/replorigin_checkpoint" at all, + // but now we should handle (skip) it for backward compatibility. + continue; } + let header = new_tar_header(&path, content.len() as u64)?; + self.ar + .append(&header, &*content) + .await + .context("could not add aux file to basebackup tarball")?; } + if min_restart_lsn != Lsn::MAX { info!( "Min restart LSN for logical replication is {}", From 9c32604aa98f86089b2f74863bebb7aad67424d9 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Mon, 1 Jul 2024 16:42:23 +0100 Subject: [PATCH 007/194] CI(gather-rust-build-stats): fix build with libpq (#8219) ## Problem I've missed setting `PQ_LIB_DIR` in https://github.com/neondatabase/neon/pull/8206 in `gather-rust-build-stats` job and it fails now: ``` = note: /usr/bin/ld: cannot find -lpq collect2: error: ld returned 1 exit status error: could not compile `storage_controller` (bin "storage_controller") due to 1 previous error ``` https://github.com/neondatabase/neon/actions/runs/9743960062/job/26888597735 ## Summary of changes - Set `PQ_LIB_DIR` for `gather-rust-build-stats` job --- .github/workflows/neon_extra_builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/neon_extra_builds.yml b/.github/workflows/neon_extra_builds.yml index 330d858c0eab..11ff634b6c65 100644 --- a/.github/workflows/neon_extra_builds.yml +++ b/.github/workflows/neon_extra_builds.yml @@ -385,7 +385,7 @@ jobs: run: make walproposer-lib -j$(nproc) - name: Produce the build stats - run: cargo build --all --release --timings -j$(nproc) + run: PQ_LIB_DIR=$(pwd)/pg_install/v16/lib cargo build --all --release --timings -j$(nproc) - name: Upload the build stats id: upload-stats From 0789160ffad0cd13b1e378fa5f19250fbd908afd Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 1 Jul 2024 18:55:18 +0300 Subject: [PATCH 008/194] tests: Make neon_xlogflush() flush all WAL, if you omit the LSN arg (#8215) This makes it much more convenient to use in the common case that you want to flush all the WAL. (Passing pg_current_wal_insert_lsn() as the argument doesn't work for the same reasons as explained in the comments: we need to be back off to the beginning of a page if the previous record ended at page boundary.) I plan to use this to fix the issue that Arseny Sher called out at https://github.com/neondatabase/neon/pull/7288#discussion_r1660063852 --- pgxn/neon_test_utils/neontest.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pgxn/neon_test_utils/neontest.c b/pgxn/neon_test_utils/neontest.c index 944936d39517..071dc122edbd 100644 --- a/pgxn/neon_test_utils/neontest.c +++ b/pgxn/neon_test_utils/neontest.c @@ -469,9 +469,9 @@ neon_xlogflush(PG_FUNCTION_ARGS) * The LSN returned by GetXLogInsertRecPtr() is the position where the * next inserted record would begin. If the last record ended just at * the page boundary, the next record will begin after the page header - * on the next page, and that's what GetXLogInsertRecPtr().returns, - * but the page header has not been written yet. If we tried to flush - * it, XLogFlush() would throw an error: + * on the next page, but the next page's page header has not been + * written yet. If we tried to flush it, XLogFlush() would throw an + * error: * * ERROR : xlog flush request %X/%X is not satisfied --- flushed only to %X/%X * From 9882ac8e0690c69df9091b48243cbde52153c492 Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Mon, 1 Jul 2024 18:44:28 +0100 Subject: [PATCH 009/194] docs: Graceful storage controller cluster restarts RFC (#7704) RFC for "Graceful Restarts of Storage Controller Managed Clusters". Related https://github.com/neondatabase/neon/issues/7387 --- .../033-storage-controller-drain-and-fill.md | 345 ++++++++++++++++++ 1 file changed, 345 insertions(+) create mode 100644 docs/rfcs/033-storage-controller-drain-and-fill.md diff --git a/docs/rfcs/033-storage-controller-drain-and-fill.md b/docs/rfcs/033-storage-controller-drain-and-fill.md new file mode 100644 index 000000000000..77c84cd2a525 --- /dev/null +++ b/docs/rfcs/033-storage-controller-drain-and-fill.md @@ -0,0 +1,345 @@ +# Graceful Restarts of Storage Controller Managed Clusters + +## Summary +This RFC describes new storage controller APIs for draining and filling tenant shards from/on pageserver nodes. +It also covers how these new APIs should be used by an orchestrator (e.g. Ansible) in order to implement +graceful cluster restarts. + +## Motivation + +Pageserver restarts cause read availablity downtime for tenants. + +For example pageserver-3 @ us-east-1 was unavailable for a randomly +picked tenant (which requested on-demand activation) for around 30 seconds +during the restart at 2024-04-03 16:37 UTC. + +Note that lots of shutdowns on loaded pageservers do not finish within the +[10 second systemd enforced timeout](https://github.com/neondatabase/aws/blob/0a5280b383e43c063d43cbf87fa026543f6d6ad4/.github/ansible/systemd/pageserver.service#L16). This means we are shutting down without flushing ephemeral layers +and have to reingest data in order to serve requests after restarting, potentially making first request latencies worse. + +This problem is not yet very acutely felt in storage controller managed pageservers since +tenant density is much lower there. However, we are planning on eventually migrating all +pageservers to storage controller management, so it makes sense to solve the issue proactively. + +## Requirements + +- Pageserver re-deployments cause minimal downtime for tenants +- The storage controller exposes HTTP API hooks for draining and filling tenant shards +from a given pageserver. Said hooks can be used by an orchestrator proces or a human operator. +- The storage controller exposes some HTTP API to cancel draining and filling background operations. +- Failures to drain or fill the node should not be fatal. In such cases, cluster restarts should proceed +as usual (with downtime). +- Progress of draining/filling is visible through metrics + +## Non Goals + +- Integration with the control plane +- Graceful restarts for large non-HA tenants. + +## Impacted Components + +- storage controller +- deployment orchestrator (i.e. Ansible) +- pageserver (indirectly) + +## Terminology + +** Draining ** is the process through which all tenant shards that can be migrated from a given pageserver +are distributed across the rest of the cluster. + +** Filling ** is the symmetric opposite of draining. In this process tenant shards are migrated onto a given +pageserver until the cluster reaches a resonable, quiescent distribution of tenant shards across pageservers. + +** Node scheduling policies ** act as constraints to the scheduler. For instance, when a +node is set in the `Paused` policy, no further shards will be scheduled on it. + +** Node ** is a pageserver. Term is used interchangeably in this RFC. + +** Deployment orchestrator ** is a generic term for whatever drives our deployments. +Currently, it's an Ansible playbook. + +## Background + +### Storage Controller Basics (skip if already familiar) + +Fundamentally, the storage controller is a reconciler which aims to move from the observed mapping between pageservers and tenant shards to an intended mapping. Pageserver nodes and tenant shards metadata is durably persisted in a database, but note that the mapping between the two entities is not durably persisted. Instead, this mapping (*observed state*) is constructed at startup by sending `GET location_config` requests to registered pageservers. + +An internal scheduler maps tenant shards to pageservers while respecting certain constraints. The result of scheduling is the *intent state*. When the intent state changes, a *reconciliation* will inform pageservers about the new assigment via `PUT location_config` requests and will notify the compute via the configured hook. + +### Background Optimizations + +The storage controller performs scheduling optimizations in the background. It will +migrate attachments to warm secondaries and replace secondaries in order to balance +the cluster out. + +### Reconciliations Concurrency Limiting + +There's a hard limit on the number of reconciles that the storage controller +can have in flight at any given time. To get an idea of scales, the limit is +128 at the time of writing. + +## Implementation + +Note: this section focuses on the core functionality of the graceful restart process. +It doesn't neccesarily describe the most efficient approach. Optimizations are described +separately in a later section. + +### Overall Flow + +This section describes how to implement graceful restarts from the perspective +of Ansible, the deployment orchestrator. Pageservers are already restarted sequentially. +The orchestrator shall implement the following epilogue and prologue steps for each +pageserver restart: + +#### Prologue + +The orchestrator shall first fetch the pageserver node id from the control plane or +the pageserver it aims to restart directly. Next, it issues an HTTP request +to the storage controller in order to start the drain of said pageserver node. +All error responses are retried with a short back-off. When a 202 (Accepted) +HTTP code is returned, the drain has started. Now the orchestrator polls the +node status endpoint exposed by the storage controller in order to await the +end of the drain process. When the `policy` field of the node status response +becomes `PauseForRestart`, the drain has completed and the orchestrator can +proceed with restarting the pageserver. + +The prologue is subject to an overall timeout. It will have a value in the ballpark +of minutes. As storage controller managed pageservers become more loaded this timeout +will likely have to increase. + +#### Epilogue + +After restarting the pageserver, the orchestrator issues an HTTP request +to the storage controller to kick off the filling process. This API call +may be retried for all error codes with a short backoff. This also serves +as a synchronization primitive as the fill will be refused if the pageserver +has not yet re-attached to the storage controller. When a 202(Accepted) HTTP +code is returned, the fill has started. Now the orchestrator polls the node +status endpoint exposed by the storage controller in order to await the end of +the filling process. When the `policy` field of the node status response becomes +`Active`, the fill has completed and the orchestrator may proceed to the next pageserver. + +Again, the epilogue is subject to an overall timeout. We can start off with +using the same timeout as for the prologue, but can also consider relying on +the storage controller's background optimizations with a shorter timeout. + +In the case that the deployment orchestrator times out, it attempts to cancel +the fill. This operation shall be retried with a short back-off. If it ultimately +fails it will require manual intervention to set the nodes scheduling policy to +`NodeSchedulingPolicy::Active`. Not doing that is not immediately problematic, +but it constrains the scheduler as mentioned previously. + +### Node Scheduling Policy State Machine + +The state machine below encodes the behaviours discussed above and +the various failover situations described in a later section. + +Assuming no failures and/or timeouts the flow should be: +`Active -> Draining -> PauseForRestart -> Active -> Filling -> Active` + +``` + Operator requested drain + +-----------------------------------------+ + | | + +-------+-------+ +-------v-------+ + | | | | + | Pause | +-----------> Draining +----------+ + | | | | | | + +---------------+ | +-------+-------+ | + | | | + | | | + Drain requested| | | + | |Drain complete | Drain failed + | | | Cancelled/PS reattach/Storcon restart + | | | + +-------+-------+ | | + | | | | + +-------------+ Active <-----------+------------------+ + | | | | +Fill requested | +---^---^-------+ | + | | | | + | | | | + | | | | + | Fill completed| | | + | | |PS reattach | + | | |after restart | + +-------v-------+ | | +-------v-------+ + | | | | | | + | Filling +---------+ +-----------+PauseForRestart| + | | | | + +---------------+ +---------------+ +``` + +### Draining/Filling APIs + +The storage controller API to trigger the draining of a given node is: +`PUT /v1/control/node/:node_id/{drain,fill}`. + +The following HTTP non-success return codes are used. +All of them are safely retriable from the perspective of the storage controller. +- 404: Requested node was not found +- 503: Requested node is known to the storage controller, but unavailable +- 412: Drain precondition failed: there is no other node to drain to or the node's schedulling policy forbids draining +- 409: A {drain, fill} is already in progress. Only one such background operation +is allowed per node. + +When the drain is accepted and commenced a 202 HTTP code is returned. + +Drains and fills shall be cancellable by the deployment orchestrator or a +human operator via: `DELETE /v1/control/node/:node_id/{drain,fill}`. A 200 +response is returned when the cancelation is successful. Errors are retriable. + +### Drain Process + +Before accpeting a drain request the following validations is applied: +* Ensure that the node is known the storage controller +* Ensure that the schedulling policy is `NodeSchedulingPolicy::Active` or `NodeSchedulingPolicy::Pause` +* Ensure that another drain or fill is not already running on the node +* Ensure that a drain is possible (i.e. check that there is at least one +schedulable node to drain to) + +After accepting the drain, the scheduling policy of the node is set to +`NodeSchedulingPolicy::Draining` and persisted in both memory and the database. +This disallows the optimizer from adding or removing shards from the node which +is desirable to avoid them racing. + +Next, a separate Tokio task is spawned to manage the draining. For each tenant +shard attached to the node being drained, demote the node to a secondary and +attempt to schedule the node away. Scheduling might fail due to unsatisfiable +constraints, but that is fine. Draining is a best effort process since it might +not always be possible to cut over all shards. + +Importantly, this task manages the concurrency of issued reconciles in order to +avoid drowning out the target pageservers and to allow other important reconciles +to proceed. + +Once the triggered reconciles have finished or timed out, set the node's scheduling +policy to `NodeSchedulingPolicy::PauseForRestart` to signal the end of the drain. + +A note on non HA tenants: These tenants do not have secondaries, so by the description +above, they would not be migrated. It makes sense to skip them (especially the large ones) +since, depending on tenant size, this might be more disruptive than the restart since the +pageserver we've moved to do will need to on-demand download the entire working set for the tenant. +We can consider expanding to small non-HA tenants in the future. + +### Fill Process + +Before accpeting a fill request the following validations is applied: +* Ensure that the node is known the storage controller +* Ensure that the schedulling policy is `NodeSchedulingPolicy::Active`. +This is the only acceptable policy for the fill starting state. When a node re-attaches, +it set the scheduling policy to `NodeSchedulingPolicy::Active` if it was equal to +`NodeSchedulingPolicy::PauseForRestart` or `NodeSchedulingPolicy::Draining` (possible end states for a node drain). +* Ensure that another drain or fill is not already running on the node + +After accepting the drain, the scheduling policy of the node is set to +`NodeSchedulingPolicy::Filling` and persisted in both memory and the database. +This disallows the optimizer from adding or removing shards from the node which +is desirable to avoid them racing. + +Next, a separate Tokio task is spawned to manage the draining. For each tenant +shard where the filled node is a secondary, promote the secondary. This is done +until we run out of shards or the counts of attached shards become balanced across +the cluster. + +Like for draining, the concurrency of spawned reconciles is limited. + +### Failure Modes & Handling + +Failures are generally handled by transition back into the `Active` +(neutral) state. This simplifies the implementation greatly at the +cost of adding transitions to the state machine. For example, we +could detect the `Draining` state upon restart and proceed with a drain, +but how should the storage controller know that's what the orchestrator +needs still? + +#### Storage Controller Crash + +When the storage controller starts up reset the node scheduling policy +of all nodes in states `Draining`, `Filling` or `PauseForRestart` to +`Active`. The rationale is that when the storage controller restarts, +we have lost context of what the deployment orchestrator wants. It also +has the benefit of making things easier to reason about. + +#### Pageserver Crash During Drain + +The pageserver will attempt to re-attach during restart at which +point the node scheduling policy will be set back to `Active`, thus +reenabling the scheduler to use the node. + +#### Non-drained Pageserver Crash During Drain + +What should happen when a pageserver we are draining to crashes during the +process. Two reasonable options are: cancel the drain and focus on the failover +*or* do both, but prioritise failover. Since the number of concurrent reconciles +produced by drains/fills are limited, we get the later behaviour for free. +My suggestion is we take this approach, but the cancellation option is trivial +to implement as well. + +#### Pageserver Crash During Fill + +The pageserver will attempt to re-attach during restart at which +point the node scheduling policy will be set back to `Active`, thus +reenabling the scheduler to use the node. + +#### Pageserver Goes unavailable During Drain/Fill + +The drain and fill jobs handle this by stopping early. When the pageserver +is detected as online by storage controller heartbeats, reset its scheduling +policy to `Active`. If a restart happens instead, see the pageserver crash +failure mode. + +#### Orchestrator Drain Times Out + +Orchestrator will still proceed with the restart. +When the pageserver re-attaches, the scheduling policy is set back to +`Active`. + +#### Orchestrator Fill Times Out + +Orchestrator will attempt to cancel the fill operation. If that fails, +the fill will continue until it quiesces and the node will be left +in the `Filling` scheduling policy. This hinders the scheduler, but is +otherwise harmless. A human operator can handle this by setting the scheduling +policy to `Active`, or we can bake in a fill timeout into the storage controller. + +## Optimizations + +### Location Warmth + +When cutting over to a secondary, the storage controller will wait for it to +become "warm" (i.e. download enough of the tenants data). This means that some +reconciliations can take significantly longer than others and hold up precious +reconciliations units. As an optimization, the drain stage can only cut over +tenants that are already "warm". Similarly, the fill stage can prioritise the +"warmest" tenants in the fill. + +Given that the number of tenants by the storage controller will be fairly low +for the foreseable future, the first implementation could simply query the tenants +for secondary status. This doesn't scale well with increasing tenant counts, so +eventually we will need new pageserver API endpoints to report the sets of +"warm" and "cold" nodes. + +## Alternatives Considered + +### Draining and Filling Purely as Scheduling Constraints + +At its core, the storage controller is a big background loop that detects changes +in the environment and reacts on them. One could express draining and filling +of nodes purely in terms of constraining the scheduler (as opposed to having +such background tasks). + +While theoretically nice, I think that's harder to implement and more importantly operate and reason about. +Consider cancellation of a drain/fill operation. We would have to update the scheduler state, create +an entirely new schedule (intent state) and start work on applying that. It gets trickier if we wish +to cancel the reconciliation tasks spawned by drain/fill nodes. How would we know which ones belong +to the conceptual drain/fill? One could add labels to reconciliations, but it gets messy in my opinion. + +It would also mean that reconciliations themselves have side effects that persist in the database +(persist something to the databse when the drain is done), which I'm not conceptually fond of. + +## Proof of Concept + +This RFC is accompanied by a POC which implements nearly everything mentioned here +apart from the optimizations and some of the failure handling: +https://github.com/neondatabase/neon/pull/7682 From 0497b99f3abbb95d07fd80727da5c565afd72e0a Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 2 Jul 2024 06:56:10 +0300 Subject: [PATCH 010/194] Check status of connection after PQconnectStartParams (#8210) ## Problem See https://github.com/neondatabase/cloud/issues/14289 ## Summary of changes Check connection status after calling PQconnectStartParams ## Checklist before requesting a review - [ ] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. ## Checklist before merging - [ ] Do not forget to reformat commit message to not include the above checklist --------- Co-authored-by: Konstantin Knizhnik --- pgxn/neon/libpagestore.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pgxn/neon/libpagestore.c b/pgxn/neon/libpagestore.c index a665cafafe71..a3fdcc537ead 100644 --- a/pgxn/neon/libpagestore.c +++ b/pgxn/neon/libpagestore.c @@ -432,7 +432,17 @@ pageserver_connect(shardno_t shard_no, int elevel) neon_shard_log(shard_no, elevel, "Failed to connect to pageserver: out of memory"); return false; } - + if (PQstatus(shard->conn) == CONNECTION_BAD) + { + char *msg = pchomp(PQerrorMessage(shard->conn)); + CLEANUP_AND_DISCONNECT(shard); + ereport(elevel, + (errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION), + errmsg(NEON_TAG "[shard %d] could not establish connection to pageserver", shard_no), + errdetail_internal("%s", msg))); + pfree(msg); + return false; + } shard->state = PS_Connecting_Startup; /* fallthrough */ } From 7dcdbaa25e00233f79199a30748e08f8b5d72c33 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 2 Jul 2024 12:53:08 +0200 Subject: [PATCH 011/194] remote_storage config: move handling of empty inline table `{}` to callers (#8193) Before this PR, `RemoteStorageConfig::from_toml` would support deserializing an empty `{}` TOML inline table to a `None`, otherwise try `Some()`. We can instead let * in proxy: let clap derive handle the Option * in PS & SK: assume that if the field is specified, it must be a valid RemtoeStorageConfig (This PR started with a much simpler goal of factoring out the `deserialize_item` function because I need that in another PR). --- Cargo.lock | 1 + libs/remote_storage/src/config.rs | 25 ++++++------------------- libs/utils/Cargo.toml | 1 + libs/utils/src/lib.rs | 2 ++ libs/utils/src/toml_edit_ext.rs | 22 ++++++++++++++++++++++ pageserver/ctl/src/main.rs | 2 +- pageserver/src/config.rs | 19 ++++++++++++++++--- proxy/src/bin/proxy.rs | 9 ++++----- proxy/src/config.rs | 8 ++------ proxy/src/context/parquet.rs | 15 ++++++--------- safekeeper/src/bin/safekeeper.rs | 13 ++----------- test_runner/fixtures/neon_fixtures.py | 4 +++- 12 files changed, 66 insertions(+), 55 deletions(-) create mode 100644 libs/utils/src/toml_edit_ext.rs diff --git a/Cargo.lock b/Cargo.lock index 5393538c5902..6dae8e340348 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6811,6 +6811,7 @@ dependencies = [ "tokio-stream", "tokio-tar", "tokio-util", + "toml_edit 0.19.10", "tracing", "tracing-error", "tracing-subscriber", diff --git a/libs/remote_storage/src/config.rs b/libs/remote_storage/src/config.rs index 8a8f6212e99b..fa3f2cba58d7 100644 --- a/libs/remote_storage/src/config.rs +++ b/libs/remote_storage/src/config.rs @@ -1,6 +1,5 @@ use std::{fmt::Debug, num::NonZeroUsize, str::FromStr, time::Duration}; -use anyhow::bail; use aws_sdk_s3::types::StorageClass; use camino::Utf8PathBuf; @@ -176,20 +175,8 @@ fn serialize_storage_class( impl RemoteStorageConfig { pub const DEFAULT_TIMEOUT: Duration = std::time::Duration::from_secs(120); - pub fn from_toml(toml: &toml_edit::Item) -> anyhow::Result> { - let document: toml_edit::Document = match toml { - toml_edit::Item::Table(toml) => toml.clone().into(), - toml_edit::Item::Value(toml_edit::Value::InlineTable(toml)) => { - toml.clone().into_table().into() - } - _ => bail!("toml not a table or inline table"), - }; - - if document.is_empty() { - return Ok(None); - } - - Ok(Some(toml_edit::de::from_document(document)?)) + pub fn from_toml(toml: &toml_edit::Item) -> anyhow::Result { + Ok(utils::toml_edit_ext::deserialize_item(toml)?) } } @@ -197,7 +184,7 @@ impl RemoteStorageConfig { mod tests { use super::*; - fn parse(input: &str) -> anyhow::Result> { + fn parse(input: &str) -> anyhow::Result { let toml = input.parse::().unwrap(); RemoteStorageConfig::from_toml(toml.as_item()) } @@ -207,7 +194,7 @@ mod tests { let input = "local_path = '.' timeout = '5s'"; - let config = parse(input).unwrap().expect("it exists"); + let config = parse(input).unwrap(); assert_eq!( config, @@ -229,7 +216,7 @@ timeout = '5s'"; timeout = '7s' "; - let config = parse(toml).unwrap().expect("it exists"); + let config = parse(toml).unwrap(); assert_eq!( config, @@ -257,7 +244,7 @@ timeout = '5s'"; timeout = '7s' "; - let config = parse(toml).unwrap().expect("it exists"); + let config = parse(toml).unwrap(); assert_eq!( config, diff --git a/libs/utils/Cargo.toml b/libs/utils/Cargo.toml index a6a081c5c144..261ca2cc1ac0 100644 --- a/libs/utils/Cargo.toml +++ b/libs/utils/Cargo.toml @@ -40,6 +40,7 @@ thiserror.workspace = true tokio.workspace = true tokio-tar.workspace = true tokio-util.workspace = true +toml_edit.workspace = true tracing.workspace = true tracing-error.workspace = true tracing-subscriber = { workspace = true, features = ["json", "registry"] } diff --git a/libs/utils/src/lib.rs b/libs/utils/src/lib.rs index 2953f0aad4fd..2a397d97d2b9 100644 --- a/libs/utils/src/lib.rs +++ b/libs/utils/src/lib.rs @@ -94,6 +94,8 @@ pub mod env; pub mod poison; +pub mod toml_edit_ext; + /// This is a shortcut to embed git sha into binaries and avoid copying the same build script to all packages /// /// we have several cases: diff --git a/libs/utils/src/toml_edit_ext.rs b/libs/utils/src/toml_edit_ext.rs new file mode 100644 index 000000000000..ab5f7bdd95ab --- /dev/null +++ b/libs/utils/src/toml_edit_ext.rs @@ -0,0 +1,22 @@ +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("item is not a document")] + ItemIsNotADocument, + #[error(transparent)] + Serde(toml_edit::de::Error), +} + +pub fn deserialize_item(item: &toml_edit::Item) -> Result +where + T: serde::de::DeserializeOwned, +{ + let document: toml_edit::Document = match item { + toml_edit::Item::Table(toml) => toml.clone().into(), + toml_edit::Item::Value(toml_edit::Value::InlineTable(toml)) => { + toml.clone().into_table().into() + } + _ => return Err(Error::ItemIsNotADocument), + }; + + toml_edit::de::from_document(document).map_err(Error::Serde) +} diff --git a/pageserver/ctl/src/main.rs b/pageserver/ctl/src/main.rs index 50c3ac4c6143..ea09a011e5cf 100644 --- a/pageserver/ctl/src/main.rs +++ b/pageserver/ctl/src/main.rs @@ -178,7 +178,7 @@ async fn main() -> anyhow::Result<()> { let toml_item = toml_document .get("remote_storage") .expect("need remote_storage"); - let config = RemoteStorageConfig::from_toml(toml_item)?.expect("incomplete config"); + let config = RemoteStorageConfig::from_toml(toml_item)?; let storage = remote_storage::GenericRemoteStorage::from_config(&config); let cancel = CancellationToken::new(); storage diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index f36e63f035c7..2b698b75dcb1 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -159,7 +159,7 @@ pub mod defaults { #ephemeral_bytes_per_memory_kb = {DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB} -[remote_storage] +#[remote_storage] "# ); @@ -918,7 +918,7 @@ impl PageServerConf { "http_auth_type" => builder.http_auth_type(parse_toml_from_str(key, item)?), "pg_auth_type" => builder.pg_auth_type(parse_toml_from_str(key, item)?), "remote_storage" => { - builder.remote_storage_config(RemoteStorageConfig::from_toml(item)?) + builder.remote_storage_config(Some(RemoteStorageConfig::from_toml(item).context("remote_storage")?)) } "tenant_config" => { t_conf = TenantConfOpt::try_from(item.to_owned()).context(format!("failed to parse: '{key}'"))?; @@ -946,7 +946,7 @@ impl PageServerConf { builder.metric_collection_endpoint(Some(endpoint)); }, "metric_collection_bucket" => { - builder.metric_collection_bucket(RemoteStorageConfig::from_toml(item)?) + builder.metric_collection_bucket(Some(RemoteStorageConfig::from_toml(item)?)) } "synthetic_size_calculation_interval" => builder.synthetic_size_calculation_interval(parse_toml_duration(key, item)?), @@ -1681,6 +1681,19 @@ threshold = "20m" } } + #[test] + fn empty_remote_storage_is_error() { + let tempdir = tempdir().unwrap(); + let (workdir, _) = prepare_fs(&tempdir).unwrap(); + let input = r#" +remote_storage = {} + "#; + let doc = toml_edit::Document::from_str(input).unwrap(); + let err = PageServerConf::parse_and_validate(&doc, &workdir) + .expect_err("empty remote_storage field should fail, don't specify it if you want no remote_storage"); + assert!(format!("{err}").contains("remote_storage"), "{err}"); + } + fn prepare_fs(tempdir: &Utf8TempDir) -> anyhow::Result<(Utf8PathBuf, Utf8PathBuf)> { let tempdir_path = tempdir.path(); diff --git a/proxy/src/bin/proxy.rs b/proxy/src/bin/proxy.rs index dffebf55800c..7f4cb2c0100c 100644 --- a/proxy/src/bin/proxy.rs +++ b/proxy/src/bin/proxy.rs @@ -35,6 +35,7 @@ use proxy::usage_metrics; use anyhow::bail; use proxy::config::{self, ProxyConfig}; use proxy::serverless; +use remote_storage::RemoteStorageConfig; use std::net::SocketAddr; use std::pin::pin; use std::sync::Arc; @@ -205,8 +206,8 @@ struct ProxyCliArgs { /// remote storage configuration for backup metric collection /// Encoded as toml (same format as pageservers), eg /// `{bucket_name='the-bucket',bucket_region='us-east-1',prefix_in_bucket='proxy',endpoint='http://minio:9000'}` - #[clap(long, default_value = "{}")] - metric_backup_collection_remote_storage: String, + #[clap(long, value_parser = remote_storage_from_toml)] + metric_backup_collection_remote_storage: Option, /// chunk size for backup metric collection /// Size of each event is no more than 400 bytes, so 2**22 is about 200MB before the compression. #[clap(long, default_value = "4194304")] @@ -511,9 +512,7 @@ fn build_config(args: &ProxyCliArgs) -> anyhow::Result<&'static ProxyConfig> { } let backup_metric_collection_config = config::MetricBackupCollectionConfig { interval: args.metric_backup_collection_interval, - remote_storage_config: remote_storage_from_toml( - &args.metric_backup_collection_remote_storage, - )?, + remote_storage_config: args.metric_backup_collection_remote_storage.clone(), chunk_size: args.metric_backup_collection_chunk_size, }; diff --git a/proxy/src/config.rs b/proxy/src/config.rs index f4707a33aa79..af5511d7ec24 100644 --- a/proxy/src/config.rs +++ b/proxy/src/config.rs @@ -399,15 +399,11 @@ impl FromStr for EndpointCacheConfig { #[derive(Debug)] pub struct MetricBackupCollectionConfig { pub interval: Duration, - pub remote_storage_config: OptRemoteStorageConfig, + pub remote_storage_config: Option, pub chunk_size: usize, } -/// Hack to avoid clap being smarter. If you don't use this type alias, clap assumes more about the optional state and you get -/// runtime type errors from the value parser we use. -pub type OptRemoteStorageConfig = Option; - -pub fn remote_storage_from_toml(s: &str) -> anyhow::Result { +pub fn remote_storage_from_toml(s: &str) -> anyhow::Result { RemoteStorageConfig::from_toml(&s.parse()?) } diff --git a/proxy/src/context/parquet.rs b/proxy/src/context/parquet.rs index e72bf199e362..cfc1f8e89e3f 100644 --- a/proxy/src/context/parquet.rs +++ b/proxy/src/context/parquet.rs @@ -14,17 +14,14 @@ use parquet::{ record::RecordWriter, }; use pq_proto::StartupMessageParams; -use remote_storage::{GenericRemoteStorage, RemotePath, TimeoutOrCancel}; +use remote_storage::{GenericRemoteStorage, RemotePath, RemoteStorageConfig, TimeoutOrCancel}; use serde::ser::SerializeMap; use tokio::{sync::mpsc, time}; use tokio_util::sync::CancellationToken; use tracing::{debug, info, Span}; use utils::backoff; -use crate::{ - config::{remote_storage_from_toml, OptRemoteStorageConfig}, - context::LOG_CHAN_DISCONNECT, -}; +use crate::{config::remote_storage_from_toml, context::LOG_CHAN_DISCONNECT}; use super::{RequestMonitoring, LOG_CHAN}; @@ -33,11 +30,11 @@ pub struct ParquetUploadArgs { /// Storage location to upload the parquet files to. /// Encoded as toml (same format as pageservers), eg /// `{bucket_name='the-bucket',bucket_region='us-east-1',prefix_in_bucket='proxy',endpoint='http://minio:9000'}` - #[clap(long, default_value = "{}", value_parser = remote_storage_from_toml)] - parquet_upload_remote_storage: OptRemoteStorageConfig, + #[clap(long, value_parser = remote_storage_from_toml)] + parquet_upload_remote_storage: Option, - #[clap(long, default_value = "{}", value_parser = remote_storage_from_toml)] - parquet_upload_disconnect_events_remote_storage: OptRemoteStorageConfig, + #[clap(long, value_parser = remote_storage_from_toml)] + parquet_upload_disconnect_events_remote_storage: Option, /// How many rows to include in a row group #[clap(long, default_value_t = 8192)] diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index c81373c77c7d..d25b8722ac23 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -12,7 +12,6 @@ use sd_notify::NotifyState; use tokio::runtime::Handle; use tokio::signal::unix::{signal, SignalKind}; use tokio::task::JoinError; -use toml_edit::Document; use utils::logging::SecretString; use std::env::{var, VarError}; @@ -126,7 +125,7 @@ struct Args { peer_recovery: bool, /// Remote storage configuration for WAL backup (offloading to s3) as TOML /// inline table, e.g. - /// {"max_concurrent_syncs" = 17, "max_sync_errors": 13, "bucket_name": "", "bucket_region":"", "concurrency_limit": 119} + /// {max_concurrent_syncs = 17, max_sync_errors = 13, bucket_name = "", bucket_region = "", concurrency_limit = 119} /// Safekeeper offloads WAL to /// [prefix_in_bucket/]//, mirroring /// structure on the file system. @@ -553,16 +552,8 @@ fn set_id(workdir: &Utf8Path, given_id: Option) -> Result { Ok(my_id) } -// Parse RemoteStorage from TOML table. fn parse_remote_storage(storage_conf: &str) -> anyhow::Result { - // funny toml doesn't consider plain inline table as valid document, so wrap in a key to parse - let storage_conf_toml = format!("remote_storage = {storage_conf}"); - let parsed_toml = storage_conf_toml.parse::()?; // parse - let (_, storage_conf_parsed_toml) = parsed_toml.iter().next().unwrap(); // and strip key off again - RemoteStorageConfig::from_toml(storage_conf_parsed_toml).and_then(|parsed_config| { - // XXX: Don't print the original toml here, there might be some sensitive data - parsed_config.context("Incorrectly parsed remote storage toml as no remote storage config") - }) + RemoteStorageConfig::from_toml(&storage_conf.parse()?) } #[test] diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index e1c851435142..565aaba6e0dc 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1167,7 +1167,9 @@ def __init__(self, config: NeonEnvBuilder): if config.auth_enabled: sk_cfg["auth_enabled"] = True if self.safekeepers_remote_storage is not None: - sk_cfg["remote_storage"] = self.safekeepers_remote_storage.to_toml_inline_table() + sk_cfg[ + "remote_storage" + ] = self.safekeepers_remote_storage.to_toml_inline_table().strip() self.safekeepers.append(Safekeeper(env=self, id=id, port=port)) cfg["safekeepers"].append(sk_cfg) From 1a0f545c16de5e105a3b22990ce0953e078ac1dc Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 2 Jul 2024 13:45:04 +0100 Subject: [PATCH 012/194] pageserver: simpler, stricter config error handling (#8177) ## Problem Tenant attachment has error paths for failures to write local configuration, but these types of local storage I/O errors should be considered fatal for the process. Related thread on an earlier PR that touched this code: https://github.com/neondatabase/neon/pull/7947#discussion_r1655134114 ## Summary of changes - Make errors writing tenant config fatal (abort process) - When reading tenant config, make all I/O errors except ENOENT fatal - Replace use of bare anyhow errors with `LoadConfigError` --- pageserver/src/http/routes.rs | 4 +- pageserver/src/tenant.rs | 78 +++++------ pageserver/src/tenant/mgr.rs | 193 +++++++++++++--------------- test_runner/regress/test_tenants.py | 25 +++- 4 files changed, 155 insertions(+), 145 deletions(-) diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 1fda2eaa854e..f726ba115d83 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -227,7 +227,7 @@ impl From for ApiError { BadRequest(e) => ApiError::BadRequest(e), Unavailable(_) => ApiError::ShuttingDown, e @ InProgress => ApiError::Conflict(format!("{e}")), - Flush(e) | Other(e) => ApiError::InternalServerError(e), + Flush(e) | InternalError(e) => ApiError::InternalServerError(e), } } } @@ -1296,7 +1296,7 @@ async fn update_tenant_config_handler( crate::tenant::Tenant::persist_tenant_config(state.conf, &tenant_shard_id, &location_conf) .await - .map_err(ApiError::InternalServerError)?; + .map_err(|e| ApiError::InternalServerError(anyhow::anyhow!(e)))?; tenant.set_new_tenant_config(new_tenant_conf); json_response(StatusCode::OK, ()) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 3ffbaf98c69f..116481a1ebbb 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -529,6 +529,15 @@ impl From for GcError { } } +#[derive(thiserror::Error, Debug)] +pub(crate) enum LoadConfigError { + #[error("TOML deserialization error: '{0}'")] + DeserializeToml(#[from] toml_edit::de::Error), + + #[error("Config not found at {0}")] + NotFound(Utf8PathBuf), +} + impl Tenant { /// Yet another helper for timeline initialization. /// @@ -2563,36 +2572,35 @@ impl Tenant { pub(super) fn load_tenant_config( conf: &'static PageServerConf, tenant_shard_id: &TenantShardId, - ) -> anyhow::Result { + ) -> Result { let config_path = conf.tenant_location_config_path(tenant_shard_id); - if config_path.exists() { - // New-style config takes precedence - let deserialized = Self::read_config(&config_path)?; - Ok(toml_edit::de::from_document::(deserialized)?) - } else { - // The config should almost always exist for a tenant directory: - // - When attaching a tenant, the config is the first thing we write - // - When detaching a tenant, we atomically move the directory to a tmp location - // before deleting contents. - // - // The very rare edge case that can result in a missing config is if we crash during attach - // between creating directory and writing config. Callers should handle that as if the - // directory didn't exist. - anyhow::bail!("tenant config not found in {}", config_path); - } - } - - fn read_config(path: &Utf8Path) -> anyhow::Result { - info!("loading tenant configuration from {path}"); + info!("loading tenant configuration from {config_path}"); // load and parse file - let config = fs::read_to_string(path) - .with_context(|| format!("Failed to load config from path '{path}'"))?; + let config = fs::read_to_string(&config_path).map_err(|e| { + match e.kind() { + std::io::ErrorKind::NotFound => { + // The config should almost always exist for a tenant directory: + // - When attaching a tenant, the config is the first thing we write + // - When detaching a tenant, we atomically move the directory to a tmp location + // before deleting contents. + // + // The very rare edge case that can result in a missing config is if we crash during attach + // between creating directory and writing config. Callers should handle that as if the + // directory didn't exist. + + LoadConfigError::NotFound(config_path) + } + _ => { + // No IO errors except NotFound are acceptable here: other kinds of error indicate local storage or permissions issues + // that we cannot cleanly recover + crate::virtual_file::on_fatal_io_error(&e, "Reading tenant config file") + } + } + })?; - config - .parse::() - .with_context(|| format!("Failed to parse config from file '{path}' as toml file")) + Ok(toml_edit::de::from_str::(&config)?) } #[tracing::instrument(skip_all, fields(tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug()))] @@ -2600,7 +2608,7 @@ impl Tenant { conf: &'static PageServerConf, tenant_shard_id: &TenantShardId, location_conf: &LocationConf, - ) -> anyhow::Result<()> { + ) -> std::io::Result<()> { let config_path = conf.tenant_location_config_path(tenant_shard_id); Self::persist_tenant_config_at(tenant_shard_id, &config_path, location_conf).await @@ -2611,7 +2619,7 @@ impl Tenant { tenant_shard_id: &TenantShardId, config_path: &Utf8Path, location_conf: &LocationConf, - ) -> anyhow::Result<()> { + ) -> std::io::Result<()> { debug!("persisting tenantconf to {config_path}"); let mut conf_content = r#"# This file contains a specific per-tenant's config. @@ -2620,22 +2628,20 @@ impl Tenant { .to_string(); fail::fail_point!("tenant-config-before-write", |_| { - anyhow::bail!("tenant-config-before-write"); + Err(std::io::Error::new( + std::io::ErrorKind::Other, + "tenant-config-before-write", + )) }); // Convert the config to a toml file. - conf_content += &toml_edit::ser::to_string_pretty(&location_conf)?; + conf_content += + &toml_edit::ser::to_string_pretty(&location_conf).expect("Config serialization failed"); let temp_path = path_with_suffix_extension(config_path, TEMP_FILE_SUFFIX); - let tenant_shard_id = *tenant_shard_id; - let config_path = config_path.to_owned(); let conf_content = conf_content.into_bytes(); - VirtualFile::crashsafe_overwrite(config_path.clone(), temp_path, conf_content) - .await - .with_context(|| format!("write tenant {tenant_shard_id} config to {config_path}"))?; - - Ok(()) + VirtualFile::crashsafe_overwrite(config_path.to_owned(), temp_path, conf_content).await } // diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 08c3f19b6f75..c1da1d2c55fd 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -43,7 +43,8 @@ use crate::tenant::config::{ use crate::tenant::span::debug_assert_current_span_has_tenant_id; use crate::tenant::storage_layer::inmemory_layer; use crate::tenant::timeline::ShutdownMode; -use crate::tenant::{AttachedTenantConf, GcError, SpawnMode, Tenant, TenantState}; +use crate::tenant::{AttachedTenantConf, GcError, LoadConfigError, SpawnMode, Tenant, TenantState}; +use crate::virtual_file::MaybeFatalIo; use crate::{InitializationOrder, TEMP_FILE_SUFFIX}; use utils::crashsafe::path_with_suffix_extension; @@ -272,7 +273,7 @@ pub struct TenantManager { } fn emergency_generations( - tenant_confs: &HashMap>, + tenant_confs: &HashMap>, ) -> HashMap { tenant_confs .iter() @@ -296,7 +297,7 @@ fn emergency_generations( async fn init_load_generations( conf: &'static PageServerConf, - tenant_confs: &HashMap>, + tenant_confs: &HashMap>, resources: &TenantSharedResources, cancel: &CancellationToken, ) -> anyhow::Result>> { @@ -346,56 +347,32 @@ async fn init_load_generations( /// Given a directory discovered in the pageserver's tenants/ directory, attempt /// to load a tenant config from it. /// -/// If file is missing, return Ok(None) +/// If we cleaned up something expected (like an empty dir or a temp dir), return None. fn load_tenant_config( conf: &'static PageServerConf, + tenant_shard_id: TenantShardId, dentry: Utf8DirEntry, -) -> anyhow::Result)>> { +) -> Option> { let tenant_dir_path = dentry.path().to_path_buf(); if crate::is_temporary(&tenant_dir_path) { info!("Found temporary tenant directory, removing: {tenant_dir_path}"); // No need to use safe_remove_tenant_dir_all because this is already // a temporary path - if let Err(e) = std::fs::remove_dir_all(&tenant_dir_path) { - error!( - "Failed to remove temporary directory '{}': {:?}", - tenant_dir_path, e - ); - } - return Ok(None); + std::fs::remove_dir_all(&tenant_dir_path).fatal_err("Deleting temporary tenant dir"); + return None; } // This case happens if we crash during attachment before writing a config into the dir let is_empty = tenant_dir_path .is_empty_dir() - .with_context(|| format!("Failed to check whether {tenant_dir_path:?} is an empty dir"))?; + .fatal_err("Checking for empty tenant dir"); if is_empty { info!("removing empty tenant directory {tenant_dir_path:?}"); - if let Err(e) = std::fs::remove_dir(&tenant_dir_path) { - error!( - "Failed to remove empty tenant directory '{}': {e:#}", - tenant_dir_path - ) - } - return Ok(None); + std::fs::remove_dir(&tenant_dir_path).fatal_err("Deleting empty tenant dir"); + return None; } - let tenant_shard_id = match tenant_dir_path - .file_name() - .unwrap_or_default() - .parse::() - { - Ok(id) => id, - Err(_) => { - warn!("Invalid tenant path (garbage in our repo directory?): {tenant_dir_path}",); - return Ok(None); - } - }; - - Ok(Some(( - tenant_shard_id, - Tenant::load_tenant_config(conf, &tenant_shard_id), - ))) + Some(Tenant::load_tenant_config(conf, &tenant_shard_id)) } /// Initial stage of load: walk the local tenants directory, clean up any temp files, @@ -405,32 +382,51 @@ fn load_tenant_config( /// seconds even on reasonably fast drives. async fn init_load_tenant_configs( conf: &'static PageServerConf, -) -> anyhow::Result>> { +) -> HashMap> { let tenants_dir = conf.tenants_path(); - let dentries = tokio::task::spawn_blocking(move || -> anyhow::Result> { - let dir_entries = tenants_dir - .read_dir_utf8() - .with_context(|| format!("Failed to list tenants dir {tenants_dir:?}"))?; + let dentries = tokio::task::spawn_blocking(move || -> Vec { + let context = format!("Reading tenants dir {tenants_dir}"); + let dir_entries = tenants_dir.read_dir_utf8().fatal_err(&context); - Ok(dir_entries.collect::, std::io::Error>>()?) + dir_entries + .collect::, std::io::Error>>() + .fatal_err(&context) }) - .await??; + .await + .expect("Config load task panicked"); let mut configs = HashMap::new(); let mut join_set = JoinSet::new(); for dentry in dentries { - join_set.spawn_blocking(move || load_tenant_config(conf, dentry)); + let tenant_shard_id = match dentry.file_name().parse::() { + Ok(id) => id, + Err(_) => { + warn!( + "Invalid tenant path (garbage in our repo directory?): '{}'", + dentry.file_name() + ); + continue; + } + }; + + join_set.spawn_blocking(move || { + ( + tenant_shard_id, + load_tenant_config(conf, tenant_shard_id, dentry), + ) + }); } while let Some(r) = join_set.join_next().await { - if let Some((tenant_id, tenant_config)) = r?? { - configs.insert(tenant_id, tenant_config); + let (tenant_shard_id, tenant_config) = r.expect("Panic in config load task"); + if let Some(tenant_config) = tenant_config { + configs.insert(tenant_shard_id, tenant_config); } } - Ok(configs) + configs } #[derive(Debug, thiserror::Error)] @@ -472,7 +468,7 @@ pub async fn init_tenant_mgr( ); // Scan local filesystem for attached tenants - let tenant_configs = init_load_tenant_configs(conf).await?; + let tenant_configs = init_load_tenant_configs(conf).await; // Determine which tenants are to be secondary or attached, and in which generation let tenant_modes = init_load_generations(conf, &tenant_configs, &resources, &cancel).await?; @@ -590,31 +586,23 @@ pub async fn init_tenant_mgr( ); // For those shards that have live configurations, construct `Tenant` or `SecondaryTenant` objects and start them running for (tenant_shard_id, location_conf, config_write_result) in config_write_results { - // Errors writing configs are fatal - config_write_result?; + // Writing a config to local disk is foundational to startup up tenants: panic if we can't. + config_write_result.fatal_err("writing tenant shard config file"); let tenant_dir_path = conf.tenant_path(&tenant_shard_id); let shard_identity = location_conf.shard; let slot = match location_conf.mode { - LocationMode::Attached(attached_conf) => { - match tenant_spawn( - conf, - tenant_shard_id, - &tenant_dir_path, - resources.clone(), - AttachedTenantConf::new(location_conf.tenant_conf, attached_conf), - shard_identity, - Some(init_order.clone()), - SpawnMode::Lazy, - &ctx, - ) { - Ok(tenant) => TenantSlot::Attached(tenant), - Err(e) => { - error!(tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), "Failed to start tenant: {e:#}"); - continue; - } - } - } + LocationMode::Attached(attached_conf) => TenantSlot::Attached(tenant_spawn( + conf, + tenant_shard_id, + &tenant_dir_path, + resources.clone(), + AttachedTenantConf::new(location_conf.tenant_conf, attached_conf), + shard_identity, + Some(init_order.clone()), + SpawnMode::Lazy, + &ctx, + )), LocationMode::Secondary(secondary_conf) => { info!( tenant_id = %tenant_shard_id.tenant_id, @@ -649,8 +637,7 @@ pub async fn init_tenant_mgr( }) } -/// Wrapper for Tenant::spawn that checks invariants before running, and inserts -/// a broken tenant in the map if Tenant::spawn fails. +/// Wrapper for Tenant::spawn that checks invariants before running #[allow(clippy::too_many_arguments)] fn tenant_spawn( conf: &'static PageServerConf, @@ -662,23 +649,18 @@ fn tenant_spawn( init_order: Option, mode: SpawnMode, ctx: &RequestContext, -) -> anyhow::Result> { - anyhow::ensure!( - tenant_path.is_dir(), - "Cannot load tenant from path {tenant_path:?}, it either does not exist or not a directory" - ); - anyhow::ensure!( - !crate::is_temporary(tenant_path), - "Cannot load tenant from temporary path {tenant_path:?}" - ); - anyhow::ensure!( - !tenant_path.is_empty_dir().with_context(|| { - format!("Failed to check whether {tenant_path:?} is an empty dir") - })?, - "Cannot load tenant from empty directory {tenant_path:?}" - ); - - let tenant = Tenant::spawn( +) -> Arc { + // All these conditions should have been satisfied by our caller: the tenant dir exists, is a well formed + // path, and contains a configuration file. Assertions that do synchronous I/O are limited to debug mode + // to avoid impacting prod runtime performance. + assert!(!crate::is_temporary(tenant_path)); + debug_assert!(tenant_path.is_dir()); + debug_assert!(conf + .tenant_location_config_path(&tenant_shard_id) + .try_exists() + .unwrap()); + + Tenant::spawn( conf, tenant_shard_id, resources, @@ -687,9 +669,7 @@ fn tenant_spawn( init_order, mode, ctx, - ); - - Ok(tenant) + ) } async fn shutdown_all_tenants0(tenants: &std::sync::RwLock) { @@ -840,8 +820,9 @@ pub(crate) enum UpsertLocationError { #[error("Failed to flush: {0}")] Flush(anyhow::Error), + /// This error variant is for unexpected situations (soft assertions) where the system is in an unexpected state. #[error("Internal error: {0}")] - Other(#[from] anyhow::Error), + InternalError(anyhow::Error), } impl TenantManager { @@ -971,7 +952,8 @@ impl TenantManager { match fast_path_taken { Some(FastPathModified::Attached(tenant)) => { Tenant::persist_tenant_config(self.conf, &tenant_shard_id, &new_location_config) - .await?; + .await + .fatal_err("writing tenant shard config"); // Transition to AttachedStale means we may well hold a valid generation // still, and have been requested to go stale as part of a migration. If @@ -1001,7 +983,8 @@ impl TenantManager { } Some(FastPathModified::Secondary(_secondary_tenant)) => { Tenant::persist_tenant_config(self.conf, &tenant_shard_id, &new_location_config) - .await?; + .await + .fatal_err("writing tenant shard config"); return Ok(None); } @@ -1067,7 +1050,7 @@ impl TenantManager { Some(TenantSlot::InProgress(_)) => { // This should never happen: acquire_slot should error out // if the contents of a slot were InProgress. - return Err(UpsertLocationError::Other(anyhow::anyhow!( + return Err(UpsertLocationError::InternalError(anyhow::anyhow!( "Acquired an InProgress slot, this is a bug." ))); } @@ -1086,12 +1069,14 @@ impl TenantManager { // Does not need to be fsync'd because local storage is just a cache. tokio::fs::create_dir_all(&timelines_path) .await - .with_context(|| format!("Creating {timelines_path}"))?; + .fatal_err("creating timelines/ dir"); // Before activating either secondary or attached mode, persist the // configuration, so that on restart we will re-attach (or re-start // secondary) on the tenant. - Tenant::persist_tenant_config(self.conf, &tenant_shard_id, &new_location_config).await?; + Tenant::persist_tenant_config(self.conf, &tenant_shard_id, &new_location_config) + .await + .fatal_err("writing tenant shard config"); let new_slot = match &new_location_config.mode { LocationMode::Secondary(secondary_config) => { @@ -1110,13 +1095,15 @@ impl TenantManager { // from upserts. This enables creating generation-less tenants even though neon_local // always uses generations when calling the location conf API. let attached_conf = if cfg!(feature = "testing") { - let mut conf = AttachedTenantConf::try_from(new_location_config)?; + let mut conf = AttachedTenantConf::try_from(new_location_config) + .map_err(UpsertLocationError::BadRequest)?; if self.conf.control_plane_api.is_none() { conf.location.generation = Generation::none(); } conf } else { - AttachedTenantConf::try_from(new_location_config)? + AttachedTenantConf::try_from(new_location_config) + .map_err(UpsertLocationError::BadRequest)? }; let tenant = tenant_spawn( @@ -1129,7 +1116,7 @@ impl TenantManager { None, spawn_mode, ctx, - )?; + ); TenantSlot::Attached(tenant) } @@ -1143,7 +1130,7 @@ impl TenantManager { match slot_guard.upsert(new_slot) { Err(TenantSlotUpsertError::InternalError(e)) => { - Err(UpsertLocationError::Other(anyhow::anyhow!(e))) + Err(UpsertLocationError::InternalError(anyhow::anyhow!(e))) } Err(TenantSlotUpsertError::MapState(e)) => Err(UpsertLocationError::Unavailable(e)), Err(TenantSlotUpsertError::ShuttingDown((new_slot, _completion))) => { @@ -1250,7 +1237,7 @@ impl TenantManager { None, SpawnMode::Eager, ctx, - )?; + ); slot_guard.upsert(TenantSlot::Attached(tenant))?; @@ -1984,7 +1971,7 @@ impl TenantManager { None, SpawnMode::Eager, ctx, - )?; + ); slot_guard.upsert(TenantSlot::Attached(tenant))?; diff --git a/test_runner/regress/test_tenants.py b/test_runner/regress/test_tenants.py index 93e9ad367367..3705406c2ff9 100644 --- a/test_runner/regress/test_tenants.py +++ b/test_runner/regress/test_tenants.py @@ -41,18 +41,35 @@ def test_tenant_creation_fails(neon_simple_env: NeonEnv): neon_simple_env.storage_controller.allowed_errors.extend(error_regexes) pageserver_http = neon_simple_env.pageserver.http_client() + + # Failure to write a config to local disk makes the pageserver assume that local disk is bad and abort the process pageserver_http.configure_failpoints(("tenant-config-before-write", "return")) - with pytest.raises(Exception, match="tenant-config-before-write"): + + # Storage controller will see a torn TCP connection when the crash point is reached, and follow an unclean 500 error path + neon_simple_env.storage_controller.allowed_errors.extend( + [ + ".*Reconcile not done yet while creating tenant.*", + ".*Reconcile error: receive body: error sending request.*", + ".*Error processing HTTP request: InternalServerError.*", + ] + ) + + with pytest.raises(Exception, match="error sending request"): _ = neon_simple_env.neon_cli.create_tenant() + # Any files left behind on disk during failed creation do not prevent + # a retry from succeeding. Restart pageserver with no failpoints. + neon_simple_env.pageserver.running = False + neon_simple_env.pageserver.start() + + # The failed creation should not be present in list of tenants, as when we start up we'll see + # an empty tenant dir with no config in it. + neon_simple_env.pageserver.allowed_errors.append(".*Failed to load tenant config.*") new_tenants = sorted( map(lambda t: t.split()[0], neon_simple_env.neon_cli.list_tenants().stdout.splitlines()) ) assert initial_tenants == new_tenants, "should not create new tenants" - # Any files left behind on disk during failed creation do not prevent - # a retry from succeeding. - pageserver_http.configure_failpoints(("tenant-config-before-write", "off")) neon_simple_env.neon_cli.create_tenant() From 9b4b4bbf6f4b801250ea3b683cc91a626392d12f Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 2 Jul 2024 15:13:27 +0200 Subject: [PATCH 013/194] fix: noisy logging when download gets cancelled during shutdown (#8224) Before this PR, during timeline shutdown, we'd occasionally see log lines like this one: ``` 2024-06-26T18:28:11.063402Z INFO initial_size_calculation{tenant_id=$TENANT,shard_id=0000 timeline_id=$TIMELINE}:logical_size_calculation_task:get_or_maybe_download{layer=000000000000000000000000000000000000-000000067F0001A3950001C1630100000000__0000000D88265898}: layer file download failed, and caller has been cancelled: Cancelled, shutting down Stack backtrace: 0: as core::ops::try_trait::FromResidual>>::from_residual at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/result.rs:1964:27 pageserver::tenant::remote_timeline_client::RemoteTimelineClient::download_layer_file::{{closure}} at /home/nonroot/pageserver/src/tenant/remote_timeline_client.rs:531:13 pageserver::tenant::storage_layer::layer::LayerInner::download_and_init::{{closure}} at /home/nonroot/pageserver/src/tenant/storage_layer/layer.rs:1136:14 pageserver::tenant::storage_layer::layer::LayerInner::download_init_and_wait::{{closure}}::{{closure}} at /home/nonroot/pageserver/src/tenant/storage_layer/layer.rs:1082:74 ``` We can eliminate the anyhow backtrace with no loss of information because the conversion to anyhow::Error happens in exactly one place. refs #7427 --- pageserver/src/tenant/remote_timeline_client.rs | 2 +- pageserver/src/tenant/storage_layer/layer.rs | 17 ++++------------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index e33e4b84aa97..bc9364de61d4 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -519,7 +519,7 @@ impl RemoteTimelineClient { local_path: &Utf8Path, cancel: &CancellationToken, ctx: &RequestContext, - ) -> anyhow::Result { + ) -> Result { let downloaded_size = { let _unfinished_gauge_guard = self.metrics.call_begin( &RemoteOpFileKind::Layer, diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index 5dd947253578..02069c29d264 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -1096,19 +1096,10 @@ impl LayerInner { match rx.await { Ok(Ok(res)) => Ok(res), - Ok(Err(e)) => { - // sleep already happened in the spawned task, if it was not cancelled - match e.downcast_ref::() { - // If the download failed due to its cancellation token, - // propagate the cancellation error upstream. - Some(remote_storage::DownloadError::Cancelled) => { - Err(DownloadError::DownloadCancelled) - } - // FIXME: this is not embedding the error because historically it would had - // been output to compute, however that is no longer the case. - _ => Err(DownloadError::DownloadFailed), - } + Ok(Err(remote_storage::DownloadError::Cancelled)) => { + Err(DownloadError::DownloadCancelled) } + Ok(Err(_)) => Err(DownloadError::DownloadFailed), Err(_gone) => Err(DownloadError::DownloadCancelled), } } @@ -1118,7 +1109,7 @@ impl LayerInner { timeline: Arc, permit: heavier_once_cell::InitPermit, ctx: &RequestContext, - ) -> anyhow::Result> { + ) -> Result, remote_storage::DownloadError> { let result = timeline .remote_client .download_layer_file( From 28929d9cfa03a003cc96925458a434ac31ec8f27 Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Tue, 2 Jul 2024 14:14:10 +0100 Subject: [PATCH 014/194] pageserver: rate limit log for loads of layers visited (#8228) ## Problem At high percentiles we see more than 800 layers being visited by the read path. We need the tenant/timeline to investigate. ## Summary of changes Add a rate limited log line when the average number of layers visited per key is in the last specified histogram bucket. I plan to use this to identify tenants in us-east-2 staging that exhibit this behaviour. Will revert before next week's release. --- libs/pageserver_api/src/keyspace.rs | 10 ++++++++++ pageserver/src/tenant/timeline.rs | 22 +++++++++++++++++++--- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/libs/pageserver_api/src/keyspace.rs b/libs/pageserver_api/src/keyspace.rs index 9a61f2ad81ae..401887d3629c 100644 --- a/libs/pageserver_api/src/keyspace.rs +++ b/libs/pageserver_api/src/keyspace.rs @@ -17,6 +17,16 @@ pub struct KeySpace { pub ranges: Vec>, } +impl std::fmt::Display for KeySpace { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "[")?; + for range in &self.ranges { + write!(f, "{}..{},", range.start, range.end)?; + } + write!(f, "]") + } +} + /// A wrapper type for sparse keyspaces. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct SparseKeySpace(pub KeySpace); diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 8dd0a23f4637..ec94ed3a56db 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -996,6 +996,7 @@ impl Timeline { } pub(crate) const MAX_GET_VECTORED_KEYS: u64 = 32; + pub(crate) const VEC_GET_LAYERS_VISITED_WARN_THRESH: f64 = 512.0; /// Look up multiple page versions at a given LSN /// @@ -1228,7 +1229,7 @@ impl Timeline { let get_data_timer = crate::metrics::GET_RECONSTRUCT_DATA_TIME .for_get_kind(get_kind) .start_timer(); - self.get_vectored_reconstruct_data(keyspace, lsn, reconstruct_state, ctx) + self.get_vectored_reconstruct_data(keyspace.clone(), lsn, reconstruct_state, ctx) .await?; get_data_timer.stop_and_record(); @@ -1258,11 +1259,26 @@ impl Timeline { // (this is a requirement, not a bug). Skip updating the metric in these cases // to avoid infinite results. if !results.is_empty() { + let avg = layers_visited as f64 / results.len() as f64; + if avg >= Self::VEC_GET_LAYERS_VISITED_WARN_THRESH { + use utils::rate_limit::RateLimit; + static LOGGED: Lazy> = + Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10)))); + let mut rate_limit = LOGGED.lock().unwrap(); + rate_limit.call(|| { + tracing::info!( + tenant_id = %self.tenant_shard_id.tenant_id, + shard_id = %self.tenant_shard_id.shard_slug(), + timeline_id = %self.timeline_id, + "Vectored read for {} visited {} layers on average per key and {} in total. {}/{} pages were returned", + keyspace, avg, layers_visited, results.len(), keyspace.total_raw_size()); + }); + } + // Note that this is an approximation. Tracking the exact number of layers visited // per key requires virtually unbounded memory usage and is inefficient // (i.e. segment tree tracking each range queried from a layer) - crate::metrics::VEC_READ_NUM_LAYERS_VISITED - .observe(layers_visited as f64 / results.len() as f64); + crate::metrics::VEC_READ_NUM_LAYERS_VISITED.observe(avg); } Ok(results) From 25eefdeb1fe2f217ec4e3b8f4d2dff9fd702ab60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Tue, 2 Jul 2024 16:14:12 +0200 Subject: [PATCH 015/194] Add support for reading and writing compressed blobs (#8106) Add support for reading and writing zstd-compressed blobs for use in image layer generation, but maybe one day useful also for delta layers. The reading of them is unconditional while the writing is controlled by the `image_compression` config variable allowing for experiments. For the on-disk format, we re-use some of the bitpatterns we currently keep reserved for blobs larger than 256 MiB. This assumes that we have never ever written any such large blobs to image layers. After the preparation in #7852, we now are unable to read blobs with a size larger than 256 MiB (or write them). A non-goal of this PR is to come up with good heuristics of when to compress a bitpattern. This is left for future work. Parts of the PR were inspired by #7091. cc #7879 Part of #5431 --- libs/pageserver_api/src/models.rs | 18 ++ pageserver/src/config.rs | 21 ++- pageserver/src/tenant/blob_io.rs | 155 +++++++++++++++--- .../src/tenant/storage_layer/delta_layer.rs | 7 +- 4 files changed, 177 insertions(+), 24 deletions(-) diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index 61a255cdbc80..959e161c167a 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -432,6 +432,24 @@ pub enum CompactionAlgorithm { Tiered, } +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + Serialize, + Deserialize, + strum_macros::FromRepr, + strum_macros::EnumString, +)] +#[strum(serialize_all = "kebab-case")] +pub enum ImageCompressionAlgorithm { + /// Zstandard compression. Level 0 means and None mean the same (default level). Levels can be negative as well. + /// For details, see the [manual](http://facebook.github.io/zstd/zstd_manual.html). + Zstd { level: Option }, +} + #[derive(Eq, PartialEq, Debug, Clone, Serialize, Deserialize)] pub struct CompactionAlgorithmSettings { pub kind: CompactionAlgorithm, diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index 2b698b75dcb1..470e941c33f9 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -5,7 +5,7 @@ //! See also `settings.md` for better description on every parameter. use anyhow::{anyhow, bail, ensure, Context, Result}; -use pageserver_api::shard::TenantShardId; +use pageserver_api::{models::ImageCompressionAlgorithm, shard::TenantShardId}; use remote_storage::{RemotePath, RemoteStorageConfig}; use serde; use serde::de::IntoDeserializer; @@ -50,6 +50,7 @@ pub mod defaults { DEFAULT_HTTP_LISTEN_ADDR, DEFAULT_HTTP_LISTEN_PORT, DEFAULT_PG_LISTEN_ADDR, DEFAULT_PG_LISTEN_PORT, }; + use pageserver_api::models::ImageCompressionAlgorithm; pub use storage_broker::DEFAULT_ENDPOINT as BROKER_DEFAULT_ENDPOINT; pub const DEFAULT_WAIT_LSN_TIMEOUT: &str = "60 s"; @@ -90,6 +91,8 @@ pub mod defaults { pub const DEFAULT_MAX_VECTORED_READ_BYTES: usize = 128 * 1024; // 128 KiB + pub const DEFAULT_IMAGE_COMPRESSION: Option = None; + pub const DEFAULT_VALIDATE_VECTORED_GET: bool = true; pub const DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB: usize = 0; @@ -285,6 +288,8 @@ pub struct PageServerConf { pub validate_vectored_get: bool, + pub image_compression: Option, + /// How many bytes of ephemeral layer content will we allow per kilobyte of RAM. When this /// is exceeded, we start proactively closing ephemeral layers to limit the total amount /// of ephemeral data. @@ -395,6 +400,8 @@ struct PageServerConfigBuilder { validate_vectored_get: BuilderValue, + image_compression: BuilderValue>, + ephemeral_bytes_per_memory_kb: BuilderValue, } @@ -482,6 +489,7 @@ impl PageServerConfigBuilder { max_vectored_read_bytes: Set(MaxVectoredReadBytes( NonZeroUsize::new(DEFAULT_MAX_VECTORED_READ_BYTES).unwrap(), )), + image_compression: Set(DEFAULT_IMAGE_COMPRESSION), validate_vectored_get: Set(DEFAULT_VALIDATE_VECTORED_GET), ephemeral_bytes_per_memory_kb: Set(DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB), } @@ -667,6 +675,10 @@ impl PageServerConfigBuilder { self.validate_vectored_get = BuilderValue::Set(value); } + pub fn get_image_compression(&mut self, value: Option) { + self.image_compression = BuilderValue::Set(value); + } + pub fn get_ephemeral_bytes_per_memory_kb(&mut self, value: usize) { self.ephemeral_bytes_per_memory_kb = BuilderValue::Set(value); } @@ -727,6 +739,7 @@ impl PageServerConfigBuilder { get_impl, max_vectored_read_bytes, validate_vectored_get, + image_compression, ephemeral_bytes_per_memory_kb, } CUSTOM LOGIC @@ -1004,6 +1017,9 @@ impl PageServerConf { "validate_vectored_get" => { builder.get_validate_vectored_get(parse_toml_bool("validate_vectored_get", item)?) } + "image_compression" => { + builder.get_image_compression(Some(parse_toml_from_str("image_compression", item)?)) + } "ephemeral_bytes_per_memory_kb" => { builder.get_ephemeral_bytes_per_memory_kb(parse_toml_u64("ephemeral_bytes_per_memory_kb", item)? as usize) } @@ -1088,6 +1104,7 @@ impl PageServerConf { NonZeroUsize::new(defaults::DEFAULT_MAX_VECTORED_READ_BYTES) .expect("Invalid default constant"), ), + image_compression: defaults::DEFAULT_IMAGE_COMPRESSION, validate_vectored_get: defaults::DEFAULT_VALIDATE_VECTORED_GET, ephemeral_bytes_per_memory_kb: defaults::DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB, } @@ -1328,6 +1345,7 @@ background_task_maximum_delay = '334 s' .expect("Invalid default constant") ), validate_vectored_get: defaults::DEFAULT_VALIDATE_VECTORED_GET, + image_compression: defaults::DEFAULT_IMAGE_COMPRESSION, ephemeral_bytes_per_memory_kb: defaults::DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB, }, "Correct defaults should be used when no config values are provided" @@ -1401,6 +1419,7 @@ background_task_maximum_delay = '334 s' .expect("Invalid default constant") ), validate_vectored_get: defaults::DEFAULT_VALIDATE_VECTORED_GET, + image_compression: defaults::DEFAULT_IMAGE_COMPRESSION, ephemeral_bytes_per_memory_kb: defaults::DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB, }, "Should be able to parse all basic config values correctly" diff --git a/pageserver/src/tenant/blob_io.rs b/pageserver/src/tenant/blob_io.rs index 2be8816cefbd..022801b17fba 100644 --- a/pageserver/src/tenant/blob_io.rs +++ b/pageserver/src/tenant/blob_io.rs @@ -6,12 +6,18 @@ //! is written as a one byte. If it's larger than that, the length //! is written as a four-byte integer, in big-endian, with the high //! bit set. This way, we can detect whether it's 1- or 4-byte header -//! by peeking at the first byte. +//! by peeking at the first byte. For blobs larger than 128 bits, +//! we also specify three reserved bits, only one of the three bit +//! patterns is currently in use (0b011) and signifies compression +//! with zstd. //! //! len < 128: 0XXXXXXX -//! len >= 128: 1XXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX +//! len >= 128: 1CCCXXXX XXXXXXXX XXXXXXXX XXXXXXXX //! +use async_compression::Level; use bytes::{BufMut, BytesMut}; +use pageserver_api::models::ImageCompressionAlgorithm; +use tokio::io::AsyncWriteExt; use tokio_epoll_uring::{BoundedBuf, IoBuf, Slice}; use crate::context::RequestContext; @@ -66,12 +72,29 @@ impl<'a> BlockCursor<'a> { len_buf.copy_from_slice(&buf[off..off + 4]); off += 4; } - len_buf[0] &= 0x7f; + len_buf[0] &= !LEN_COMPRESSION_BIT_MASK; u32::from_be_bytes(len_buf) as usize }; + let compression_bits = first_len_byte & LEN_COMPRESSION_BIT_MASK; - dstbuf.clear(); - dstbuf.reserve(len); + let mut tmp_buf = Vec::new(); + let buf_to_write; + let compression = if compression_bits <= BYTE_UNCOMPRESSED { + buf_to_write = dstbuf; + None + } else if compression_bits == BYTE_ZSTD { + buf_to_write = &mut tmp_buf; + Some(dstbuf) + } else { + let error = std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("invalid compression byte {compression_bits:x}"), + ); + return Err(error); + }; + + buf_to_write.clear(); + buf_to_write.reserve(len); // Read the payload let mut remain = len; @@ -85,14 +108,35 @@ impl<'a> BlockCursor<'a> { page_remain = PAGE_SZ; } let this_blk_len = min(remain, page_remain); - dstbuf.extend_from_slice(&buf[off..off + this_blk_len]); + buf_to_write.extend_from_slice(&buf[off..off + this_blk_len]); remain -= this_blk_len; off += this_blk_len; } + + if let Some(dstbuf) = compression { + if compression_bits == BYTE_ZSTD { + let mut decoder = async_compression::tokio::write::ZstdDecoder::new(dstbuf); + decoder.write_all(buf_to_write).await?; + decoder.flush().await?; + } else { + unreachable!("already checked above") + } + } + Ok(()) } } +/// Reserved bits for length and compression +const LEN_COMPRESSION_BIT_MASK: u8 = 0xf0; + +/// The maximum size of blobs we support. The highest few bits +/// are reserved for compression and other further uses. +const MAX_SUPPORTED_LEN: usize = 0x0fff_ffff; + +const BYTE_UNCOMPRESSED: u8 = 0x80; +const BYTE_ZSTD: u8 = BYTE_UNCOMPRESSED | 0x10; + /// A wrapper of `VirtualFile` that allows users to write blobs. /// /// If a `BlobWriter` is dropped, the internal buffer will be @@ -219,6 +263,17 @@ impl BlobWriter { &mut self, srcbuf: B, ctx: &RequestContext, + ) -> (B::Buf, Result) { + self.write_blob_maybe_compressed(srcbuf, ctx, None).await + } + + /// Write a blob of data. Returns the offset that it was written to, + /// which can be used to retrieve the data later. + pub async fn write_blob_maybe_compressed, Buf: IoBuf + Send>( + &mut self, + srcbuf: B, + ctx: &RequestContext, + algorithm: Option, ) -> (B::Buf, Result) { let offset = self.offset; @@ -226,29 +281,58 @@ impl BlobWriter { let mut io_buf = self.io_buf.take().expect("we always put it back below"); io_buf.clear(); - let (io_buf, hdr_res) = async { + let mut compressed_buf = None; + let ((io_buf, hdr_res), srcbuf) = async { if len < 128 { // Short blob. Write a 1-byte length header io_buf.put_u8(len as u8); - self.write_all(io_buf, ctx).await + ( + self.write_all(io_buf, ctx).await, + srcbuf.slice_full().into_inner(), + ) } else { // Write a 4-byte length header - if len > 0x7fff_ffff { + if len > MAX_SUPPORTED_LEN { return ( - io_buf, - Err(Error::new( - ErrorKind::Other, - format!("blob too large ({len} bytes)"), - )), + ( + io_buf, + Err(Error::new( + ErrorKind::Other, + format!("blob too large ({len} bytes)"), + )), + ), + srcbuf.slice_full().into_inner(), ); } - if len > 0x0fff_ffff { - tracing::warn!("writing blob above future limit ({len} bytes)"); - } - let mut len_buf = (len as u32).to_be_bytes(); - len_buf[0] |= 0x80; + let (high_bit_mask, len_written, srcbuf) = match algorithm { + Some(ImageCompressionAlgorithm::Zstd { level }) => { + let mut encoder = if let Some(level) = level { + async_compression::tokio::write::ZstdEncoder::with_quality( + Vec::new(), + Level::Precise(level.into()), + ) + } else { + async_compression::tokio::write::ZstdEncoder::new(Vec::new()) + }; + let slice = srcbuf.slice_full(); + encoder.write_all(&slice[..]).await.unwrap(); + encoder.shutdown().await.unwrap(); + let compressed = encoder.into_inner(); + if compressed.len() < len { + let compressed_len = compressed.len(); + compressed_buf = Some(compressed); + (BYTE_ZSTD, compressed_len, slice.into_inner()) + } else { + (BYTE_UNCOMPRESSED, len, slice.into_inner()) + } + } + None => (BYTE_UNCOMPRESSED, len, srcbuf.slice_full().into_inner()), + }; + let mut len_buf = (len_written as u32).to_be_bytes(); + assert_eq!(len_buf[0] & 0xf0, 0); + len_buf[0] |= high_bit_mask; io_buf.extend_from_slice(&len_buf[..]); - self.write_all(io_buf, ctx).await + (self.write_all(io_buf, ctx).await, srcbuf) } } .await; @@ -257,7 +341,12 @@ impl BlobWriter { Ok(_) => (), Err(e) => return (Slice::into_inner(srcbuf.slice(..)), Err(e)), } - let (srcbuf, res) = self.write_all(srcbuf, ctx).await; + let (srcbuf, res) = if let Some(compressed_buf) = compressed_buf { + let (_buf, res) = self.write_all(compressed_buf, ctx).await; + (Slice::into_inner(srcbuf.slice(..)), res) + } else { + self.write_all(srcbuf, ctx).await + }; (srcbuf, res.map(|_| offset)) } } @@ -295,6 +384,12 @@ mod tests { use rand::{Rng, SeedableRng}; async fn round_trip_test(blobs: &[Vec]) -> Result<(), Error> { + round_trip_test_compressed::(blobs).await + } + + async fn round_trip_test_compressed( + blobs: &[Vec], + ) -> Result<(), Error> { let temp_dir = camino_tempfile::tempdir()?; let pathbuf = temp_dir.path().join("file"); let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error); @@ -305,7 +400,18 @@ mod tests { let file = VirtualFile::create(pathbuf.as_path(), &ctx).await?; let mut wtr = BlobWriter::::new(file, 0); for blob in blobs.iter() { - let (_, res) = wtr.write_blob(blob.clone(), &ctx).await; + let (_, res) = match COMPRESSION { + 0 => wtr.write_blob(blob.clone(), &ctx).await, + 1 => { + wtr.write_blob_maybe_compressed( + blob.clone(), + &ctx, + Some(ImageCompressionAlgorithm::Zstd { level: Some(1) }), + ) + .await + } + _ => unreachable!("Invalid compression {COMPRESSION}"), + }; let offs = res?; offsets.push(offs); } @@ -361,10 +467,15 @@ mod tests { let blobs = &[ b"test".to_vec(), random_array(10 * PAGE_SZ), + b"hello".to_vec(), + random_array(66 * PAGE_SZ), + vec![0xf3; 24 * PAGE_SZ], b"foobar".to_vec(), ]; round_trip_test::(blobs).await?; round_trip_test::(blobs).await?; + round_trip_test_compressed::(blobs).await?; + round_trip_test_compressed::(blobs).await?; Ok(()) } diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index c2d4a2776b1d..e6a4d6d5c45a 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -452,7 +452,12 @@ impl DeltaLayerWriterInner { ctx: &RequestContext, ) -> (Vec, anyhow::Result<()>) { assert!(self.lsn_range.start <= lsn); - let (val, res) = self.blob_writer.write_blob(val, ctx).await; + // We don't want to use compression in delta layer creation + let compression = None; + let (val, res) = self + .blob_writer + .write_blob_maybe_compressed(val, ctx, compression) + .await; let off = match res { Ok(off) => off, Err(e) => return (val, Err(anyhow::anyhow!(e))), From 5de896e7d890271362966ebb6a42f16b5b8cd966 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 2 Jul 2024 16:29:09 +0200 Subject: [PATCH 016/194] L0 flush: opt-in mechanism to bypass PageCache reads and writes (#8190) part of https://github.com/neondatabase/neon/issues/7418 # Motivation (reproducing #7418) When we do an `InMemoryLayer::write_to_disk`, there is a tremendous amount of random read I/O, as deltas from the ephemeral file (written in LSN order) are written out to the delta layer in key order. In benchmarks (https://github.com/neondatabase/neon/pull/7409) we can see that this delta layer writing phase is substantially more expensive than the initial ingest of data, and that within the delta layer write a significant amount of the CPU time is spent traversing the page cache. # High-Level Changes Add a new mode for L0 flush that works as follows: * Read the full ephemeral file into memory -- layers are much smaller than total memory, so this is afforable * Do all the random reads directly from this in memory buffer instead of using blob IO/page cache/disk reads. * Add a semaphore to limit how many timelines may concurrently do this (limit peak memory). * Make the semaphore configurable via PS config. # Implementation Details The new `BlobReaderRef::Slice` is a temporary hack until we can ditch `blob_io` for `InMemoryLayer` => Plan for this is laid out in https://github.com/neondatabase/neon/issues/8183 # Correctness The correctness of this change is quite obvious to me: we do what we did before (`blob_io`) but read from memory instead of going to disk. The highest bug potential is in doing owned-buffers IO. I refactored the API a bit in preliminary PR https://github.com/neondatabase/neon/pull/8186 to make it less error-prone, but still, careful review is requested. # Performance I manually measured single-client ingest performance from `pgbench -i ...`. Full report: https://neondatabase.notion.site/2024-06-28-benchmarking-l0-flush-performance-e98cff3807f94cb38f2054d8c818fe84?pvs=4 tl;dr: * no speed improvements during ingest, but * significantly lower pressure on PS PageCache (eviction rate drops to 1/3) * (that's why I'm working on this) * noticable but modestly lower CPU time This is good enough for merging this PR because the changes require opt-in. We'll do more testing in staging & pre-prod. # Stability / Monitoring **memory consumption**: there's no _hard_ limit on max `InMemoryLayer` size (aka "checkpoint distance") , hence there's no hard limit on the memory allocation we do for flushing. In practice, we a) [log a warning](https://github.com/neondatabase/neon/blob/23827c6b0d400cbb9a972d4d05d49834816c40d1/pageserver/src/tenant/timeline.rs#L5741-L5743) when we flush oversized layers, so we'd know which tenant is to blame and b) if we were to put a hard limit in place, we would have to decide what to do if there is an InMemoryLayer that exceeds the limit. It seems like a better option to guarantee a max size for frozen layer, dependent on `checkpoint_distance`. Then limit concurrency based on that. **metrics**: we do have the [flush_time_histo](https://github.com/neondatabase/neon/blob/23827c6b0d400cbb9a972d4d05d49834816c40d1/pageserver/src/tenant/timeline.rs#L3725-L3726), but that includes the wait time for the semaphore. We could add a separate metric for the time spent after acquiring the semaphore, so one can infer the wait time. Seems unnecessary at this point, though. --- pageserver/src/bin/pageserver.rs | 5 + pageserver/src/config.rs | 18 ++- pageserver/src/l0_flush.rs | 46 ++++++ pageserver/src/lib.rs | 1 + pageserver/src/tenant.rs | 13 ++ pageserver/src/tenant/block_io.rs | 22 +++ pageserver/src/tenant/ephemeral_file.rs | 8 +- .../src/tenant/ephemeral_file/page_caching.rs | 146 +++++++++++++----- .../ephemeral_file/zero_padded_read_write.rs | 15 ++ .../tenant/storage_layer/inmemory_layer.rs | 96 +++++++++--- pageserver/src/tenant/timeline.rs | 10 +- pageserver/src/tenant/timeline/delete.rs | 1 + 12 files changed, 323 insertions(+), 58 deletions(-) create mode 100644 pageserver/src/l0_flush.rs diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index ba5b2608bdf1..39d4e46c9663 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -421,6 +421,10 @@ fn start_pageserver( background_jobs_can_start: background_jobs_barrier.clone(), }; + info!(config=?conf.l0_flush, "using l0_flush config"); + let l0_flush_global_state = + pageserver::l0_flush::L0FlushGlobalState::new(conf.l0_flush.clone()); + // Scan the local 'tenants/' directory and start loading the tenants let deletion_queue_client = deletion_queue.new_client(); let tenant_manager = BACKGROUND_RUNTIME.block_on(mgr::init_tenant_mgr( @@ -429,6 +433,7 @@ fn start_pageserver( broker_client: broker_client.clone(), remote_storage: remote_storage.clone(), deletion_queue_client, + l0_flush_global_state, }, order, shutdown_pageserver.clone(), diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index 470e941c33f9..fa7f7d8d97c0 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -30,11 +30,11 @@ use utils::{ logging::LogFormat, }; -use crate::tenant::timeline::GetVectoredImpl; use crate::tenant::vectored_blob_io::MaxVectoredReadBytes; use crate::tenant::{config::TenantConfOpt, timeline::GetImpl}; use crate::tenant::{TENANTS_SEGMENT_NAME, TIMELINES_SEGMENT_NAME}; use crate::{disk_usage_eviction_task::DiskUsageEvictionTaskConfig, virtual_file::io_engine}; +use crate::{l0_flush::L0FlushConfig, tenant::timeline::GetVectoredImpl}; use crate::{tenant::config::TenantConf, virtual_file}; use crate::{TENANT_HEATMAP_BASENAME, TENANT_LOCATION_CONFIG_NAME, TIMELINE_DELETE_MARK_SUFFIX}; @@ -296,6 +296,8 @@ pub struct PageServerConf { /// /// Setting this to zero disables limits on total ephemeral layer size. pub ephemeral_bytes_per_memory_kb: usize, + + pub l0_flush: L0FlushConfig, } /// We do not want to store this in a PageServerConf because the latter may be logged @@ -403,6 +405,8 @@ struct PageServerConfigBuilder { image_compression: BuilderValue>, ephemeral_bytes_per_memory_kb: BuilderValue, + + l0_flush: BuilderValue, } impl PageServerConfigBuilder { @@ -492,6 +496,7 @@ impl PageServerConfigBuilder { image_compression: Set(DEFAULT_IMAGE_COMPRESSION), validate_vectored_get: Set(DEFAULT_VALIDATE_VECTORED_GET), ephemeral_bytes_per_memory_kb: Set(DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB), + l0_flush: Set(L0FlushConfig::default()), } } } @@ -683,6 +688,10 @@ impl PageServerConfigBuilder { self.ephemeral_bytes_per_memory_kb = BuilderValue::Set(value); } + pub fn l0_flush(&mut self, value: L0FlushConfig) { + self.l0_flush = BuilderValue::Set(value); + } + pub fn build(self) -> anyhow::Result { let default = Self::default_values(); @@ -741,6 +750,7 @@ impl PageServerConfigBuilder { validate_vectored_get, image_compression, ephemeral_bytes_per_memory_kb, + l0_flush, } CUSTOM LOGIC { @@ -1023,6 +1033,9 @@ impl PageServerConf { "ephemeral_bytes_per_memory_kb" => { builder.get_ephemeral_bytes_per_memory_kb(parse_toml_u64("ephemeral_bytes_per_memory_kb", item)? as usize) } + "l0_flush" => { + builder.l0_flush(utils::toml_edit_ext::deserialize_item(item).context("l0_flush")?) + } _ => bail!("unrecognized pageserver option '{key}'"), } } @@ -1107,6 +1120,7 @@ impl PageServerConf { image_compression: defaults::DEFAULT_IMAGE_COMPRESSION, validate_vectored_get: defaults::DEFAULT_VALIDATE_VECTORED_GET, ephemeral_bytes_per_memory_kb: defaults::DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB, + l0_flush: L0FlushConfig::default(), } } } @@ -1347,6 +1361,7 @@ background_task_maximum_delay = '334 s' validate_vectored_get: defaults::DEFAULT_VALIDATE_VECTORED_GET, image_compression: defaults::DEFAULT_IMAGE_COMPRESSION, ephemeral_bytes_per_memory_kb: defaults::DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB, + l0_flush: L0FlushConfig::default(), }, "Correct defaults should be used when no config values are provided" ); @@ -1421,6 +1436,7 @@ background_task_maximum_delay = '334 s' validate_vectored_get: defaults::DEFAULT_VALIDATE_VECTORED_GET, image_compression: defaults::DEFAULT_IMAGE_COMPRESSION, ephemeral_bytes_per_memory_kb: defaults::DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB, + l0_flush: L0FlushConfig::default(), }, "Should be able to parse all basic config values correctly" ); diff --git a/pageserver/src/l0_flush.rs b/pageserver/src/l0_flush.rs new file mode 100644 index 000000000000..7fe8fedc6394 --- /dev/null +++ b/pageserver/src/l0_flush.rs @@ -0,0 +1,46 @@ +use std::{num::NonZeroUsize, sync::Arc}; + +use crate::tenant::ephemeral_file; + +#[derive(Default, Debug, PartialEq, Eq, Clone, serde::Deserialize)] +#[serde(tag = "mode", rename_all = "kebab-case", deny_unknown_fields)] +pub enum L0FlushConfig { + #[default] + PageCached, + #[serde(rename_all = "snake_case")] + Direct { max_concurrency: NonZeroUsize }, +} + +#[derive(Clone)] +pub struct L0FlushGlobalState(Arc); + +pub(crate) enum Inner { + PageCached, + Direct { semaphore: tokio::sync::Semaphore }, +} + +impl L0FlushGlobalState { + pub fn new(config: L0FlushConfig) -> Self { + match config { + L0FlushConfig::PageCached => Self(Arc::new(Inner::PageCached)), + L0FlushConfig::Direct { max_concurrency } => { + let semaphore = tokio::sync::Semaphore::new(max_concurrency.get()); + Self(Arc::new(Inner::Direct { semaphore })) + } + } + } + + pub(crate) fn inner(&self) -> &Arc { + &self.0 + } +} + +impl L0FlushConfig { + pub(crate) fn prewarm_on_write(&self) -> ephemeral_file::PrewarmPageCacheOnWrite { + use L0FlushConfig::*; + match self { + PageCached => ephemeral_file::PrewarmPageCacheOnWrite::Yes, + Direct { .. } => ephemeral_file::PrewarmPageCacheOnWrite::No, + } + } +} diff --git a/pageserver/src/lib.rs b/pageserver/src/lib.rs index 353f97264c5f..ac6b9b4f2a60 100644 --- a/pageserver/src/lib.rs +++ b/pageserver/src/lib.rs @@ -11,6 +11,7 @@ pub mod deletion_queue; pub mod disk_usage_eviction_task; pub mod http; pub mod import_datadir; +pub mod l0_flush; pub use pageserver_api::keyspace; pub mod aux_file; pub mod metrics; diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 116481a1ebbb..89bf89471cef 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -73,6 +73,7 @@ use crate::deletion_queue::DeletionQueueClient; use crate::deletion_queue::DeletionQueueError; use crate::import_datadir; use crate::is_uninit_mark; +use crate::l0_flush::L0FlushGlobalState; use crate::metrics::TENANT; use crate::metrics::{ remove_tenant_metrics, BROKEN_TENANTS_SET, TENANT_STATE_METRIC, TENANT_SYNTHETIC_SIZE_METRIC, @@ -166,6 +167,7 @@ pub struct TenantSharedResources { pub broker_client: storage_broker::BrokerClientChannel, pub remote_storage: GenericRemoteStorage, pub deletion_queue_client: DeletionQueueClient, + pub l0_flush_global_state: L0FlushGlobalState, } /// A [`Tenant`] is really an _attached_ tenant. The configuration @@ -294,6 +296,8 @@ pub struct Tenant { /// An ongoing timeline detach must be checked during attempts to GC or compact a timeline. ongoing_timeline_detach: std::sync::Mutex>, + + l0_flush_global_state: L0FlushGlobalState, } impl std::fmt::Debug for Tenant { @@ -676,6 +680,7 @@ impl Tenant { broker_client, remote_storage, deletion_queue_client, + l0_flush_global_state, } = resources; let attach_mode = attached_conf.location.attach_mode; @@ -690,6 +695,7 @@ impl Tenant { tenant_shard_id, remote_storage.clone(), deletion_queue_client, + l0_flush_global_state, )); // The attach task will carry a GateGuard, so that shutdown() reliably waits for it to drop out if @@ -989,6 +995,7 @@ impl Tenant { TimelineResources { remote_client, timeline_get_throttle: self.timeline_get_throttle.clone(), + l0_flush_global_state: self.l0_flush_global_state.clone(), }, ctx, ) @@ -2478,6 +2485,7 @@ impl Tenant { tenant_shard_id: TenantShardId, remote_storage: GenericRemoteStorage, deletion_queue_client: DeletionQueueClient, + l0_flush_global_state: L0FlushGlobalState, ) -> Tenant { debug_assert!( !attached_conf.location.generation.is_none() || conf.control_plane_api.is_none() @@ -2565,6 +2573,7 @@ impl Tenant { )), tenant_conf: Arc::new(ArcSwap::from_pointee(attached_conf)), ongoing_timeline_detach: std::sync::Mutex::default(), + l0_flush_global_state, } } @@ -3302,6 +3311,7 @@ impl Tenant { TimelineResources { remote_client, timeline_get_throttle: self.timeline_get_throttle.clone(), + l0_flush_global_state: self.l0_flush_global_state.clone(), } } @@ -3638,6 +3648,7 @@ pub(crate) mod harness { use utils::logging; use crate::deletion_queue::mock::MockDeletionQueue; + use crate::l0_flush::L0FlushConfig; use crate::walredo::apply_neon; use crate::{repository::Key, walrecord::NeonWalRecord}; @@ -3827,6 +3838,8 @@ pub(crate) mod harness { self.tenant_shard_id, self.remote_storage.clone(), self.deletion_queue.new_client(), + // TODO: ideally we should run all unit tests with both configs + L0FlushGlobalState::new(L0FlushConfig::default()), )); let preload = tenant diff --git a/pageserver/src/tenant/block_io.rs b/pageserver/src/tenant/block_io.rs index b406d5033243..85f3b1c79942 100644 --- a/pageserver/src/tenant/block_io.rs +++ b/pageserver/src/tenant/block_io.rs @@ -37,6 +37,7 @@ where pub enum BlockLease<'a> { PageReadGuard(PageReadGuard<'static>), EphemeralFileMutableTail(&'a [u8; PAGE_SZ]), + Slice(&'a [u8; PAGE_SZ]), #[cfg(test)] Arc(std::sync::Arc<[u8; PAGE_SZ]>), #[cfg(test)] @@ -63,6 +64,7 @@ impl<'a> Deref for BlockLease<'a> { match self { BlockLease::PageReadGuard(v) => v.deref(), BlockLease::EphemeralFileMutableTail(v) => v, + BlockLease::Slice(v) => v, #[cfg(test)] BlockLease::Arc(v) => v.deref(), #[cfg(test)] @@ -81,6 +83,7 @@ pub(crate) enum BlockReaderRef<'a> { FileBlockReader(&'a FileBlockReader<'a>), EphemeralFile(&'a EphemeralFile), Adapter(Adapter<&'a DeltaLayerInner>), + Slice(&'a [u8]), #[cfg(test)] TestDisk(&'a super::disk_btree::tests::TestDisk), #[cfg(test)] @@ -99,6 +102,7 @@ impl<'a> BlockReaderRef<'a> { FileBlockReader(r) => r.read_blk(blknum, ctx).await, EphemeralFile(r) => r.read_blk(blknum, ctx).await, Adapter(r) => r.read_blk(blknum, ctx).await, + Slice(s) => Self::read_blk_slice(s, blknum), #[cfg(test)] TestDisk(r) => r.read_blk(blknum), #[cfg(test)] @@ -107,6 +111,24 @@ impl<'a> BlockReaderRef<'a> { } } +impl<'a> BlockReaderRef<'a> { + fn read_blk_slice(slice: &[u8], blknum: u32) -> std::io::Result { + let start = (blknum as usize).checked_mul(PAGE_SZ).unwrap(); + let end = start.checked_add(PAGE_SZ).unwrap(); + if end > slice.len() { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + format!("slice too short, len={} end={}", slice.len(), end), + )); + } + let slice = &slice[start..end]; + let page_sized: &[u8; PAGE_SZ] = slice + .try_into() + .expect("we add PAGE_SZ to start, so the slice must have PAGE_SZ"); + Ok(BlockLease::Slice(page_sized)) + } +} + /// /// A "cursor" for efficiently reading multiple pages from a BlockReader /// diff --git a/pageserver/src/tenant/ephemeral_file.rs b/pageserver/src/tenant/ephemeral_file.rs index 79cc7bf15373..bb65ae24fc5e 100644 --- a/pageserver/src/tenant/ephemeral_file.rs +++ b/pageserver/src/tenant/ephemeral_file.rs @@ -21,6 +21,7 @@ pub struct EphemeralFile { } mod page_caching; +pub(crate) use page_caching::PrewarmOnWrite as PrewarmPageCacheOnWrite; mod zero_padded_read_write; impl EphemeralFile { @@ -53,7 +54,7 @@ impl EphemeralFile { Ok(EphemeralFile { _tenant_shard_id: tenant_shard_id, _timeline_id: timeline_id, - rw: page_caching::RW::new(file), + rw: page_caching::RW::new(file, conf.l0_flush.prewarm_on_write()), }) } @@ -65,6 +66,11 @@ impl EphemeralFile { self.rw.page_cache_file_id() } + /// See [`self::page_caching::RW::load_to_vec`]. + pub(crate) async fn load_to_vec(&self, ctx: &RequestContext) -> Result, io::Error> { + self.rw.load_to_vec(ctx).await + } + pub(crate) async fn read_blk( &self, blknum: u32, diff --git a/pageserver/src/tenant/ephemeral_file/page_caching.rs b/pageserver/src/tenant/ephemeral_file/page_caching.rs index 276ac8706493..43b9fff28d98 100644 --- a/pageserver/src/tenant/ephemeral_file/page_caching.rs +++ b/pageserver/src/tenant/ephemeral_file/page_caching.rs @@ -8,6 +8,7 @@ use crate::virtual_file::VirtualFile; use once_cell::sync::Lazy; use std::io::{self, ErrorKind}; +use std::ops::{Deref, Range}; use tokio_epoll_uring::BoundedBuf; use tracing::*; @@ -19,14 +20,23 @@ pub struct RW { rw: super::zero_padded_read_write::RW, } +/// When we flush a block to the underlying [`crate::virtual_file::VirtualFile`], +/// should we pre-warm the [`crate::page_cache`] with the contents? +#[derive(Clone, Copy)] +pub enum PrewarmOnWrite { + Yes, + No, +} + impl RW { - pub fn new(file: VirtualFile) -> Self { + pub fn new(file: VirtualFile, prewarm_on_write: PrewarmOnWrite) -> Self { let page_cache_file_id = page_cache::next_file_id(); Self { page_cache_file_id, rw: super::zero_padded_read_write::RW::new(PreWarmingWriter::new( page_cache_file_id, file, + prewarm_on_write, )), } } @@ -49,6 +59,43 @@ impl RW { self.rw.bytes_written() } + /// Load all blocks that can be read via [`Self::read_blk`] into a contiguous memory buffer. + /// + /// This includes the blocks that aren't yet flushed to disk by the internal buffered writer. + /// The last block is zero-padded to [`PAGE_SZ`], so, the returned buffer is always a multiple of [`PAGE_SZ`]. + pub(super) async fn load_to_vec(&self, ctx: &RequestContext) -> Result, io::Error> { + // round up to the next PAGE_SZ multiple, required by blob_io + let size = { + let s = usize::try_from(self.bytes_written()).unwrap(); + if s % PAGE_SZ == 0 { + s + } else { + s.checked_add(PAGE_SZ - (s % PAGE_SZ)).unwrap() + } + }; + let vec = Vec::with_capacity(size); + + // read from disk what we've already flushed + let writer = self.rw.as_writer(); + let flushed_range = writer.written_range(); + let mut vec = writer + .file + .read_exact_at( + vec.slice(0..(flushed_range.end - flushed_range.start)), + u64::try_from(flushed_range.start).unwrap(), + ctx, + ) + .await? + .into_inner(); + + // copy from in-memory buffer what we haven't flushed yet but would return when accessed via read_blk + let buffered = self.rw.get_tail_zero_padded(); + vec.extend_from_slice(buffered); + assert_eq!(vec.len(), size); + assert_eq!(vec.len() % PAGE_SZ, 0); + Ok(vec) + } + pub(crate) async fn read_blk( &self, blknum: u32, @@ -116,19 +163,40 @@ impl Drop for RW { } struct PreWarmingWriter { + prewarm_on_write: PrewarmOnWrite, nwritten_blocks: u32, page_cache_file_id: page_cache::FileId, file: VirtualFile, } impl PreWarmingWriter { - fn new(page_cache_file_id: page_cache::FileId, file: VirtualFile) -> Self { + fn new( + page_cache_file_id: page_cache::FileId, + file: VirtualFile, + prewarm_on_write: PrewarmOnWrite, + ) -> Self { Self { + prewarm_on_write, nwritten_blocks: 0, page_cache_file_id, file, } } + + /// Return the byte range within `file` that has been written though `write_all`. + /// + /// The returned range would be invalidated by another `write_all`. To prevent that, we capture `&_`. + fn written_range(&self) -> (impl Deref> + '_) { + let nwritten_blocks = usize::try_from(self.nwritten_blocks).unwrap(); + struct Wrapper(Range); + impl Deref for Wrapper { + type Target = Range; + fn deref(&self) -> &Range { + &self.0 + } + } + Wrapper(0..nwritten_blocks * PAGE_SZ) + } } impl crate::virtual_file::owned_buffers_io::write::OwnedAsyncWriter for PreWarmingWriter { @@ -178,45 +246,51 @@ impl crate::virtual_file::owned_buffers_io::write::OwnedAsyncWriter for PreWarmi assert_eq!(&check_bounds_stuff_works, &*buf); } - // Pre-warm page cache with the contents. - // At least in isolated bulk ingest benchmarks (test_bulk_insert.py), the pre-warming - // benefits the code that writes InMemoryLayer=>L0 layers. let nblocks = buflen / PAGE_SZ; let nblocks32 = u32::try_from(nblocks).unwrap(); - let cache = page_cache::get(); - static CTX: Lazy = Lazy::new(|| { - RequestContext::new( - crate::task_mgr::TaskKind::EphemeralFilePreWarmPageCache, - crate::context::DownloadBehavior::Error, - ) - }); - for blknum_in_buffer in 0..nblocks { - let blk_in_buffer = &buf[blknum_in_buffer * PAGE_SZ..(blknum_in_buffer + 1) * PAGE_SZ]; - let blknum = self - .nwritten_blocks - .checked_add(blknum_in_buffer as u32) - .unwrap(); - match cache - .read_immutable_buf(self.page_cache_file_id, blknum, &CTX) - .await - { - Err(e) => { - error!("ephemeral_file write_blob failed to get immutable buf to pre-warm page cache: {e:?}"); - // fail gracefully, it's not the end of the world if we can't pre-warm the cache here - } - Ok(v) => match v { - page_cache::ReadBufResult::Found(_guard) => { - // This function takes &mut self, so, it shouldn't be possible to reach this point. - unreachable!("we just wrote block {blknum} to the VirtualFile, which is owned by Self, \ - and this function takes &mut self, so, no concurrent read_blk is possible"); - } - page_cache::ReadBufResult::NotFound(mut write_guard) => { - write_guard.copy_from_slice(blk_in_buffer); - let _ = write_guard.mark_valid(); + + if matches!(self.prewarm_on_write, PrewarmOnWrite::Yes) { + // Pre-warm page cache with the contents. + // At least in isolated bulk ingest benchmarks (test_bulk_insert.py), the pre-warming + // benefits the code that writes InMemoryLayer=>L0 layers. + + let cache = page_cache::get(); + static CTX: Lazy = Lazy::new(|| { + RequestContext::new( + crate::task_mgr::TaskKind::EphemeralFilePreWarmPageCache, + crate::context::DownloadBehavior::Error, + ) + }); + for blknum_in_buffer in 0..nblocks { + let blk_in_buffer = + &buf[blknum_in_buffer * PAGE_SZ..(blknum_in_buffer + 1) * PAGE_SZ]; + let blknum = self + .nwritten_blocks + .checked_add(blknum_in_buffer as u32) + .unwrap(); + match cache + .read_immutable_buf(self.page_cache_file_id, blknum, &CTX) + .await + { + Err(e) => { + error!("ephemeral_file write_blob failed to get immutable buf to pre-warm page cache: {e:?}"); + // fail gracefully, it's not the end of the world if we can't pre-warm the cache here } - }, + Ok(v) => match v { + page_cache::ReadBufResult::Found(_guard) => { + // This function takes &mut self, so, it shouldn't be possible to reach this point. + unreachable!("we just wrote block {blknum} to the VirtualFile, which is owned by Self, \ + and this function takes &mut self, so, no concurrent read_blk is possible"); + } + page_cache::ReadBufResult::NotFound(mut write_guard) => { + write_guard.copy_from_slice(blk_in_buffer); + let _ = write_guard.mark_valid(); + } + }, + } } } + self.nwritten_blocks = self.nwritten_blocks.checked_add(nblocks32).unwrap(); Ok((buflen, buf.into_inner())) } diff --git a/pageserver/src/tenant/ephemeral_file/zero_padded_read_write.rs b/pageserver/src/tenant/ephemeral_file/zero_padded_read_write.rs index b37eafb52c5b..fe310acab888 100644 --- a/pageserver/src/tenant/ephemeral_file/zero_padded_read_write.rs +++ b/pageserver/src/tenant/ephemeral_file/zero_padded_read_write.rs @@ -75,6 +75,21 @@ where flushed_offset + u64::try_from(buffer.pending()).unwrap() } + /// Get a slice of all blocks that [`Self::read_blk`] would return as [`ReadResult::ServedFromZeroPaddedMutableTail`]. + pub fn get_tail_zero_padded(&self) -> &[u8] { + let buffer: &zero_padded::Buffer = self.buffered_writer.inspect_buffer(); + let buffer_written_up_to = buffer.pending(); + // pad to next page boundary + let read_up_to = if buffer_written_up_to % PAGE_SZ == 0 { + buffer_written_up_to + } else { + buffer_written_up_to + .checked_add(PAGE_SZ - (buffer_written_up_to % PAGE_SZ)) + .unwrap() + }; + &buffer.as_zero_padded_slice()[0..read_up_to] + } + pub(crate) async fn read_blk(&self, blknum: u32) -> Result, std::io::Error> { let flushed_offset = self.buffered_writer.as_inner().bytes_written(); let buffer: &zero_padded::Buffer = self.buffered_writer.inspect_buffer(); diff --git a/pageserver/src/tenant/storage_layer/inmemory_layer.rs b/pageserver/src/tenant/storage_layer/inmemory_layer.rs index 6624fb7e6ba5..e1eaea90af57 100644 --- a/pageserver/src/tenant/storage_layer/inmemory_layer.rs +++ b/pageserver/src/tenant/storage_layer/inmemory_layer.rs @@ -6,13 +6,14 @@ //! use crate::config::PageServerConf; use crate::context::{PageContentKind, RequestContext, RequestContextBuilder}; +use crate::page_cache::PAGE_SZ; use crate::repository::{Key, Value}; -use crate::tenant::block_io::BlockReader; +use crate::tenant::block_io::{BlockCursor, BlockReader, BlockReaderRef}; use crate::tenant::ephemeral_file::EphemeralFile; use crate::tenant::storage_layer::ValueReconstructResult; use crate::tenant::timeline::GetVectoredError; use crate::tenant::{PageReconstructError, Timeline}; -use crate::{page_cache, walrecord}; +use crate::{l0_flush, page_cache, walrecord}; use anyhow::{anyhow, ensure, Result}; use pageserver_api::keyspace::KeySpace; use pageserver_api::models::InMemoryLayerInfo; @@ -410,6 +411,7 @@ impl InMemoryLayer { continue; } + // TODO: this uses the page cache => https://github.com/neondatabase/neon/issues/8183 let buf = reader.read_blob(block_read.block_offset, &ctx).await; if let Err(e) = buf { reconstruct_state @@ -620,6 +622,13 @@ impl InMemoryLayer { // rare though, so we just accept the potential latency hit for now. let inner = self.inner.read().await; + let l0_flush_global_state = timeline.l0_flush_global_state.inner().clone(); + use l0_flush::Inner; + let _concurrency_permit = match &*l0_flush_global_state { + Inner::PageCached => None, + Inner::Direct { semaphore, .. } => Some(semaphore.acquire().await), + }; + let end_lsn = *self.end_lsn.get().unwrap(); let key_count = if let Some(key_range) = key_range { @@ -645,28 +654,77 @@ impl InMemoryLayer { ) .await?; - let mut buf = Vec::new(); - - let cursor = inner.file.block_cursor(); + match &*l0_flush_global_state { + l0_flush::Inner::PageCached => { + let ctx = RequestContextBuilder::extend(ctx) + .page_content_kind(PageContentKind::InMemoryLayer) + .build(); + + let mut buf = Vec::new(); + + let cursor = inner.file.block_cursor(); + + for (key, vec_map) in inner.index.iter() { + // Write all page versions + for (lsn, pos) in vec_map.as_slice() { + cursor.read_blob_into_buf(*pos, &mut buf, &ctx).await?; + let will_init = Value::des(&buf)?.will_init(); + let res; + (buf, res) = delta_layer_writer + .put_value_bytes(*key, *lsn, buf, will_init, &ctx) + .await; + res?; + } + } + } + l0_flush::Inner::Direct { .. } => { + let file_contents: Vec = inner.file.load_to_vec(ctx).await?; + assert_eq!( + file_contents.len() % PAGE_SZ, + 0, + "needed by BlockReaderRef::Slice" + ); + assert_eq!(file_contents.len(), { + let written = usize::try_from(inner.file.len()).unwrap(); + if written % PAGE_SZ == 0 { + written + } else { + written.checked_add(PAGE_SZ - (written % PAGE_SZ)).unwrap() + } + }); + + let cursor = BlockCursor::new(BlockReaderRef::Slice(&file_contents)); + + let mut buf = Vec::new(); + + for (key, vec_map) in inner.index.iter() { + // Write all page versions + for (lsn, pos) in vec_map.as_slice() { + // TODO: once we have blob lengths in the in-memory index, we can + // 1. get rid of the blob_io / BlockReaderRef::Slice business and + // 2. load the file contents into a Bytes and + // 3. the use `Bytes::slice` to get the `buf` that is our blob + // 4. pass that `buf` into `put_value_bytes` + // => https://github.com/neondatabase/neon/issues/8183 + cursor.read_blob_into_buf(*pos, &mut buf, ctx).await?; + let will_init = Value::des(&buf)?.will_init(); + let res; + (buf, res) = delta_layer_writer + .put_value_bytes(*key, *lsn, buf, will_init, ctx) + .await; + res?; + } + } - let ctx = RequestContextBuilder::extend(ctx) - .page_content_kind(PageContentKind::InMemoryLayer) - .build(); - for (key, vec_map) in inner.index.iter() { - // Write all page versions - for (lsn, pos) in vec_map.as_slice() { - cursor.read_blob_into_buf(*pos, &mut buf, &ctx).await?; - let will_init = Value::des(&buf)?.will_init(); - let res; - (buf, res) = delta_layer_writer - .put_value_bytes(*key, *lsn, buf, will_init, &ctx) - .await; - res?; + // Hold the permit until the IO is done; if we didn't, one could drop this future, + // thereby releasing the permit, but the Vec remains allocated until the IO completes. + // => we'd have more concurrenct Vec than allowed as per the semaphore. + drop(_concurrency_permit); } } // MAX is used here because we identify L0 layers by full key range - let delta_layer = delta_layer_writer.finish(Key::MAX, timeline, &ctx).await?; + let delta_layer = delta_layer_writer.finish(Key::MAX, timeline, ctx).await?; Ok(Some(delta_layer)) } } diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index ec94ed3a56db..de9361d72103 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -65,7 +65,6 @@ use std::{ ops::{Deref, Range}, }; -use crate::metrics::GetKind; use crate::pgdatadir_mapping::MAX_AUX_FILE_V2_DELTAS; use crate::{ aux_file::AuxFileSizeEstimator, @@ -90,6 +89,10 @@ use crate::{ use crate::{ disk_usage_eviction_task::EvictionCandidate, tenant::storage_layer::delta_layer::DeltaEntry, }; +use crate::{ + l0_flush::{self, L0FlushGlobalState}, + metrics::GetKind, +}; use crate::{ metrics::ScanLatencyOngoingRecording, tenant::timeline::logical_size::CurrentLogicalSize, }; @@ -208,6 +211,7 @@ pub struct TimelineResources { pub timeline_get_throttle: Arc< crate::tenant::throttle::Throttle<&'static crate::metrics::tenant_throttling::TimelineGet>, >, + pub l0_flush_global_state: l0_flush::L0FlushGlobalState, } pub(crate) struct AuxFilesState { @@ -433,6 +437,8 @@ pub struct Timeline { /// in the future, add `extra_test_sparse_keyspace` if necessary. #[cfg(test)] pub(crate) extra_test_dense_keyspace: ArcSwap, + + pub(crate) l0_flush_global_state: L0FlushGlobalState, } pub struct WalReceiverInfo { @@ -2392,6 +2398,8 @@ impl Timeline { #[cfg(test)] extra_test_dense_keyspace: ArcSwap::new(Arc::new(KeySpace::default())), + + l0_flush_global_state: resources.l0_flush_global_state, }; result.repartition_threshold = result.get_checkpoint_distance() / REPARTITION_FREQ_IN_CHECKPOINT_DISTANCE; diff --git a/pageserver/src/tenant/timeline/delete.rs b/pageserver/src/tenant/timeline/delete.rs index 6d747d424dde..b0088f4ea228 100644 --- a/pageserver/src/tenant/timeline/delete.rs +++ b/pageserver/src/tenant/timeline/delete.rs @@ -272,6 +272,7 @@ impl DeleteTimelineFlow { TimelineResources { remote_client, timeline_get_throttle: tenant.timeline_get_throttle.clone(), + l0_flush_global_state: tenant.l0_flush_global_state.clone(), }, // Important. We dont pass ancestor above because it can be missing. // Thus we need to skip the validation here. From 6216df776549ab79e45f50c7e1befcc9593960bb Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Tue, 2 Jul 2024 16:21:23 +0100 Subject: [PATCH 017/194] CI(benchmarking): move psql queries to actions/run-python-test-set (#8230) ## Problem Some of the Nightly benchmarks fail with the error ``` + /tmp/neon/pg_install/v14/bin/pgbench --version /tmp/neon/pg_install/v14/bin/pgbench: error while loading shared libraries: libpq.so.5: cannot open shared object file: No such file or directory ``` Originally, we added the `pgbench --version` call to check that `pgbench` is installed and to fail earlier if it's not. The failure happens because we don't have `LD_LIBRARY_PATH` set for every job, and it also affects `psql` command. We can move it to `actions/run-python-test-set` so as not to duplicate code (as it already have `LD_LIBRARY_PATH` set). ## Summary of changes - Remove `pgbench --version` call - Move `psql` commands to common `actions/run-python-test-set` --- .../actions/run-python-test-set/action.yml | 10 ++- .github/workflows/benchmarking.yml | 83 +------------------ 2 files changed, 12 insertions(+), 81 deletions(-) diff --git a/.github/actions/run-python-test-set/action.yml b/.github/actions/run-python-test-set/action.yml index a2aae0772b15..7f843de1a55c 100644 --- a/.github/actions/run-python-test-set/action.yml +++ b/.github/actions/run-python-test-set/action.yml @@ -179,7 +179,15 @@ runs: # Wake up the cluster if we use remote neon instance if [ "${{ inputs.build_type }}" = "remote" ] && [ -n "${BENCHMARK_CONNSTR}" ]; then - ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/psql ${BENCHMARK_CONNSTR} -c "SELECT version();" + QUERIES=("SELECT version()") + if [[ "${PLATFORM}" = "neon"* ]]; then + QUERIES+=("SHOW neon.tenant_id") + QUERIES+=("SHOW neon.timeline_id") + fi + + for q in "${QUERIES[@]}"; do + ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/psql ${BENCHMARK_CONNSTR} -c "${q}" + done fi # Run the tests. diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index 0e748adeb69e..db04b5de7ddc 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -239,11 +239,6 @@ jobs: path: /tmp/neon/ prefix: latest - - name: Add Postgres binaries to PATH - run: | - ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/pgbench --version - echo "${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin" >> $GITHUB_PATH - - name: Create Neon Project if: contains(fromJson('["neon-captest-new", "neon-captest-freetier", "neonvm-captest-new", "neonvm-captest-freetier"]'), matrix.platform) id: create-neon-project @@ -282,16 +277,6 @@ jobs: echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT - QUERIES=("SELECT version()") - if [[ "${PLATFORM}" = "neon"* ]]; then - QUERIES+=("SHOW neon.tenant_id") - QUERIES+=("SHOW neon.timeline_id") - fi - - for q in "${QUERIES[@]}"; do - psql ${CONNSTR} -c "${q}" - done - - name: Benchmark init uses: ./.github/actions/run-python-test-set with: @@ -377,29 +362,12 @@ jobs: path: /tmp/neon/ prefix: latest - - name: Add Postgres binaries to PATH - run: | - LD_LIBRARY_PATH="${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/lib" - export LD_LIBRARY_PATH - echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV - - ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/pgbench --version - echo "${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin" >> $GITHUB_PATH - - name: Set up Connection String id: set-up-connstr run: | CONNSTR=${{ secrets.BENCHMARK_PGVECTOR_CONNSTR }} - - echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT - QUERIES=("SELECT version()") - QUERIES+=("SHOW neon.tenant_id") - QUERIES+=("SHOW neon.timeline_id") - - for q in "${QUERIES[@]}"; do - psql ${CONNSTR} -c "${q}" - done + echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT - name: Benchmark pgvector hnsw indexing uses: ./.github/actions/run-python-test-set @@ -421,12 +389,12 @@ jobs: test_selection: performance/test_perf_pgvector_queries.py run_in_parallel: false save_perf_report: ${{ env.SAVE_PERF_REPORT }} - extra_params: -m remote_cluster --timeout 21600 + extra_params: -m remote_cluster --timeout 21600 env: BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }} VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" - + - name: Create Allure report if: ${{ !cancelled() }} uses: ./.github/actions/allure-report-generate @@ -481,11 +449,6 @@ jobs: path: /tmp/neon/ prefix: latest - - name: Add Postgres binaries to PATH - run: | - ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/pgbench --version - echo "${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin" >> $GITHUB_PATH - - name: Set up Connection String id: set-up-connstr run: | @@ -507,16 +470,6 @@ jobs: echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT - QUERIES=("SELECT version()") - if [[ "${PLATFORM}" = "neon"* ]]; then - QUERIES+=("SHOW neon.tenant_id") - QUERIES+=("SHOW neon.timeline_id") - fi - - for q in "${QUERIES[@]}"; do - psql ${CONNSTR} -c "${q}" - done - - name: ClickBench benchmark uses: ./.github/actions/run-python-test-set with: @@ -584,11 +537,6 @@ jobs: path: /tmp/neon/ prefix: latest - - name: Add Postgres binaries to PATH - run: | - ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/pgbench --version - echo "${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin" >> $GITHUB_PATH - - name: Get Connstring Secret Name run: | case "${PLATFORM}" in @@ -617,16 +565,6 @@ jobs: echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT - QUERIES=("SELECT version()") - if [[ "${PLATFORM}" = "neon"* ]]; then - QUERIES+=("SHOW neon.tenant_id") - QUERIES+=("SHOW neon.timeline_id") - fi - - for q in "${QUERIES[@]}"; do - psql ${CONNSTR} -c "${q}" - done - - name: Run TPC-H benchmark uses: ./.github/actions/run-python-test-set with: @@ -685,11 +623,6 @@ jobs: path: /tmp/neon/ prefix: latest - - name: Add Postgres binaries to PATH - run: | - ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/pgbench --version - echo "${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin" >> $GITHUB_PATH - - name: Set up Connection String id: set-up-connstr run: | @@ -711,16 +644,6 @@ jobs: echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT - QUERIES=("SELECT version()") - if [[ "${PLATFORM}" = "neon"* ]]; then - QUERIES+=("SHOW neon.tenant_id") - QUERIES+=("SHOW neon.timeline_id") - fi - - for q in "${QUERIES[@]}"; do - psql ${CONNSTR} -c "${q}" - done - - name: Run user examples uses: ./.github/actions/run-python-test-set with: From f5832329ac57e4a08c4d3b26b61864c2f1552ddf Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 2 Jul 2024 17:17:22 +0100 Subject: [PATCH 018/194] tense of errors (#8234) I forgot a commit when merging https://github.com/neondatabase/neon/pull/8177 --- pageserver/src/tenant/mgr.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index c1da1d2c55fd..b0159e22bfc0 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -358,7 +358,7 @@ fn load_tenant_config( info!("Found temporary tenant directory, removing: {tenant_dir_path}"); // No need to use safe_remove_tenant_dir_all because this is already // a temporary path - std::fs::remove_dir_all(&tenant_dir_path).fatal_err("Deleting temporary tenant dir"); + std::fs::remove_dir_all(&tenant_dir_path).fatal_err("delete temporary tenant dir"); return None; } @@ -368,7 +368,7 @@ fn load_tenant_config( .fatal_err("Checking for empty tenant dir"); if is_empty { info!("removing empty tenant directory {tenant_dir_path:?}"); - std::fs::remove_dir(&tenant_dir_path).fatal_err("Deleting empty tenant dir"); + std::fs::remove_dir(&tenant_dir_path).fatal_err("delete empty tenant dir"); return None; } @@ -386,7 +386,7 @@ async fn init_load_tenant_configs( let tenants_dir = conf.tenants_path(); let dentries = tokio::task::spawn_blocking(move || -> Vec { - let context = format!("Reading tenants dir {tenants_dir}"); + let context = format!("read tenants dir {tenants_dir}"); let dir_entries = tenants_dir.read_dir_utf8().fatal_err(&context); dir_entries @@ -587,7 +587,7 @@ pub async fn init_tenant_mgr( // For those shards that have live configurations, construct `Tenant` or `SecondaryTenant` objects and start them running for (tenant_shard_id, location_conf, config_write_result) in config_write_results { // Writing a config to local disk is foundational to startup up tenants: panic if we can't. - config_write_result.fatal_err("writing tenant shard config file"); + config_write_result.fatal_err("write tenant shard config file"); let tenant_dir_path = conf.tenant_path(&tenant_shard_id); let shard_identity = location_conf.shard; @@ -953,7 +953,7 @@ impl TenantManager { Some(FastPathModified::Attached(tenant)) => { Tenant::persist_tenant_config(self.conf, &tenant_shard_id, &new_location_config) .await - .fatal_err("writing tenant shard config"); + .fatal_err("write tenant shard config"); // Transition to AttachedStale means we may well hold a valid generation // still, and have been requested to go stale as part of a migration. If @@ -984,7 +984,7 @@ impl TenantManager { Some(FastPathModified::Secondary(_secondary_tenant)) => { Tenant::persist_tenant_config(self.conf, &tenant_shard_id, &new_location_config) .await - .fatal_err("writing tenant shard config"); + .fatal_err("write tenant shard config"); return Ok(None); } @@ -1069,14 +1069,14 @@ impl TenantManager { // Does not need to be fsync'd because local storage is just a cache. tokio::fs::create_dir_all(&timelines_path) .await - .fatal_err("creating timelines/ dir"); + .fatal_err("create timelines/ dir"); // Before activating either secondary or attached mode, persist the // configuration, so that on restart we will re-attach (or re-start // secondary) on the tenant. Tenant::persist_tenant_config(self.conf, &tenant_shard_id, &new_location_config) .await - .fatal_err("writing tenant shard config"); + .fatal_err("write tenant shard config"); let new_slot = match &new_location_config.mode { LocationMode::Secondary(secondary_config) => { From 891cb5a9a8eb90242997f4517a4d06cd635fd931 Mon Sep 17 00:00:00 2001 From: Alex Chi Z Date: Tue, 2 Jul 2024 12:54:32 -0400 Subject: [PATCH 019/194] fix(pageserver): comments about metadata key range (#8236) Signed-off-by: Alex Chi Z --- libs/pageserver_api/src/key.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/pageserver_api/src/key.rs b/libs/pageserver_api/src/key.rs index cd430bfab7d4..0acd83753eff 100644 --- a/libs/pageserver_api/src/key.rs +++ b/libs/pageserver_api/src/key.rs @@ -29,7 +29,7 @@ pub const KEY_SIZE: usize = 18; /// See [`Key::to_i128`] for more information on the encoding. pub const METADATA_KEY_SIZE: usize = 16; -/// The key prefix start range for the metadata keys. All keys with the first byte >= 0x40 is a metadata key. +/// The key prefix start range for the metadata keys. All keys with the first byte >= 0x60 is a metadata key. pub const METADATA_KEY_BEGIN_PREFIX: u8 = 0x60; pub const METADATA_KEY_END_PREFIX: u8 = 0x7F; From 4a0c2aebe03dc388aeefc4cbd62006ae3eb8fc60 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 2 Jul 2024 21:45:42 +0300 Subject: [PATCH 020/194] Add test for proper handling of connection failure to avoid 'cannot wait on socket event without a socket' error (#8231) ## Problem See https://github.com/neondatabase/cloud/issues/14289 and PR #8210 ## Summary of changes Add test for problems fixed in #8210 ## Checklist before requesting a review - [ ] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. ## Checklist before merging - [ ] Do not forget to reformat commit message to not include the above checklist --------- Co-authored-by: Konstantin Knizhnik --- pgxn/neon/libpagestore.c | 5 ---- .../regress/test_pageserver_reconnect.py | 24 +++++++++++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/pgxn/neon/libpagestore.c b/pgxn/neon/libpagestore.c index a3fdcc537ead..73a001b6ba72 100644 --- a/pgxn/neon/libpagestore.c +++ b/pgxn/neon/libpagestore.c @@ -427,11 +427,6 @@ pageserver_connect(shardno_t shard_no, int elevel) values[n_pgsql_params] = NULL; shard->conn = PQconnectStartParams(keywords, values, 1); - if (!shard->conn) - { - neon_shard_log(shard_no, elevel, "Failed to connect to pageserver: out of memory"); - return false; - } if (PQstatus(shard->conn) == CONNECTION_BAD) { char *msg = pchomp(PQerrorMessage(shard->conn)); diff --git a/test_runner/regress/test_pageserver_reconnect.py b/test_runner/regress/test_pageserver_reconnect.py index aecfcdd262e5..37ff923632d2 100644 --- a/test_runner/regress/test_pageserver_reconnect.py +++ b/test_runner/regress/test_pageserver_reconnect.py @@ -2,6 +2,7 @@ import time from contextlib import closing +import psycopg2.errors from fixtures.log_helper import log from fixtures.neon_fixtures import NeonEnv, PgBin @@ -40,3 +41,26 @@ def run_pgbench(connstr: str): c.execute("select pg_reload_conf()") thread.join() + + +# Test handling errors during page server reconnect +def test_pageserver_reconnect_failure(neon_simple_env: NeonEnv): + env = neon_simple_env + env.neon_cli.create_branch("test_pageserver_reconnect") + endpoint = env.endpoints.create_start("test_pageserver_reconnect") + + con = endpoint.connect() + cur = con.cursor() + + cur.execute("set statement_timeout='2s'") + cur.execute("SELECT setting FROM pg_settings WHERE name='neon.pageserver_connstring'") + connstring = cur.fetchall()[0][0] + cur.execute( + f"alter system set neon.pageserver_connstring='{connstring}?some_invalid_param=xyz'" + ) + cur.execute("select pg_reload_conf()") + try: + cur.execute("select count(*) from pg_class") + except psycopg2.errors.QueryCanceled: + log.info("Connection to PS failed") + assert not endpoint.log_contains("ERROR: cannot wait on socket event without a socket.*") From 4273309962df6b8921c0f50de2d9dc4226a28636 Mon Sep 17 00:00:00 2001 From: Alex Chi Z Date: Wed, 3 Jul 2024 04:48:56 -0400 Subject: [PATCH 021/194] docker: add storage_scrubber into the docker image (#8239) ## Problem We will run this tool in the k8s cluster. To make it accessible from k8s, we need to package it into the docker image. part of https://github.com/neondatabase/cloud/issues/14024 --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index f0197758e48b..a41598ef72cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,6 +57,7 @@ RUN set -e \ --bin storage_controller \ --bin proxy \ --bin neon_local \ + --bin storage_scrubber \ --locked --release \ && cachepot -s @@ -83,6 +84,7 @@ COPY --from=build --chown=neon:neon /home/nonroot/target/release/storage_broker COPY --from=build --chown=neon:neon /home/nonroot/target/release/storage_controller /usr/local/bin COPY --from=build --chown=neon:neon /home/nonroot/target/release/proxy /usr/local/bin COPY --from=build --chown=neon:neon /home/nonroot/target/release/neon_local /usr/local/bin +COPY --from=build --chown=neon:neon /home/nonroot/target/release/storage_scrubber /usr/local/bin COPY --from=pg-build /home/nonroot/pg_install/v14 /usr/local/v14/ COPY --from=pg-build /home/nonroot/pg_install/v15 /usr/local/v15/ From dae55badf343627599f7dcb94086a981d54f082c Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 3 Jul 2024 13:22:53 +0300 Subject: [PATCH 022/194] Simplify test_wal_page_boundary_start test (#8214) All the code to ensure the WAL record lands at a page boundary was unnecessary for reproducing the original problem. In fact, it's a pretty basic test that checks that outbound replication (= neon as publisher) still works after restarting the endpoint. It just used to be very broken before commit 5ceccdc7de, which also added this test. To verify that: 1. Check out commit f3af5f4660 (because the next commit, 7dd58e1449, fixed the same bug in a different way, making it infeasible to revert the bug fix in an easy way) 2. Revert the bug fix from commit 5ceccdc7de with this: ``` diff --git a/pgxn/neon/walproposer_pg.c b/pgxn/neon/walproposer_pg.c index 7debb6325..9f03bbd99 100644 --- a/pgxn/neon/walproposer_pg.c +++ b/pgxn/neon/walproposer_pg.c @@ -1437,8 +1437,10 @@ XLogWalPropWrite(WalProposer *wp, char *buf, Size nbytes, XLogRecPtr recptr) * * https://github.com/neondatabase/neon/issues/5749 */ +#if 0 if (!wp->config->syncSafekeepers) XLogUpdateWalBuffers(buf, recptr, nbytes); +#endif while (nbytes > 0) { ``` 3. Run the test_wal_page_boundary_start regression test. It fails, as expected 4. Apply this commit to the test, and run it again. It still fails, with the same error mentioned in issue #5749: ``` PG:2024-06-30 20:49:08.805 GMT [1248196] STATEMENT: START_REPLICATION SLOT "sub1" LOGICAL 0/0 (proto_version '4', origin 'any', publication_names '"pub1"') PG:2024-06-30 21:37:52.567 GMT [1467972] LOG: starting logical decoding for slot "sub1" PG:2024-06-30 21:37:52.567 GMT [1467972] DETAIL: Streaming transactions committing after 0/1532330, reading WAL from 0/1531C78. PG:2024-06-30 21:37:52.567 GMT [1467972] STATEMENT: START_REPLICATION SLOT "sub1" LOGICAL 0/0 (proto_version '4', origin 'any', publication_names '"pub1"') PG:2024-06-30 21:37:52.567 GMT [1467972] LOG: logical decoding found consistent point at 0/1531C78 PG:2024-06-30 21:37:52.567 GMT [1467972] DETAIL: There are no running transactions. PG:2024-06-30 21:37:52.567 GMT [1467972] STATEMENT: START_REPLICATION SLOT "sub1" LOGICAL 0/0 (proto_version '4', origin 'any', publication_names '"pub1"') PG:2024-06-30 21:37:52.568 GMT [1467972] ERROR: could not find record while sending logically-decoded data: invalid contrecord length 312 (expected 6) at 0/1533FD8 ``` --- .../regress/test_logical_replication.py | 60 +++---------------- 1 file changed, 9 insertions(+), 51 deletions(-) diff --git a/test_runner/regress/test_logical_replication.py b/test_runner/regress/test_logical_replication.py index ca3c81d6e51d..41283e4d2ca0 100644 --- a/test_runner/regress/test_logical_replication.py +++ b/test_runner/regress/test_logical_replication.py @@ -4,7 +4,6 @@ from string import ascii_lowercase import pytest -from fixtures.common_types import Lsn from fixtures.log_helper import log from fixtures.neon_fixtures import ( AuxFileStore, @@ -13,7 +12,7 @@ logical_replication_sync, wait_for_last_flush_lsn, ) -from fixtures.utils import query_scalar, wait_until +from fixtures.utils import wait_until def random_string(n: int): @@ -326,12 +325,17 @@ def test_lr_with_slow_safekeeper(neon_env_builder: NeonEnvBuilder, vanilla_pg): assert "could not receive data from WAL stream" not in logs -# Test compute start at LSN page of which starts with contrecord -# https://github.com/neondatabase/neon/issues/5749 +# Test replication of WAL record spanning page boundary (with contrecord) after +# compute restart and WAL write of the page. +# +# See https://github.com/neondatabase/neon/issues/5749 +# +# Most pages start with a contrecord, so we don't do anything special +# to ensure that. @pytest.mark.parametrize( "pageserver_aux_file_policy", [AuxFileStore.V1, AuxFileStore.CrossValidation] ) -def test_wal_page_boundary_start(neon_simple_env: NeonEnv, vanilla_pg): +def test_restart_endpoint(neon_simple_env: NeonEnv, vanilla_pg): env = neon_simple_env env.neon_cli.create_branch("init") @@ -356,52 +360,6 @@ def test_wal_page_boundary_start(neon_simple_env: NeonEnv, vanilla_pg): logical_replication_sync(vanilla_pg, endpoint) vanilla_pg.stop() - with endpoint.cursor() as cur: - # measure how much space logical message takes. Sometimes first attempt - # creates huge message and then it stabilizes, have no idea why. - for _ in range(3): - lsn_before = Lsn(query_scalar(cur, "select pg_current_wal_lsn()")) - log.info(f"current_lsn={lsn_before}") - # Non-transactional logical message doesn't write WAL, only XLogInsert's - # it, so use transactional. Which is a bit problematic as transactional - # necessitates commit record. Alternatively we can do smth like - # select neon_xlogflush(pg_current_wal_insert_lsn()); - # but isn't much better + that particular call complains on 'xlog flush - # request 0/282C018 is not satisfied' as pg_current_wal_insert_lsn skips - # page headers. - payload = "blahblah" - cur.execute(f"select pg_logical_emit_message(true, 'pref', '{payload}')") - lsn_after_by_curr_wal_lsn = Lsn(query_scalar(cur, "select pg_current_wal_lsn()")) - lsn_diff = lsn_after_by_curr_wal_lsn - lsn_before - logical_message_base = lsn_after_by_curr_wal_lsn - lsn_before - len(payload) - log.info( - f"before {lsn_before}, after {lsn_after_by_curr_wal_lsn}, lsn diff is {lsn_diff}, base {logical_message_base}" - ) - - # and write logical message spanning exactly as we want - lsn_before = Lsn(query_scalar(cur, "select pg_current_wal_lsn()")) - log.info(f"current_lsn={lsn_before}") - curr_lsn = Lsn(query_scalar(cur, "select pg_current_wal_lsn()")) - offs = int(curr_lsn) % 8192 - till_page = 8192 - offs - payload_len = ( - till_page - logical_message_base - 8 - ) # not sure why 8 is here, it is deduced from experiments - log.info(f"current_lsn={curr_lsn}, offs {offs}, till_page {till_page}") - - # payload_len above would go exactly till the page boundary; but we want contrecord, so make it slightly longer - payload_len += 8 - - cur.execute(f"select pg_logical_emit_message(true, 'pref', 'f{'a' * payload_len}')") - supposedly_contrecord_end = Lsn(query_scalar(cur, "select pg_current_wal_lsn()")) - log.info(f"supposedly_page_boundary={supposedly_contrecord_end}") - # The calculations to hit the page boundary are very fuzzy, so just - # ignore test if we fail to reach it. - if not (int(supposedly_contrecord_end) % 8192 == 32): - pytest.skip("missed page boundary, bad luck") - - cur.execute("insert into replication_example values (2, 3)") - wait_for_last_flush_lsn(env, endpoint, tenant_id, timeline_id) endpoint.stop().start() From aae38763188203ad1937cead961e7809e679ccfd Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Wed, 3 Jul 2024 12:19:13 +0100 Subject: [PATCH 023/194] CI: update docker/* actions to latest versions (#7694) ## Problem GitHub Actions complain that we use actions that depend on deprecated Node 16: ``` Node.js 16 actions are deprecated. Please update the following actions to use Node.js 20: docker/setup-buildx-action@v2 ``` But also, the latest `docker/setup-buildx-action` fails with the following error: ``` /nvme/actions-runner/_work/_actions/docker/setup-buildx-action/v3/webpack:/docker-setup-buildx/node_modules/@actions/cache/lib/cache.js:175 throw new Error(`Path Validation Error: Path(s) specified in the action for caching do(es) not exist, hence no cache is being saved.`); ^ Error: Path Validation Error: Path(s) specified in the action for caching do(es) not exist, hence no cache is being saved. at Object.rejected (/nvme/actions-runner/_work/_actions/docker/setup-buildx-action/v3/webpack:/docker-setup-buildx/node_modules/@actions/cache/lib/cache.js:175:1) at Generator.next () at fulfilled (/nvme/actions-runner/_work/_actions/docker/setup-buildx-action/v3/webpack:/docker-setup-buildx/node_modules/@actions/cache/lib/cache.js:29:1) ``` We can work this around by setting `cache-binary: false` for `uses: docker/setup-buildx-action@v3` ## Summary of changes - Update `docker/setup-buildx-action` from `v2` to `v3`, set `cache-binary: false` - Update `docker/login-action` from `v2` to `v3` - Update `docker/build-push-action` from `v4`/`v5` to `v6` --- .github/workflows/build-build-tools-image.yml | 8 +++++--- .github/workflows/build_and_test.yml | 17 ++++++++++------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-build-tools-image.yml b/.github/workflows/build-build-tools-image.yml index f1c39e7e4f5b..a69686bf2a6e 100644 --- a/.github/workflows/build-build-tools-image.yml +++ b/.github/workflows/build-build-tools-image.yml @@ -63,14 +63,16 @@ jobs: mkdir -p /tmp/.docker-custom echo DOCKER_CONFIG=/tmp/.docker-custom >> $GITHUB_ENV - - uses: docker/setup-buildx-action@v2 + - uses: docker/setup-buildx-action@v3 + with: + cache-binary: false - - uses: docker/login-action@v2 + - uses: docker/login-action@v3 with: username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} - - uses: docker/build-push-action@v4 + - uses: docker/build-push-action@v6 with: context: . provenance: false diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 24ad26205b60..5ac8c6ec2744 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -751,14 +751,16 @@ jobs: run: | mkdir -p .docker-custom echo DOCKER_CONFIG=$(pwd)/.docker-custom >> $GITHUB_ENV - - uses: docker/setup-buildx-action@v2 + - uses: docker/setup-buildx-action@v3 + with: + cache-binary: false - uses: docker/login-action@v3 with: username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} - - uses: docker/build-push-action@v5 + - uses: docker/build-push-action@v6 with: context: . build-args: | @@ -829,11 +831,12 @@ jobs: run: | mkdir -p .docker-custom echo DOCKER_CONFIG=$(pwd)/.docker-custom >> $GITHUB_ENV - - uses: docker/setup-buildx-action@v2 + - uses: docker/setup-buildx-action@v3 with: + cache-binary: false # Disable parallelism for docker buildkit. # As we already build everything with `make -j$(nproc)`, running it in additional level of parallelisam blows up the Runner. - config-inline: | + buildkitd-config-inline: | [worker.oci] max-parallelism = 1 @@ -849,7 +852,7 @@ jobs: password: ${{ secrets.AWS_SECRET_KEY_DEV }} - name: Build compute-node image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . build-args: | @@ -868,7 +871,7 @@ jobs: - name: Build neon extensions test image if: matrix.version == 'v16' - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . build-args: | @@ -889,7 +892,7 @@ jobs: - name: Build compute-tools image # compute-tools are Postgres independent, so build it only once if: matrix.version == 'v16' - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: target: compute-tools-image context: . From 97f7188a07a7992cb058d654a79c91acf4a1b975 Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 3 Jul 2024 14:13:06 +0100 Subject: [PATCH 024/194] pageserver: don't try to flush if shutdown during attach (#8235) ## Problem test_location_conf_churn fails on log errors when it tries to shutdown a pageserver immediately after starting a tenant attach, like this: https://neon-github-public-dev.s3.amazonaws.com/reports/pr-8224/9761000525/index.html#/testresult/15fb6beca5c7327c ``` shutdown:shutdown{tenant_id=35f5c55eb34e7e5e12288c5d8ab8b909 shard_id=0000}:timeline_shutdown{timeline_id=30936747043353a98661735ad09cbbfe shutdown_mode=FreezeAndFlush}: failed to freeze and flush: cannot flush frozen layers when flush_loop is not running, state is Exited\n') ``` This is happening because Tenant::shutdown fires its cancellation token early if the tenant is not fully attached by the time shutdown is called, so the flush loop is shutdown by the time we try and flush. ## Summary of changes - In the early-cancellation case, also set the shutdown mode to Hard to skip trying to do a flush that will fail. --- pageserver/src/tenant.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 89bf89471cef..0c911939e848 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -1816,9 +1816,15 @@ impl Tenant { // If we're still attaching, fire the cancellation token early to drop out: this // will prevent us flushing, but ensures timely shutdown if some I/O during attach // is very slow. - if matches!(self.current_state(), TenantState::Attaching) { + let shutdown_mode = if matches!(self.current_state(), TenantState::Attaching) { self.cancel.cancel(); - } + + // Having fired our cancellation token, do not try and flush timelines: their cancellation tokens + // are children of ours, so their flush loops will have shut down already + timeline::ShutdownMode::Hard + } else { + shutdown_mode + }; match self.set_stopping(shutdown_progress, false, false).await { Ok(()) => {} From e0891ec8c8d07d1e2f91413a56c961103d4ef245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Wed, 3 Jul 2024 18:02:10 +0200 Subject: [PATCH 025/194] Only support compressed reads if the compression setting is present (#8238) PR #8106 was created with the assumption that no blob is larger than `256 MiB`. Due to #7852 we have checking for *writes* of blobs larger than that limit, but we didn't have checking for *reads* of such large blobs: in theory, we could be reading these blobs every day but we just don't happen to write the blobs for some reason. Therefore, we now add a warning for *reads* of such large blobs as well. To make deploying compression less dangerous, we therefore only assume a blob is compressed if the compression setting is present in the config. This also means that we can't back out of compression once we enabled it. Part of https://github.com/neondatabase/neon/issues/5431 --- pageserver/src/tenant/blob_io.rs | 45 +++++++++++-------- pageserver/src/tenant/block_io.rs | 31 +++++++++++-- .../src/tenant/storage_layer/image_layer.rs | 28 ++++++++---- pageserver/src/tenant/storage_layer/layer.rs | 1 + 4 files changed, 75 insertions(+), 30 deletions(-) diff --git a/pageserver/src/tenant/blob_io.rs b/pageserver/src/tenant/blob_io.rs index 022801b17fba..de74066b81bc 100644 --- a/pageserver/src/tenant/blob_io.rs +++ b/pageserver/src/tenant/blob_io.rs @@ -19,6 +19,7 @@ use bytes::{BufMut, BytesMut}; use pageserver_api::models::ImageCompressionAlgorithm; use tokio::io::AsyncWriteExt; use tokio_epoll_uring::{BoundedBuf, IoBuf, Slice}; +use tracing::warn; use crate::context::RequestContext; use crate::page_cache::PAGE_SZ; @@ -72,14 +73,22 @@ impl<'a> BlockCursor<'a> { len_buf.copy_from_slice(&buf[off..off + 4]); off += 4; } - len_buf[0] &= !LEN_COMPRESSION_BIT_MASK; + let bit_mask = if self.read_compressed { + !LEN_COMPRESSION_BIT_MASK + } else { + 0x7f + }; + len_buf[0] &= bit_mask; u32::from_be_bytes(len_buf) as usize }; let compression_bits = first_len_byte & LEN_COMPRESSION_BIT_MASK; let mut tmp_buf = Vec::new(); let buf_to_write; - let compression = if compression_bits <= BYTE_UNCOMPRESSED { + let compression = if compression_bits <= BYTE_UNCOMPRESSED || !self.read_compressed { + if compression_bits > BYTE_UNCOMPRESSED { + warn!("reading key above future limit ({len} bytes)"); + } buf_to_write = dstbuf; None } else if compression_bits == BYTE_ZSTD { @@ -384,10 +393,10 @@ mod tests { use rand::{Rng, SeedableRng}; async fn round_trip_test(blobs: &[Vec]) -> Result<(), Error> { - round_trip_test_compressed::(blobs).await + round_trip_test_compressed::(blobs).await } - async fn round_trip_test_compressed( + async fn round_trip_test_compressed( blobs: &[Vec], ) -> Result<(), Error> { let temp_dir = camino_tempfile::tempdir()?; @@ -400,17 +409,15 @@ mod tests { let file = VirtualFile::create(pathbuf.as_path(), &ctx).await?; let mut wtr = BlobWriter::::new(file, 0); for blob in blobs.iter() { - let (_, res) = match COMPRESSION { - 0 => wtr.write_blob(blob.clone(), &ctx).await, - 1 => { - wtr.write_blob_maybe_compressed( - blob.clone(), - &ctx, - Some(ImageCompressionAlgorithm::Zstd { level: Some(1) }), - ) - .await - } - _ => unreachable!("Invalid compression {COMPRESSION}"), + let (_, res) = if COMPRESSION { + wtr.write_blob_maybe_compressed( + blob.clone(), + &ctx, + Some(ImageCompressionAlgorithm::Zstd { level: Some(1) }), + ) + .await + } else { + wtr.write_blob(blob.clone(), &ctx).await }; let offs = res?; offsets.push(offs); @@ -425,7 +432,7 @@ mod tests { let file = VirtualFile::open(pathbuf.as_path(), &ctx).await?; let rdr = BlockReaderRef::VirtualFile(&file); - let rdr = BlockCursor::new(rdr); + let rdr = BlockCursor::new_with_compression(rdr, COMPRESSION); for (idx, (blob, offset)) in blobs.iter().zip(offsets.iter()).enumerate() { let blob_read = rdr.read_blob(*offset, &ctx).await?; assert_eq!( @@ -459,6 +466,8 @@ mod tests { ]; round_trip_test::(blobs).await?; round_trip_test::(blobs).await?; + round_trip_test_compressed::(blobs).await?; + round_trip_test_compressed::(blobs).await?; Ok(()) } @@ -474,8 +483,8 @@ mod tests { ]; round_trip_test::(blobs).await?; round_trip_test::(blobs).await?; - round_trip_test_compressed::(blobs).await?; - round_trip_test_compressed::(blobs).await?; + round_trip_test_compressed::(blobs).await?; + round_trip_test_compressed::(blobs).await?; Ok(()) } diff --git a/pageserver/src/tenant/block_io.rs b/pageserver/src/tenant/block_io.rs index 85f3b1c79942..3324e840ecf1 100644 --- a/pageserver/src/tenant/block_io.rs +++ b/pageserver/src/tenant/block_io.rs @@ -149,16 +149,24 @@ impl<'a> BlockReaderRef<'a> { /// ``` /// pub struct BlockCursor<'a> { + pub(super) read_compressed: bool, reader: BlockReaderRef<'a>, } impl<'a> BlockCursor<'a> { pub(crate) fn new(reader: BlockReaderRef<'a>) -> Self { - BlockCursor { reader } + Self::new_with_compression(reader, false) + } + pub(crate) fn new_with_compression(reader: BlockReaderRef<'a>, read_compressed: bool) -> Self { + BlockCursor { + read_compressed, + reader, + } } // Needed by cli pub fn new_fileblockreader(reader: &'a FileBlockReader) -> Self { BlockCursor { + read_compressed: false, reader: BlockReaderRef::FileBlockReader(reader), } } @@ -188,11 +196,25 @@ pub struct FileBlockReader<'a> { /// Unique ID of this file, used as key in the page cache. file_id: page_cache::FileId, + + compressed_reads: bool, } impl<'a> FileBlockReader<'a> { pub fn new(file: &'a VirtualFile, file_id: FileId) -> Self { - FileBlockReader { file_id, file } + Self::new_with_compression(file, file_id, false) + } + + pub fn new_with_compression( + file: &'a VirtualFile, + file_id: FileId, + compressed_reads: bool, + ) -> Self { + FileBlockReader { + file_id, + file, + compressed_reads, + } } /// Read a page from the underlying file into given buffer. @@ -239,7 +261,10 @@ impl<'a> FileBlockReader<'a> { impl BlockReader for FileBlockReader<'_> { fn block_cursor(&self) -> BlockCursor<'_> { - BlockCursor::new(BlockReaderRef::FileBlockReader(self)) + BlockCursor::new_with_compression( + BlockReaderRef::FileBlockReader(self), + self.compressed_reads, + ) } } diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index 50aacbd9ad46..4a1b3a02377a 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -165,6 +165,7 @@ pub struct ImageLayerInner { file_id: FileId, max_vectored_read_bytes: Option, + compressed_reads: bool, } impl std::fmt::Debug for ImageLayerInner { @@ -178,7 +179,8 @@ impl std::fmt::Debug for ImageLayerInner { impl ImageLayerInner { pub(super) async fn dump(&self, ctx: &RequestContext) -> anyhow::Result<()> { - let block_reader = FileBlockReader::new(&self.file, self.file_id); + let block_reader = + FileBlockReader::new_with_compression(&self.file, self.file_id, self.compressed_reads); let tree_reader = DiskBtreeReader::<_, KEY_SIZE>::new( self.index_start_blk, self.index_root_blk, @@ -266,9 +268,10 @@ impl ImageLayer { async fn load_inner(&self, ctx: &RequestContext) -> Result { let path = self.path(); - let loaded = ImageLayerInner::load(&path, self.desc.image_layer_lsn(), None, None, ctx) - .await - .and_then(|res| res)?; + let loaded = + ImageLayerInner::load(&path, self.desc.image_layer_lsn(), None, None, false, ctx) + .await + .and_then(|res| res)?; // not production code let actual_layer_name = LayerName::from_str(path.file_name().unwrap()).unwrap(); @@ -377,6 +380,7 @@ impl ImageLayerInner { lsn: Lsn, summary: Option, max_vectored_read_bytes: Option, + support_compressed_reads: bool, ctx: &RequestContext, ) -> Result, anyhow::Error> { let file = match VirtualFile::open(path, ctx).await { @@ -420,6 +424,7 @@ impl ImageLayerInner { file, file_id, max_vectored_read_bytes, + compressed_reads: support_compressed_reads, key_range: actual_summary.key_range, })) } @@ -430,7 +435,8 @@ impl ImageLayerInner { reconstruct_state: &mut ValueReconstructState, ctx: &RequestContext, ) -> anyhow::Result { - let block_reader = FileBlockReader::new(&self.file, self.file_id); + let block_reader = + FileBlockReader::new_with_compression(&self.file, self.file_id, self.compressed_reads); let tree_reader = DiskBtreeReader::new(self.index_start_blk, self.index_root_blk, &block_reader); @@ -490,12 +496,14 @@ impl ImageLayerInner { &self, ctx: &RequestContext, ) -> anyhow::Result> { - let block_reader = FileBlockReader::new(&self.file, self.file_id); + let block_reader = + FileBlockReader::new_with_compression(&self.file, self.file_id, self.compressed_reads); let tree_reader = DiskBtreeReader::new(self.index_start_blk, self.index_root_blk, &block_reader); let mut result = Vec::new(); let mut stream = Box::pin(tree_reader.into_stream(&[0; KEY_SIZE], ctx)); - let block_reader = FileBlockReader::new(&self.file, self.file_id); + let block_reader = + FileBlockReader::new_with_compression(&self.file, self.file_id, self.compressed_reads); let cursor = block_reader.block_cursor(); while let Some(item) = stream.next().await { // TODO: dedup code with get_reconstruct_value @@ -530,7 +538,8 @@ impl ImageLayerInner { .into(), ); - let block_reader = FileBlockReader::new(&self.file, self.file_id); + let block_reader = + FileBlockReader::new_with_compression(&self.file, self.file_id, self.compressed_reads); let tree_reader = DiskBtreeReader::new(self.index_start_blk, self.index_root_blk, block_reader); @@ -691,7 +700,8 @@ impl ImageLayerInner { #[cfg(test)] pub(crate) fn iter<'a>(&'a self, ctx: &'a RequestContext) -> ImageLayerIterator<'a> { - let block_reader = FileBlockReader::new(&self.file, self.file_id); + let block_reader = + FileBlockReader::new_with_compression(&self.file, self.file_id, self.compressed_reads); let tree_reader = DiskBtreeReader::new(self.index_start_blk, self.index_root_blk, block_reader); ImageLayerIterator { diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index 02069c29d264..d1f5cc8f43a7 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -1685,6 +1685,7 @@ impl DownloadedLayer { lsn, summary, Some(owner.conf.max_vectored_read_bytes), + owner.conf.image_compression.is_some(), ctx, ) .await From 392a58bdce6ffda454fe6e78f6158f817d6effc3 Mon Sep 17 00:00:00 2001 From: Peter Bendel Date: Wed, 3 Jul 2024 18:22:33 +0200 Subject: [PATCH 026/194] add pagebench test cases for periodic pagebench on dedicated hardware (#8233) we want to run some specific pagebench test cases on dedicated hardware to get reproducible results run1: 1 client per tenant => characterize throughput with n tenants. - 500 tenants - scale 13 (200 MB database) - 1 hour duration - ca 380 GB layer snapshot files run2.singleclient: 1 client per tenant => characterize latencies run2.manyclient: N clients per tenant => characterize throughput scalability within one tenant. - 1 tenant with 1 client for latencies - 1 tenant with 64 clients because typically for a high number of connections we recommend the connection pooler which by default uses 64 connections (for scalability) - scale 136 (2048 MB database) - 20 minutes each --- .github/workflows/periodic_pagebench.yml | 144 ++++++++++++++++++ ...er_max_throughput_getpage_at_latest_lsn.py | 86 ++++++++--- test_runner/performance/pageserver/util.py | 2 +- 3 files changed, 212 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/periodic_pagebench.yml diff --git a/.github/workflows/periodic_pagebench.yml b/.github/workflows/periodic_pagebench.yml new file mode 100644 index 000000000000..c0219599a2e7 --- /dev/null +++ b/.github/workflows/periodic_pagebench.yml @@ -0,0 +1,144 @@ +name: Periodic pagebench performance test on dedicated EC2 machine in eu-central-1 region + +on: + schedule: + # * is a special character in YAML so you have to quote this string + # ┌───────────── minute (0 - 59) + # │ ┌───────────── hour (0 - 23) + # │ │ ┌───────────── day of the month (1 - 31) + # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) + # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) + - cron: '0 18 * * *' # Runs at 6 PM UTC every day + workflow_dispatch: # Allows manual triggering of the workflow + inputs: + commit_hash: + type: string + description: 'The long neon repo commit hash for the system under test (pageserver) to be tested.' + required: false + default: '' + +defaults: + run: + shell: bash -euo pipefail {0} + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + trigger_bench_on_ec2_machine_in_eu_central_1: + runs-on: [ self-hosted, gen3, small ] + container: + image: neondatabase/build-tools:pinned + credentials: + username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} + password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + options: --init + timeout-minutes: 360 # Set the timeout to 6 hours + env: + API_KEY: ${{ secrets.PERIODIC_PAGEBENCH_EC2_RUNNER_API_KEY }} + RUN_ID: ${{ github.run_id }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_EC2_US_TEST_RUNNER_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY : ${{ secrets.AWS_EC2_US_TEST_RUNNER_ACCESS_KEY_SECRET }} + AWS_DEFAULT_REGION : "eu-central-1" + AWS_INSTANCE_ID : "i-02a59a3bf86bc7e74" + steps: + - name: Show my own (github runner) external IP address - usefull for IP allowlisting + run: curl https://ifconfig.me + + - name: Start EC2 instance and wait for the instance to boot up + run: | + aws ec2 start-instances --instance-ids $AWS_INSTANCE_ID + aws ec2 wait instance-running --instance-ids $AWS_INSTANCE_ID + sleep 60 # sleep some time to allow cloudinit and our API server to start up + + - name: Determine public IP of the EC2 instance and set env variable EC2_MACHINE_URL_US + run: | + public_ip=$(aws ec2 describe-instances --instance-ids $AWS_INSTANCE_ID --query 'Reservations[*].Instances[*].PublicIpAddress' --output text) + echo "Public IP of the EC2 instance: $public_ip" + echo "EC2_MACHINE_URL_US=https://${public_ip}:8443" >> $GITHUB_ENV + + - name: Determine commit hash + env: + INPUT_COMMIT_HASH: ${{ github.event.inputs.commit_hash }} + run: | + if [ -z "$INPUT_COMMIT_HASH" ]; then + echo "COMMIT_HASH=$(curl -s https://api.github.com/repos/neondatabase/neon/commits/main | jq -r '.sha')" >> $GITHUB_ENV + else + echo "COMMIT_HASH=$INPUT_COMMIT_HASH" >> $GITHUB_ENV + fi + + - name: Start Bench with run_id + run: | + curl -k -X 'POST' \ + "${EC2_MACHINE_URL_US}/start_test/${GITHUB_RUN_ID}" \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -H "Authorization: Bearer $API_KEY" \ + -d "{\"neonRepoCommitHash\": \"${COMMIT_HASH}\"}" + + - name: Poll Test Status + id: poll_step + run: | + status="" + while [[ "$status" != "failure" && "$status" != "success" ]]; do + response=$(curl -k -X 'GET' \ + "${EC2_MACHINE_URL_US}/test_status/${GITHUB_RUN_ID}" \ + -H 'accept: application/json' \ + -H "Authorization: Bearer $API_KEY") + echo "Response: $response" + set +x + status=$(echo $response | jq -r '.status') + echo "Test status: $status" + if [[ "$status" == "failure" || "$status" == "success" || "$status" == "null" ]]; then + break + fi + if [[ "$status" == "too_many_runs" ]]; then + echo "Too many runs already running" + echo "too_many_runs=true" >> "$GITHUB_OUTPUT" + exit 1 + fi + + sleep 60 # Poll every 60 seconds + done + + - name: Retrieve Test Logs + run: | + curl -k -X 'GET' \ + "${EC2_MACHINE_URL_US}/test_log/${GITHUB_RUN_ID}" \ + -H 'accept: application/gzip' \ + -H "Authorization: Bearer $API_KEY" \ + --output "test_log_${GITHUB_RUN_ID}.gz" + + - name: Unzip Test Log and Print it into this job's log + run: | + gzip -d "test_log_${GITHUB_RUN_ID}.gz" + cat "test_log_${GITHUB_RUN_ID}" + + - name: Create Allure report + if: ${{ !cancelled() }} + uses: ./.github/actions/allure-report-generate + + - name: Post to a Slack channel + if: ${{ github.event.schedule && failure() }} + uses: slackapi/slack-github-action@v1 + with: + channel-id: "C033QLM5P7D" # dev-staging-stream + slack-message: "Periodic pagebench testing on dedicated hardware: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + + - name: Cleanup Test Resources + if: always() + run: | + curl -k -X 'POST' \ + "${EC2_MACHINE_URL_US}/cleanup_test/${GITHUB_RUN_ID}" \ + -H 'accept: application/json' \ + -H "Authorization: Bearer $API_KEY" \ + -d '' + + - name: Stop EC2 instance and wait for the instance to be stopped + if: always() && steps.poll_step.outputs.too_many_runs != 'true' + run: | + aws ec2 stop-instances --instance-ids $AWS_INSTANCE_ID + aws ec2 wait instance-stopped --instance-ids $AWS_INSTANCE_ID diff --git a/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py b/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py index 1d579214b0c5..a8f48fe675c6 100644 --- a/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py +++ b/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py @@ -1,4 +1,5 @@ import json +import os from pathlib import Path from typing import Any, Dict, Tuple @@ -17,30 +18,74 @@ setup_pageserver_with_tenants, ) +# The following tests use pagebench "getpage at latest LSN" to characterize the throughput of the pageserver. +# originally there was a single test named `test_pageserver_max_throughput_getpage_at_latest_lsn`` +# so you still see some references to this name in the code. +# To avoid recreating the snapshots for each test, we continue to use the name `max_throughput_latest_lsn` +# for some files and metrics. + + +# For reference, the space usage of the snapshots: +# sudo du -hs /instance_store/neon/test_output/shared-snapshots/* +# 416G /instance_store/neon/test_output/shared-snapshots/max_throughput_latest_lsn-500-13 +@pytest.mark.parametrize("duration", [60 * 60]) +@pytest.mark.parametrize("pgbench_scale", [get_scale_for_db(200)]) +@pytest.mark.parametrize("n_tenants", [500]) +@pytest.mark.timeout(10000) +@pytest.mark.skipif( + os.getenv("CI", "false") == "true", + reason="This test needs lot of resources and should run on dedicated HW, not in github action runners as part of CI", +) +def test_pageserver_characterize_throughput_with_n_tenants( + neon_env_builder: NeonEnvBuilder, + zenbenchmark: NeonBenchmarker, + pg_bin: PgBin, + n_tenants: int, + pgbench_scale: int, + duration: int, +): + setup_and_run_pagebench_benchmark( + neon_env_builder, zenbenchmark, pg_bin, n_tenants, pgbench_scale, duration, 1 + ) + # For reference, the space usage of the snapshots: -# admin@ip-172-31-13-23:[~/neon-main]: sudo du -hs /instance_store/test_output/shared-snapshots -# 137G /instance_store/test_output/shared-snapshots -# admin@ip-172-31-13-23:[~/neon-main]: sudo du -hs /instance_store/test_output/shared-snapshots/* -# 1.8G /instance_store/test_output/shared-snapshots/max_throughput_latest_lsn-1-13 -# 1.1G /instance_store/test_output/shared-snapshots/max_throughput_latest_lsn-1-6 -# 8.5G /instance_store/test_output/shared-snapshots/max_throughput_latest_lsn-10-13 -# 5.1G /instance_store/test_output/shared-snapshots/max_throughput_latest_lsn-10-6 -# 76G /instance_store/test_output/shared-snapshots/max_throughput_latest_lsn-100-13 -# 46G /instance_store/test_output/shared-snapshots/max_throughput_latest_lsn-100-6 -@pytest.mark.parametrize("duration", [30]) -@pytest.mark.parametrize("pgbench_scale", [get_scale_for_db(s) for s in [100, 200]]) -@pytest.mark.parametrize("n_tenants", [1, 10]) -@pytest.mark.timeout( - 10000 -) # TODO: this value is just "a really high number"; have this per instance type -def test_pageserver_max_throughput_getpage_at_latest_lsn( +# sudo du -hs /instance_store/neon/test_output/shared-snapshots/* +# 19G /instance_store/neon/test_output/shared-snapshots/max_throughput_latest_lsn-1-136 +@pytest.mark.parametrize("duration", [20 * 60]) +@pytest.mark.parametrize("pgbench_scale", [get_scale_for_db(2048)]) +# we use 1 client to characterize latencies, and 64 clients to characterize throughput/scalability +# we use 64 clients because typically for a high number of connections we recommend the connection pooler +# which by default uses 64 connections +@pytest.mark.parametrize("n_clients", [1, 64]) +@pytest.mark.parametrize("n_tenants", [1]) +@pytest.mark.timeout(2400) +@pytest.mark.skipif( + os.getenv("CI", "false") == "true", + reason="This test needs lot of resources and should run on dedicated HW, not in github action runners as part of CI", +) +def test_pageserver_characterize_latencies_with_1_client_and_throughput_with_many_clients_one_tenant( + neon_env_builder: NeonEnvBuilder, + zenbenchmark: NeonBenchmarker, + pg_bin: PgBin, + n_tenants: int, + pgbench_scale: int, + duration: int, + n_clients: int, +): + setup_and_run_pagebench_benchmark( + neon_env_builder, zenbenchmark, pg_bin, n_tenants, pgbench_scale, duration, n_clients + ) + + +def setup_and_run_pagebench_benchmark( neon_env_builder: NeonEnvBuilder, zenbenchmark: NeonBenchmarker, pg_bin: PgBin, n_tenants: int, pgbench_scale: int, duration: int, + n_clients: int, ): def record(metric, **kwargs): zenbenchmark.record( @@ -55,6 +100,7 @@ def record(metric, **kwargs): "n_tenants": (n_tenants, {"unit": ""}), "pgbench_scale": (pgbench_scale, {"unit": ""}), "duration": (duration, {"unit": "s"}), + "n_clients": (n_clients, {"unit": ""}), } ) @@ -96,7 +142,7 @@ def setup_wrapper(env: NeonEnv): r".*query handler for.*pagestream.*failed: unexpected message: CopyFail during COPY.*" ) - run_benchmark_max_throughput_latest_lsn(env, pg_bin, record, duration) + run_pagebench_benchmark(env, pg_bin, record, duration, n_clients) def setup_tenant_template(env: NeonEnv, pg_bin: PgBin, scale: int): @@ -157,8 +203,8 @@ def setup_tenant_template(env: NeonEnv, pg_bin: PgBin, scale: int): return (template_tenant, template_timeline, config) -def run_benchmark_max_throughput_latest_lsn( - env: NeonEnv, pg_bin: PgBin, record, duration_secs: int +def run_pagebench_benchmark( + env: NeonEnv, pg_bin: PgBin, record, duration_secs: int, n_clients: int ): """ Benchmark `env.pageserver` for max throughput @ latest LSN and record results in `zenbenchmark`. @@ -172,6 +218,8 @@ def run_benchmark_max_throughput_latest_lsn( ps_http.base_url, "--page-service-connstring", env.pageserver.connstr(password=None), + "--num-clients", + str(n_clients), "--runtime", f"{duration_secs}s", # don't specify the targets explicitly, let pagebench auto-discover them diff --git a/test_runner/performance/pageserver/util.py b/test_runner/performance/pageserver/util.py index 92e05663ce20..88296a7fbdec 100644 --- a/test_runner/performance/pageserver/util.py +++ b/test_runner/performance/pageserver/util.py @@ -22,7 +22,7 @@ def ensure_pageserver_ready_for_benchmarking(env: NeonEnv, n_tenants: int): log.info("wait for all tenants to become active") wait_until_all_tenants_state( - ps_http, "Active", iterations=n_tenants, period=1, http_error_ok=False + ps_http, "Active", iterations=10 + n_tenants, period=1, http_error_ok=False ) # ensure all layers are resident for predictiable performance From ea0b22a9b0e5dd03605a285ce6560926299628d8 Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 3 Jul 2024 17:27:34 +0100 Subject: [PATCH 027/194] pageserver: reduce ops tracked at per-timeline detail (#8245) ## Problem We record detailed histograms for all page_service op types, which mostly aren't very interesting, but make our prometheus scrapes huge. Closes: #8223 ## Summary of changes - Only track GetPageAtLsn histograms on a per-timeline granularity. For all other operation types, rely on existing node-wide histograms. --- pageserver/src/metrics.rs | 105 ++++++++++++++++++++------------------ 1 file changed, 54 insertions(+), 51 deletions(-) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 9cd7ffa0426c..a21d8780cf9e 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -8,7 +8,7 @@ use metrics::{ }; use once_cell::sync::Lazy; use pageserver_api::shard::TenantShardId; -use strum::{EnumCount, IntoEnumIterator, VariantNames}; +use strum::{EnumCount, VariantNames}; use strum_macros::{EnumVariantNames, IntoStaticStr}; use tracing::warn; use utils::id::TimelineId; @@ -1076,21 +1076,12 @@ pub(crate) mod virtual_file_io_engine { }); } -#[derive(Debug)] -struct GlobalAndPerTimelineHistogram { - global: Histogram, - per_tenant_timeline: Histogram, -} +struct GlobalAndPerTimelineHistogramTimer<'a, 'c> { + global_metric: &'a Histogram, -impl GlobalAndPerTimelineHistogram { - fn observe(&self, value: f64) { - self.global.observe(value); - self.per_tenant_timeline.observe(value); - } -} + // Optional because not all op types are tracked per-timeline + timeline_metric: Option<&'a Histogram>, -struct GlobalAndPerTimelineHistogramTimer<'a, 'c> { - h: &'a GlobalAndPerTimelineHistogram, ctx: &'c RequestContext, start: std::time::Instant, op: SmgrQueryType, @@ -1121,7 +1112,10 @@ impl<'a, 'c> Drop for GlobalAndPerTimelineHistogramTimer<'a, 'c> { elapsed } }; - self.h.observe(ex_throttled.as_secs_f64()); + self.global_metric.observe(ex_throttled.as_secs_f64()); + if let Some(timeline_metric) = self.timeline_metric { + timeline_metric.observe(ex_throttled.as_secs_f64()); + } } } @@ -1146,7 +1140,8 @@ pub enum SmgrQueryType { #[derive(Debug)] pub(crate) struct SmgrQueryTimePerTimeline { - metrics: [GlobalAndPerTimelineHistogram; SmgrQueryType::COUNT], + global_metrics: [Histogram; SmgrQueryType::COUNT], + per_timeline_getpage: Histogram, } static SMGR_QUERY_TIME_PER_TENANT_TIMELINE: Lazy = Lazy::new(|| { @@ -1224,27 +1219,32 @@ impl SmgrQueryTimePerTimeline { let tenant_id = tenant_shard_id.tenant_id.to_string(); let shard_slug = format!("{}", tenant_shard_id.shard_slug()); let timeline_id = timeline_id.to_string(); - let metrics = std::array::from_fn(|i| { + let global_metrics = std::array::from_fn(|i| { let op = SmgrQueryType::from_repr(i).unwrap(); - let global = SMGR_QUERY_TIME_GLOBAL + SMGR_QUERY_TIME_GLOBAL .get_metric_with_label_values(&[op.into()]) - .unwrap(); - let per_tenant_timeline = SMGR_QUERY_TIME_PER_TENANT_TIMELINE - .get_metric_with_label_values(&[op.into(), &tenant_id, &shard_slug, &timeline_id]) - .unwrap(); - GlobalAndPerTimelineHistogram { - global, - per_tenant_timeline, - } + .unwrap() }); - Self { metrics } + + let per_timeline_getpage = SMGR_QUERY_TIME_PER_TENANT_TIMELINE + .get_metric_with_label_values(&[ + SmgrQueryType::GetPageAtLsn.into(), + &tenant_id, + &shard_slug, + &timeline_id, + ]) + .unwrap(); + Self { + global_metrics, + per_timeline_getpage, + } } pub(crate) fn start_timer<'c: 'a, 'a>( &'a self, op: SmgrQueryType, ctx: &'c RequestContext, - ) -> impl Drop + '_ { - let metric = &self.metrics[op as usize]; + ) -> Option { + let global_metric = &self.global_metrics[op as usize]; let start = Instant::now(); match ctx.micros_spent_throttled.open() { Ok(()) => (), @@ -1263,12 +1263,20 @@ impl SmgrQueryTimePerTimeline { }); } } - GlobalAndPerTimelineHistogramTimer { - h: metric, + + let timeline_metric = if matches!(op, SmgrQueryType::GetPageAtLsn) { + Some(&self.per_timeline_getpage) + } else { + None + }; + + Some(GlobalAndPerTimelineHistogramTimer { + global_metric, + timeline_metric, ctx, start, op, - } + }) } } @@ -1315,17 +1323,9 @@ mod smgr_query_time_tests { let get_counts = || { let global: u64 = ops .iter() - .map(|op| metrics.metrics[*op as usize].global.get_sample_count()) + .map(|op| metrics.global_metrics[*op as usize].get_sample_count()) .sum(); - let per_tenant_timeline: u64 = ops - .iter() - .map(|op| { - metrics.metrics[*op as usize] - .per_tenant_timeline - .get_sample_count() - }) - .sum(); - (global, per_tenant_timeline) + (global, metrics.per_timeline_getpage.get_sample_count()) }; let (pre_global, pre_per_tenant_timeline) = get_counts(); @@ -1336,7 +1336,12 @@ mod smgr_query_time_tests { drop(timer); let (post_global, post_per_tenant_timeline) = get_counts(); - assert_eq!(post_per_tenant_timeline, 1); + if matches!(op, super::SmgrQueryType::GetPageAtLsn) { + // getpage ops are tracked per-timeline, others aren't + assert_eq!(post_per_tenant_timeline, 1); + } else { + assert_eq!(post_per_tenant_timeline, 0); + } assert!(post_global > pre_global); } } @@ -2317,14 +2322,12 @@ impl TimelineMetrics { let _ = STORAGE_IO_SIZE.remove_label_values(&[op, tenant_id, shard_id, timeline_id]); } - for op in SmgrQueryType::iter() { - let _ = SMGR_QUERY_TIME_PER_TENANT_TIMELINE.remove_label_values(&[ - op.into(), - tenant_id, - shard_id, - timeline_id, - ]); - } + let _ = SMGR_QUERY_TIME_PER_TENANT_TIMELINE.remove_label_values(&[ + SmgrQueryType::GetPageAtLsn.into(), + tenant_id, + shard_id, + timeline_id, + ]); } } From cdaed4d79c7ac592d909cd958c909fd1795da65c Mon Sep 17 00:00:00 2001 From: Japin Li Date: Thu, 4 Jul 2024 01:55:36 +0800 Subject: [PATCH 028/194] Fix outdated comment (#8149) Commit 97b48c23f changes the log wait timeout from 1 second to 100 milliseconds but forgets to update the comment. --- compute_tools/src/compute.rs | 5 ++--- compute_tools/src/pg_helpers.rs | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index a79b666409ae..41a52ef5b641 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -873,9 +873,8 @@ impl ComputeNode { Ok(()) } - // We could've wrapped this around `pg_ctl reload`, but right now we don't use - // `pg_ctl` for start / stop, so this just seems much easier to do as we already - // have opened connection to Postgres and superuser access. + // Wrapped this around `pg_ctl reload`, but right now we don't use + // `pg_ctl` for start / stop. #[instrument(skip_all)] fn pg_reload_conf(&self) -> Result<()> { let pgctl_bin = Path::new(&self.pgbin).parent().unwrap().join("pg_ctl"); diff --git a/compute_tools/src/pg_helpers.rs b/compute_tools/src/pg_helpers.rs index fa0822748b61..863fa9468ff4 100644 --- a/compute_tools/src/pg_helpers.rs +++ b/compute_tools/src/pg_helpers.rs @@ -489,7 +489,7 @@ pub fn handle_postgres_logs(stderr: std::process::ChildStderr) -> JoinHandle<()> /// Read Postgres logs from `stderr` until EOF. Buffer is flushed on one of the following conditions: /// - next line starts with timestamp /// - EOF -/// - no new lines were written for the last second +/// - no new lines were written for the last 100 milliseconds async fn handle_postgres_logs_async(stderr: tokio::process::ChildStderr) -> Result<()> { let mut lines = tokio::io::BufReader::new(stderr).lines(); let timeout_duration = Duration::from_millis(100); From a85aa03d18a788d7d4954f44099e14179ad6489f Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 3 Jul 2024 20:05:01 +0200 Subject: [PATCH 029/194] page_service: stop exposing `get_last_record_rlsn` (#8244) Compute doesn't use it, let's eliminate it. Ref to Slack thread: https://neondb.slack.com/archives/C033RQ5SPDH/p1719920261995529 --- pageserver/src/metrics.rs | 1 - pageserver/src/page_service.rs | 47 -------------------------------- test_runner/regress/test_auth.py | 2 +- 3 files changed, 1 insertion(+), 49 deletions(-) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index a21d8780cf9e..87ff8f4d6467 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -1452,7 +1452,6 @@ pub(crate) enum ComputeCommandKind { PageStreamV2, PageStream, Basebackup, - GetLastRecordRlsn, Fullbackup, ImportBasebackup, ImportWal, diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 6ea5f396d0a7..a440ad63785b 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -1656,53 +1656,6 @@ where metric_recording.observe(&res); res?; } - // return pair of prev_lsn and last_lsn - else if let Some(params) = parts.strip_prefix(&["get_last_record_rlsn"]) { - if params.len() != 2 { - return Err(QueryError::Other(anyhow::anyhow!( - "invalid param number for get_last_record_rlsn command" - ))); - } - - let tenant_id = TenantId::from_str(params[0]) - .with_context(|| format!("Failed to parse tenant id from {}", params[0]))?; - let timeline_id = TimelineId::from_str(params[1]) - .with_context(|| format!("Failed to parse timeline id from {}", params[1]))?; - - tracing::Span::current() - .record("tenant_id", field::display(tenant_id)) - .record("timeline_id", field::display(timeline_id)); - - self.check_permission(Some(tenant_id))?; - - COMPUTE_COMMANDS_COUNTERS - .for_command(ComputeCommandKind::GetLastRecordRlsn) - .inc(); - - async { - let timeline = self - .get_active_tenant_timeline(tenant_id, timeline_id, ShardSelector::Zero) - .await?; - - let end_of_timeline = timeline.get_last_record_rlsn(); - - pgb.write_message_noflush(&BeMessage::RowDescription(&[ - RowDescriptor::text_col(b"prev_lsn"), - RowDescriptor::text_col(b"last_lsn"), - ]))? - .write_message_noflush(&BeMessage::DataRow(&[ - Some(end_of_timeline.prev.to_string().as_bytes()), - Some(end_of_timeline.last.to_string().as_bytes()), - ]))? - .write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?; - anyhow::Ok(()) - } - .instrument(info_span!( - "handle_get_last_record_lsn", - shard_id = tracing::field::Empty - )) - .await?; - } // same as basebackup, but result includes relational data as well else if let Some(params) = parts.strip_prefix(&["fullbackup"]) { if params.len() < 2 { diff --git a/test_runner/regress/test_auth.py b/test_runner/regress/test_auth.py index 035ab2796f6a..922a21a99929 100644 --- a/test_runner/regress/test_auth.py +++ b/test_runner/regress/test_auth.py @@ -211,7 +211,7 @@ def op(): def check_pageserver(expect_success: bool, **conn_kwargs): check_connection( env.pageserver, - f"get_last_record_rlsn {env.initial_tenant} {timeline_id}", + f"show {env.initial_tenant}", expect_success, **conn_kwargs, ) From 90b51dcf1614614340fafaf61957b645fac34903 Mon Sep 17 00:00:00 2001 From: Alex Chi Z Date: Wed, 3 Jul 2024 14:46:58 -0400 Subject: [PATCH 030/194] fix(pageserver): ensure test creates valid layer map (#8191) I'd like to add some constraints to the layer map we generate in tests. (1) is the layer map that the current compaction algorithm will produce. There is a property that for all delta layer, all delta layer overlaps with it on the LSN axis will have the same LSN range. (2) is the layer map that cannot be produced with the legacy compaction algorithm. (3) is the layer map that will be produced by the future tiered-compaction algorithm. The current validator does not allow that but we can modify the algorithm to allow it in the future. ## Summary of changes Add a validator to check if the layer map is valid and refactor the test cases to include delta layer start/end LSN. --------- Signed-off-by: Alex Chi Z Co-authored-by: Christian Schwarz --- pageserver/src/tenant.rs | 177 ++++++++++++++++-------------- pageserver/src/tenant/timeline.rs | 92 +++++++++++++--- 2 files changed, 172 insertions(+), 97 deletions(-) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 0c911939e848..adf492ace762 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -1365,7 +1365,7 @@ impl Tenant { initdb_lsn: Lsn, pg_version: u32, ctx: &RequestContext, - delta_layer_desc: Vec>, + delta_layer_desc: Vec, image_layer_desc: Vec<(Lsn, Vec<(pageserver_api::key::Key, bytes::Bytes)>)>, end_lsn: Lsn, ) -> anyhow::Result> { @@ -2933,7 +2933,7 @@ impl Tenant { dst_id: TimelineId, ancestor_lsn: Option, ctx: &RequestContext, - delta_layer_desc: Vec>, + delta_layer_desc: Vec, image_layer_desc: Vec<(Lsn, Vec<(pageserver_api::key::Key, bytes::Bytes)>)>, end_lsn: Lsn, ) -> anyhow::Result> { @@ -3933,7 +3933,7 @@ mod tests { use storage_layer::PersistentLayerKey; use tests::storage_layer::ValuesReconstructState; use tests::timeline::{GetVectoredError, ShutdownMode}; - use timeline::GcInfo; + use timeline::{DeltaLayerTestDesc, GcInfo}; use utils::bin_ser::BeSer; use utils::id::TenantId; @@ -6229,27 +6229,6 @@ mod tests { .await .unwrap(); - async fn get_vectored_impl_wrapper( - tline: &Arc, - key: Key, - lsn: Lsn, - ctx: &RequestContext, - ) -> Result, GetVectoredError> { - let mut reconstruct_state = ValuesReconstructState::new(); - let mut res = tline - .get_vectored_impl( - KeySpace::single(key..key.next()), - lsn, - &mut reconstruct_state, - ctx, - ) - .await?; - Ok(res.pop_last().map(|(k, v)| { - assert_eq!(k, key); - v.unwrap() - })) - } - let lsn = Lsn(0x30); // test vectored get on parent timeline @@ -6325,27 +6304,6 @@ mod tests { .await .unwrap(); - async fn get_vectored_impl_wrapper( - tline: &Arc, - key: Key, - lsn: Lsn, - ctx: &RequestContext, - ) -> Result, GetVectoredError> { - let mut reconstruct_state = ValuesReconstructState::new(); - let mut res = tline - .get_vectored_impl( - KeySpace::single(key..key.next()), - lsn, - &mut reconstruct_state, - ctx, - ) - .await?; - Ok(res.pop_last().map(|(k, v)| { - assert_eq!(k, key); - v.unwrap() - })) - } - let lsn = Lsn(0x30); // test vectored get on parent timeline @@ -6421,9 +6379,18 @@ mod tests { &ctx, // delta layers vec![ - vec![(key2, Lsn(0x10), Value::Image(test_img("metadata key 2")))], - vec![(key1, Lsn(0x20), Value::Image(Bytes::new()))], - vec![(key2, Lsn(0x20), Value::Image(Bytes::new()))], + DeltaLayerTestDesc::new_with_inferred_key_range( + Lsn(0x10)..Lsn(0x20), + vec![(key2, Lsn(0x10), Value::Image(test_img("metadata key 2")))], + ), + DeltaLayerTestDesc::new_with_inferred_key_range( + Lsn(0x20)..Lsn(0x30), + vec![(key1, Lsn(0x20), Value::Image(Bytes::new()))], + ), + DeltaLayerTestDesc::new_with_inferred_key_range( + Lsn(0x20)..Lsn(0x30), + vec![(key2, Lsn(0x20), Value::Image(Bytes::new()))], + ), ], // image layers vec![ @@ -6489,17 +6456,29 @@ mod tests { &ctx, // delta layers vec![ - vec![(key2, Lsn(0x10), Value::Image(test_img("metadata key 2")))], - vec![(key1, Lsn(0x20), Value::Image(Bytes::new()))], - vec![(key2, Lsn(0x20), Value::Image(Bytes::new()))], - vec![ - (key0, Lsn(0x30), Value::Image(test_img("metadata key 0"))), - (key3, Lsn(0x30), Value::Image(test_img("metadata key 3"))), - ], + DeltaLayerTestDesc::new_with_inferred_key_range( + Lsn(0x10)..Lsn(0x20), + vec![(key2, Lsn(0x10), Value::Image(test_img("metadata key 2")))], + ), + DeltaLayerTestDesc::new_with_inferred_key_range( + Lsn(0x20)..Lsn(0x30), + vec![(key1, Lsn(0x20), Value::Image(Bytes::new()))], + ), + DeltaLayerTestDesc::new_with_inferred_key_range( + Lsn(0x20)..Lsn(0x30), + vec![(key2, Lsn(0x20), Value::Image(Bytes::new()))], + ), + DeltaLayerTestDesc::new_with_inferred_key_range( + Lsn(0x30)..Lsn(0x40), + vec![ + (key0, Lsn(0x30), Value::Image(test_img("metadata key 0"))), + (key3, Lsn(0x30), Value::Image(test_img("metadata key 3"))), + ], + ), ], // image layers vec![(Lsn(0x10), vec![(key1, test_img("metadata key 1"))])], - Lsn(0x30), + Lsn(0x40), ) .await .unwrap(); @@ -6522,7 +6501,7 @@ mod tests { // Image layers are created at last_record_lsn let images = tline - .inspect_image_layers(Lsn(0x30), &ctx) + .inspect_image_layers(Lsn(0x40), &ctx) .await .unwrap() .into_iter() @@ -6548,9 +6527,18 @@ mod tests { &ctx, // delta layers vec![ - vec![(key2, Lsn(0x10), Value::Image(test_img("metadata key 2")))], - vec![(key1, Lsn(0x20), Value::Image(Bytes::new()))], - vec![(key2, Lsn(0x20), Value::Image(Bytes::new()))], + DeltaLayerTestDesc::new_with_inferred_key_range( + Lsn(0x10)..Lsn(0x20), + vec![(key2, Lsn(0x10), Value::Image(test_img("metadata key 2")))], + ), + DeltaLayerTestDesc::new_with_inferred_key_range( + Lsn(0x20)..Lsn(0x30), + vec![(key1, Lsn(0x20), Value::Image(Bytes::new()))], + ), + DeltaLayerTestDesc::new_with_inferred_key_range( + Lsn(0x20)..Lsn(0x30), + vec![(key2, Lsn(0x20), Value::Image(Bytes::new()))], + ), ], // image layers vec![(Lsn(0x10), vec![(key1, test_img("metadata key 1"))])], @@ -6598,15 +6586,21 @@ mod tests { key } - // We create one bottom-most image layer, a delta layer D1 crossing the GC horizon, D2 below the horizon, and D3 above the horizon. + // We create + // - one bottom-most image layer, + // - a delta layer D1 crossing the GC horizon with data below and above the horizon, + // - a delta layer D2 crossing the GC horizon with data only below the horizon, + // - a delta layer D3 above the horizon. // - // | D1 | | D3 | + // | D3 | + // | D1 | // -| |-- gc horizon ----------------- // | | | D2 | // --------- img layer ------------------ // // What we should expact from this compaction is: - // | Part of D1 | | D3 | + // | D3 | + // | Part of D1 | // --------- img layer with D1+D2 at GC horizon------------------ // img layer at 0x10 @@ -6646,13 +6640,13 @@ mod tests { let delta3 = vec![ ( get_key(8), - Lsn(0x40), - Value::Image(Bytes::from("value 8@0x40")), + Lsn(0x48), + Value::Image(Bytes::from("value 8@0x48")), ), ( get_key(9), - Lsn(0x40), - Value::Image(Bytes::from("value 9@0x40")), + Lsn(0x48), + Value::Image(Bytes::from("value 9@0x48")), ), ]; @@ -6662,7 +6656,11 @@ mod tests { Lsn(0x10), DEFAULT_PG_VERSION, &ctx, - vec![delta1, delta2, delta3], // delta layers + vec![ + DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x20)..Lsn(0x48), delta1), + DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x20)..Lsn(0x48), delta2), + DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x48)..Lsn(0x50), delta3), + ], // delta layers vec![(Lsn(0x10), img_layer)], // image layers Lsn(0x50), ) @@ -6683,8 +6681,8 @@ mod tests { Bytes::from_static(b"value 5@0x20"), Bytes::from_static(b"value 6@0x20"), Bytes::from_static(b"value 7@0x10"), - Bytes::from_static(b"value 8@0x40"), - Bytes::from_static(b"value 9@0x40"), + Bytes::from_static(b"value 8@0x48"), + Bytes::from_static(b"value 9@0x48"), ]; for (idx, expected) in expected_result.iter().enumerate() { @@ -6772,10 +6770,10 @@ mod tests { lsn_range: Lsn(0x30)..Lsn(0x41), is_delta: true }, - // The delta layer we created and should not be picked for the compaction + // The delta3 layer that should not be picked for the compaction PersistentLayerKey { key_range: get_key(8)..get_key(10), - lsn_range: Lsn(0x40)..Lsn(0x41), + lsn_range: Lsn(0x48)..Lsn(0x50), is_delta: true } ] @@ -6839,7 +6837,10 @@ mod tests { Lsn(0x10), DEFAULT_PG_VERSION, &ctx, - vec![delta1], // delta layers + vec![DeltaLayerTestDesc::new_with_inferred_key_range( + Lsn(0x10)..Lsn(0x40), + delta1, + )], // delta layers vec![(Lsn(0x10), image1)], // image layers Lsn(0x50), ) @@ -6963,15 +6964,21 @@ mod tests { key } - // We create one bottom-most image layer, a delta layer D1 crossing the GC horizon, D2 below the horizon, and D3 above the horizon. + // We create + // - one bottom-most image layer, + // - a delta layer D1 crossing the GC horizon with data below and above the horizon, + // - a delta layer D2 crossing the GC horizon with data only below the horizon, + // - a delta layer D3 above the horizon. // - // | D1 | | D3 | + // | D3 | + // | D1 | // -| |-- gc horizon ----------------- // | | | D2 | // --------- img layer ------------------ // // What we should expact from this compaction is: - // | Part of D1 | | D3 | + // | D3 | + // | Part of D1 | // --------- img layer with D1+D2 at GC horizon------------------ // img layer at 0x10 @@ -7021,13 +7028,13 @@ mod tests { let delta3 = vec![ ( get_key(8), - Lsn(0x40), - Value::WalRecord(NeonWalRecord::wal_append("@0x40")), + Lsn(0x48), + Value::WalRecord(NeonWalRecord::wal_append("@0x48")), ), ( get_key(9), - Lsn(0x40), - Value::WalRecord(NeonWalRecord::wal_append("@0x40")), + Lsn(0x48), + Value::WalRecord(NeonWalRecord::wal_append("@0x48")), ), ]; @@ -7037,7 +7044,11 @@ mod tests { Lsn(0x10), DEFAULT_PG_VERSION, &ctx, - vec![delta1, delta2, delta3], // delta layers + vec![ + DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x10)..Lsn(0x48), delta1), + DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x10)..Lsn(0x48), delta2), + DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x48)..Lsn(0x50), delta3), + ], // delta layers vec![(Lsn(0x10), img_layer)], // image layers Lsn(0x50), ) @@ -7064,8 +7075,8 @@ mod tests { Bytes::from_static(b"value 5@0x10@0x20"), Bytes::from_static(b"value 6@0x10@0x20"), Bytes::from_static(b"value 7@0x10"), - Bytes::from_static(b"value 8@0x10@0x40"), - Bytes::from_static(b"value 9@0x10@0x40"), + Bytes::from_static(b"value 8@0x10@0x48"), + Bytes::from_static(b"value 9@0x10@0x48"), ]; let expected_result_at_gc_horizon = [ diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index de9361d72103..df4d252ad21e 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -4735,6 +4735,42 @@ impl DurationRecorder { } } +/// Descriptor for a delta layer used in testing infra. The start/end key/lsn range of the +/// delta layer might be different from the min/max key/lsn in the delta layer. Therefore, +/// the layer descriptor requires the user to provide the ranges, which should cover all +/// keys specified in the `data` field. +#[cfg(test)] +pub struct DeltaLayerTestDesc { + pub lsn_range: Range, + pub key_range: Range, + pub data: Vec<(Key, Lsn, Value)>, +} + +#[cfg(test)] +impl DeltaLayerTestDesc { + #[allow(dead_code)] + pub fn new(lsn_range: Range, key_range: Range, data: Vec<(Key, Lsn, Value)>) -> Self { + Self { + lsn_range, + key_range, + data, + } + } + + pub fn new_with_inferred_key_range( + lsn_range: Range, + data: Vec<(Key, Lsn, Value)>, + ) -> Self { + let key_min = data.iter().map(|(key, _, _)| key).min().unwrap(); + let key_max = data.iter().map(|(key, _, _)| key).max().unwrap(); + Self { + key_range: (*key_min)..(key_max.next()), + lsn_range, + data, + } + } +} + impl Timeline { async fn finish_compact_batch( self: &Arc, @@ -5535,37 +5571,65 @@ impl Timeline { #[cfg(test)] pub(super) async fn force_create_delta_layer( self: &Arc, - mut deltas: Vec<(Key, Lsn, Value)>, + mut deltas: DeltaLayerTestDesc, check_start_lsn: Option, ctx: &RequestContext, ) -> anyhow::Result<()> { let last_record_lsn = self.get_last_record_lsn(); - deltas.sort_unstable_by(|(ka, la, _), (kb, lb, _)| (ka, la).cmp(&(kb, lb))); - let min_key = *deltas.first().map(|(k, _, _)| k).unwrap(); - let end_key = deltas.last().map(|(k, _, _)| k).unwrap().next(); - let min_lsn = *deltas.iter().map(|(_, lsn, _)| lsn).min().unwrap(); - let max_lsn = *deltas.iter().map(|(_, lsn, _)| lsn).max().unwrap(); + deltas + .data + .sort_unstable_by(|(ka, la, _), (kb, lb, _)| (ka, la).cmp(&(kb, lb))); + assert!(deltas.data.first().unwrap().0 >= deltas.key_range.start); + assert!(deltas.data.last().unwrap().0 < deltas.key_range.end); + for (_, lsn, _) in &deltas.data { + assert!(deltas.lsn_range.start <= *lsn && *lsn < deltas.lsn_range.end); + } assert!( - max_lsn <= last_record_lsn, - "advance last record lsn before inserting a layer, max_lsn={max_lsn}, last_record_lsn={last_record_lsn}" + deltas.lsn_range.end <= last_record_lsn, + "advance last record lsn before inserting a layer, end_lsn={}, last_record_lsn={}", + deltas.lsn_range.end, + last_record_lsn ); - let end_lsn = Lsn(max_lsn.0 + 1); if let Some(check_start_lsn) = check_start_lsn { - assert!(min_lsn >= check_start_lsn); + assert!(deltas.lsn_range.start >= check_start_lsn); + } + // check if the delta layer does not violate the LSN invariant, the legacy compaction should always produce a batch of + // layers of the same start/end LSN, and so should the force inserted layer + { + /// Checks if a overlaps with b, assume a/b = [start, end). + pub fn overlaps_with(a: &Range, b: &Range) -> bool { + !(a.end <= b.start || b.end <= a.start) + } + + let guard = self.layers.read().await; + for layer in guard.layer_map().iter_historic_layers() { + if layer.is_delta() + && overlaps_with(&layer.lsn_range, &deltas.lsn_range) + && layer.lsn_range != deltas.lsn_range + { + // If a delta layer overlaps with another delta layer AND their LSN range is not the same, panic + panic!( + "inserted layer violates delta layer LSN invariant: current_lsn_range={}..{}, conflict_lsn_range={}..{}", + deltas.lsn_range.start, deltas.lsn_range.end, layer.lsn_range.start, layer.lsn_range.end + ); + } + } } let mut delta_layer_writer = DeltaLayerWriter::new( self.conf, self.timeline_id, self.tenant_shard_id, - min_key, - min_lsn..end_lsn, + deltas.key_range.start, + deltas.lsn_range, ctx, ) .await?; - for (key, lsn, val) in deltas { + for (key, lsn, val) in deltas.data { delta_layer_writer.put_value(key, lsn, val, ctx).await?; } - let delta_layer = delta_layer_writer.finish(end_key, self, ctx).await?; + let delta_layer = delta_layer_writer + .finish(deltas.key_range.end, self, ctx) + .await?; { let mut guard = self.layers.write().await; From 778787d8e97243945d58515cbe48606c947498c8 Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 3 Jul 2024 22:29:43 +0100 Subject: [PATCH 031/194] pageserver: add supplementary branch usage stats (#8131) ## Problem The metrics we have today aren't convenient for planning around the impact of timeline archival on costs. Closes: https://github.com/neondatabase/neon/issues/8108 ## Summary of changes - Add metric `pageserver_archive_size`, which indicates the logical bytes of data which we would expect to write into an archived branch. - Add metric `pageserver_pitr_history_size`, which indicates the distance between last_record_lsn and the PITR cutoff. These metrics are somewhat temporary: when we implement #8088 and associated consumption metric changes, these will reach a final form. For now, an "archived" branch is just any branch outside of its parent's PITR window: later, archival will become an explicit state (which will _usually_ correspond to falling outside the parent's PITR window). The overall volume of timeline metrics is something to watch, but we are removing many more in https://github.com/neondatabase/neon/pull/8245 than this PR is adding. --- libs/pageserver_api/src/models.rs | 10 +++++++++ pageserver/src/http/routes.rs | 4 ++++ pageserver/src/metrics.rs | 35 +++++++++++++++++++++++++++++++ pageserver/src/tenant.rs | 27 ++++++++++++++++++++++++ pageserver/src/tenant/timeline.rs | 15 +++++++++++++ test_runner/fixtures/metrics.py | 2 ++ 6 files changed, 93 insertions(+) diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index 959e161c167a..92289537613d 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -661,6 +661,16 @@ pub struct TimelineInfo { pub current_physical_size: Option, // is None when timeline is Unloaded pub current_logical_size_non_incremental: Option, + /// How many bytes of WAL are within this branch's pitr_interval. If the pitr_interval goes + /// beyond the branch's branch point, we only count up to the branch point. + pub pitr_history_size: u64, + + /// Whether this branch's branch point is within its ancestor's PITR interval (i.e. any + /// ancestor data used by this branch would have been retained anyway). If this is false, then + /// this branch may be imposing a cost on the ancestor by causing it to retain layers that it would + /// otherwise be able to GC. + pub within_ancestor_pitr: bool, + pub timeline_dir_layer_file_size_sum: Option, pub wal_source_connstr: Option, diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index f726ba115d83..6a6f17604dee 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -406,6 +406,8 @@ async fn build_timeline_info_common( let walreceiver_status = timeline.walreceiver_status(); + let (pitr_history_size, within_ancestor_pitr) = timeline.get_pitr_history_stats(); + let info = TimelineInfo { tenant_id: timeline.tenant_shard_id, timeline_id: timeline.timeline_id, @@ -426,6 +428,8 @@ async fn build_timeline_info_common( directory_entries_counts: timeline.get_directory_metrics().to_vec(), current_physical_size, current_logical_size_non_incremental: None, + pitr_history_size, + within_ancestor_pitr, timeline_dir_layer_file_size_sum: None, wal_source_connstr, last_received_msg_lsn, diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 87ff8f4d6467..9e9fe7fbb834 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -464,6 +464,24 @@ static LAST_RECORD_LSN: Lazy = Lazy::new(|| { .expect("failed to define a metric") }); +static PITR_HISTORY_SIZE: Lazy = Lazy::new(|| { + register_uint_gauge_vec!( + "pageserver_pitr_history_size", + "Data written since PITR cutoff on this timeline", + &["tenant_id", "shard_id", "timeline_id"] + ) + .expect("failed to define a metric") +}); + +static TIMELINE_ARCHIVE_SIZE: Lazy = Lazy::new(|| { + register_uint_gauge_vec!( + "pageserver_archive_size", + "Timeline's logical size if it is considered eligible for archival (outside PITR window), else zero", + &["tenant_id", "shard_id", "timeline_id"] + ) + .expect("failed to define a metric") +}); + static STANDBY_HORIZON: Lazy = Lazy::new(|| { register_int_gauge_vec!( "pageserver_standby_horizon", @@ -2106,6 +2124,8 @@ pub(crate) struct TimelineMetrics { pub garbage_collect_histo: StorageTimeMetrics, pub find_gc_cutoffs_histo: StorageTimeMetrics, pub last_record_gauge: IntGauge, + pub pitr_history_size: UIntGauge, + pub archival_size: UIntGauge, pub standby_horizon_gauge: IntGauge, pub resident_physical_size_gauge: UIntGauge, /// copy of LayeredTimeline.current_logical_size @@ -2179,6 +2199,15 @@ impl TimelineMetrics { let last_record_gauge = LAST_RECORD_LSN .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id]) .unwrap(); + + let pitr_history_size = PITR_HISTORY_SIZE + .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id]) + .unwrap(); + + let archival_size = TIMELINE_ARCHIVE_SIZE + .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id]) + .unwrap(); + let standby_horizon_gauge = STANDBY_HORIZON .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id]) .unwrap(); @@ -2231,6 +2260,8 @@ impl TimelineMetrics { find_gc_cutoffs_histo, load_layer_map_histo, last_record_gauge, + pitr_history_size, + archival_size, standby_horizon_gauge, resident_physical_size_gauge, current_logical_size_gauge, @@ -2288,6 +2319,10 @@ impl TimelineMetrics { if let Some(metric) = Lazy::get(&DIRECTORY_ENTRIES_COUNT) { let _ = metric.remove_label_values(&[tenant_id, shard_id, timeline_id]); } + + let _ = TIMELINE_ARCHIVE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]); + let _ = PITR_HISTORY_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]); + let _ = EVICTIONS.remove_label_values(&[tenant_id, shard_id, timeline_id]); let _ = AUX_FILE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]); let _ = VALID_LSN_LEASE_COUNT.remove_label_values(&[tenant_id, shard_id, timeline_id]); diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index adf492ace762..eef8dc104c69 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -2874,6 +2874,7 @@ impl Tenant { { let mut target = timeline.gc_info.write().unwrap(); + // Cull any expired leases let now = SystemTime::now(); target.leases.retain(|_, lease| !lease.is_expired(&now)); @@ -2882,6 +2883,31 @@ impl Tenant { .valid_lsn_lease_count_gauge .set(target.leases.len() as u64); + // Look up parent's PITR cutoff to update the child's knowledge of whether it is within parent's PITR + if let Some(ancestor_id) = timeline.get_ancestor_timeline_id() { + if let Some(ancestor_gc_cutoffs) = gc_cutoffs.get(&ancestor_id) { + target.within_ancestor_pitr = + timeline.get_ancestor_lsn() >= ancestor_gc_cutoffs.pitr; + } + } + + // Update metrics that depend on GC state + timeline + .metrics + .archival_size + .set(if target.within_ancestor_pitr { + timeline.metrics.current_logical_size_gauge.get() + } else { + 0 + }); + timeline.metrics.pitr_history_size.set( + timeline + .get_last_record_lsn() + .checked_sub(target.cutoffs.pitr) + .unwrap_or(Lsn(0)) + .0, + ); + match gc_cutoffs.remove(&timeline.timeline_id) { Some(cutoffs) => { target.retain_lsns = branchpoints; @@ -7063,6 +7089,7 @@ mod tests { horizon: Lsn(0x30), }, leases: Default::default(), + within_ancestor_pitr: false, }; } diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index df4d252ad21e..54bbdef56e56 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -463,6 +463,9 @@ pub(crate) struct GcInfo { /// Leases granted to particular LSNs. pub(crate) leases: BTreeMap, + + /// Whether our branch point is within our ancestor's PITR interval (for cost estimation) + pub(crate) within_ancestor_pitr: bool, } impl GcInfo { @@ -851,6 +854,18 @@ impl Timeline { .map(|ancestor| ancestor.timeline_id) } + /// Get the bytes written since the PITR cutoff on this branch, and + /// whether this branch's ancestor_lsn is within its parent's PITR. + pub(crate) fn get_pitr_history_stats(&self) -> (u64, bool) { + let gc_info = self.gc_info.read().unwrap(); + let history = self + .get_last_record_lsn() + .checked_sub(gc_info.cutoffs.pitr) + .unwrap_or(Lsn(0)) + .0; + (history, gc_info.within_ancestor_pitr) + } + /// Lock and get timeline's GC cutoff pub(crate) fn get_latest_gc_cutoff_lsn(&self) -> RcuReadGuard { self.latest_gc_cutoff_lsn.read() diff --git a/test_runner/fixtures/metrics.py b/test_runner/fixtures/metrics.py index 41fa8e679f28..c019cbbc7790 100644 --- a/test_runner/fixtures/metrics.py +++ b/test_runner/fixtures/metrics.py @@ -144,6 +144,8 @@ def histogram(prefix_without_trailing_underscore: str) -> List[str]: "pageserver_smgr_query_seconds_bucket", "pageserver_smgr_query_seconds_count", "pageserver_smgr_query_seconds_sum", + "pageserver_archive_size", + "pageserver_pitr_history_size", "pageserver_storage_operations_seconds_count_total", "pageserver_storage_operations_seconds_sum_total", "pageserver_evictions_total", From bbb2fa7cdd1284376155fcbbdf34191b335df4e6 Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Thu, 4 Jul 2024 06:04:19 +0100 Subject: [PATCH 032/194] tests: perform graceful rolling restarts in storcon scale test (#8173) ## Problem Scale test doesn't exercise drain & fill. ## Summary of changes Make scale test exercise drain & fill --- test_runner/fixtures/neon_fixtures.py | 47 +++++++ .../test_storage_controller_scale.py | 124 ++++++++++++++++-- .../regress/test_storage_controller.py | 59 ++------- 3 files changed, 171 insertions(+), 59 deletions(-) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 565aaba6e0dc..c002e11c1c08 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2113,6 +2113,21 @@ def stop(self, immediate: bool = False) -> "NeonStorageController": self.running = False return self + @staticmethod + def retryable_node_operation(op, ps_id, max_attempts, backoff): + while max_attempts > 0: + try: + op(ps_id) + return + except StorageControllerApiException as e: + max_attempts -= 1 + log.info(f"Operation failed ({max_attempts} attempts left): {e}") + + if max_attempts == 0: + raise e + + time.sleep(backoff) + @staticmethod def raise_api_exception(res: requests.Response): try: @@ -2453,6 +2468,38 @@ def consistency_check(self): ) log.info("storage controller passed consistency check") + def poll_node_status( + self, node_id: int, desired_scheduling_policy: str, max_attempts: int, backoff: int + ): + """ + Poll the node status until it reaches 'desired_scheduling_policy' or 'max_attempts' have been exhausted + """ + log.info(f"Polling {node_id} for {desired_scheduling_policy} scheduling policy") + while max_attempts > 0: + try: + status = self.node_status(node_id) + policy = status["scheduling"] + if policy == desired_scheduling_policy: + return + else: + max_attempts -= 1 + log.info(f"Status call returned {policy=} ({max_attempts} attempts left)") + + if max_attempts == 0: + raise AssertionError( + f"Status for {node_id=} did not reach {desired_scheduling_policy=}" + ) + + time.sleep(backoff) + except StorageControllerApiException as e: + max_attempts -= 1 + log.info(f"Status call failed ({max_attempts} retries left): {e}") + + if max_attempts == 0: + raise e + + time.sleep(backoff) + def configure_failpoints(self, config_strings: Tuple[str, str] | List[Tuple[str, str]]): if isinstance(config_strings, tuple): pairs = [config_strings] diff --git a/test_runner/performance/test_storage_controller_scale.py b/test_runner/performance/test_storage_controller_scale.py index a4c8c8ac421a..d65a66b01081 100644 --- a/test_runner/performance/test_storage_controller_scale.py +++ b/test_runner/performance/test_storage_controller_scale.py @@ -1,18 +1,89 @@ import concurrent.futures import random import time +from collections import defaultdict +from typing import Any, Dict import pytest from fixtures.common_types import TenantId, TenantShardId, TimelineId from fixtures.compute_reconfigure import ComputeReconfigure from fixtures.log_helper import log -from fixtures.neon_fixtures import ( - NeonEnvBuilder, -) +from fixtures.neon_fixtures import NeonEnv, NeonEnvBuilder from fixtures.pageserver.http import PageserverHttpClient from fixtures.pg_version import PgVersion +def get_consistent_node_shard_counts(env: NeonEnv, total_shards) -> defaultdict[str, int]: + """ + Get the number of shards attached to each node. + This function takes into account the intersection of the intent and the observed state. + If they do not match, it asserts out. + """ + tenants = env.storage_controller.tenant_list() + + intent = dict() + observed = dict() + + tenant_placement: defaultdict[str, Dict[str, Any]] = defaultdict( + lambda: { + "observed": {"attached": None, "secondary": []}, + "intent": {"attached": None, "secondary": []}, + } + ) + + for t in tenants: + for node_id, loc_state in t["observed"]["locations"].items(): + if ( + loc_state is not None + and "conf" in loc_state + and loc_state["conf"] is not None + and loc_state["conf"]["mode"] + in set(["AttachedSingle", "AttachedMulti", "AttachedStale"]) + ): + observed[t["tenant_shard_id"]] = int(node_id) + tenant_placement[t["tenant_shard_id"]]["observed"]["attached"] = int(node_id) + + if ( + loc_state is not None + and "conf" in loc_state + and loc_state["conf"] is not None + and loc_state["conf"]["mode"] == "Secondary" + ): + tenant_placement[t["tenant_shard_id"]]["observed"]["secondary"].append(int(node_id)) + + if "attached" in t["intent"]: + intent[t["tenant_shard_id"]] = t["intent"]["attached"] + tenant_placement[t["tenant_shard_id"]]["intent"]["attached"] = t["intent"]["attached"] + + if "secondary" in t["intent"]: + tenant_placement[t["tenant_shard_id"]]["intent"]["secondary"] += t["intent"][ + "secondary" + ] + + log.info(f"{tenant_placement=}") + + matching = { + tid: intent[tid] for tid in observed if tid in intent and intent[tid] == observed[tid] + } + assert len(matching) == total_shards + + attached_per_node: defaultdict[str, int] = defaultdict(int) + for node_id in matching.values(): + attached_per_node[node_id] += 1 + + return attached_per_node + + +def assert_consistent_balanced_attachments(env: NeonEnv, total_shards): + attached_per_node = get_consistent_node_shard_counts(env, total_shards) + + min_shard_count = min(attached_per_node.values()) + max_shard_count = max(attached_per_node.values()) + + flake_factor = 5 / 100 + assert max_shard_count - min_shard_count <= int(total_shards * flake_factor) + + @pytest.mark.timeout(3600) # super long running test: should go down as we optimize def test_storage_controller_many_tenants( neon_env_builder: NeonEnvBuilder, compute_reconfigure_listener: ComputeReconfigure @@ -44,7 +115,8 @@ def test_storage_controller_many_tenants( # A small sleep on each call into the notify hook, to simulate the latency of doing a database write compute_reconfigure_listener.register_on_notify(lambda body: time.sleep(0.01)) - env = neon_env_builder.init_start() + env = neon_env_builder.init_configs() + neon_env_builder.start() # We will intentionally stress reconciler concurrrency, which triggers a warning when lots # of shards are hitting the delayed path. @@ -79,6 +151,8 @@ def test_storage_controller_many_tenants( shard_count = 2 stripe_size = 1024 + total_shards = tenant_count * shard_count + tenants = set(TenantId.generate() for _i in range(0, tenant_count)) virtual_ps_http = PageserverHttpClient(env.storage_controller_port, lambda: True) @@ -195,10 +269,44 @@ def check_memory(): env.storage_controller.consistency_check() check_memory() - # Restart pageservers: this exercises the /re-attach API - for pageserver in env.pageservers: - pageserver.stop() - pageserver.start() + shard_counts = get_consistent_node_shard_counts(env, total_shards) + log.info(f"Shard counts before rolling restart: {shard_counts}") + + assert_consistent_balanced_attachments(env, total_shards) + + # Restart pageservers gracefully: this exercises the /re-attach pageserver API + # and the storage controller drain and fill API + for ps in env.pageservers: + env.storage_controller.retryable_node_operation( + lambda ps_id: env.storage_controller.node_drain(ps_id), ps.id, max_attempts=3, backoff=2 + ) + + env.storage_controller.poll_node_status( + ps.id, "PauseForRestart", max_attempts=24, backoff=5 + ) + + shard_counts = get_consistent_node_shard_counts(env, total_shards) + log.info(f"Shard counts after draining node {ps.id}: {shard_counts}") + # Assert that we've drained the node + assert shard_counts[str(ps.id)] == 0 + # Assert that those shards actually went somewhere + assert sum(shard_counts.values()) == total_shards + + ps.restart() + env.storage_controller.poll_node_status(ps.id, "Active", max_attempts=24, backoff=1) + + env.storage_controller.retryable_node_operation( + lambda ps_id: env.storage_controller.node_fill(ps_id), ps.id, max_attempts=3, backoff=2 + ) + env.storage_controller.poll_node_status(ps.id, "Active", max_attempts=24, backoff=5) + + shard_counts = get_consistent_node_shard_counts(env, total_shards) + log.info(f"Shard counts after filling node {ps.id}: {shard_counts}") + + assert_consistent_balanced_attachments(env, total_shards) + + env.storage_controller.reconcile_until_idle() + env.storage_controller.consistency_check() # Consistency check is safe here: restarting pageservers should not have caused any Reconcilers to spawn, # as they were not offline long enough to trigger any scheduling changes. diff --git a/test_runner/regress/test_storage_controller.py b/test_runner/regress/test_storage_controller.py index 1b294fb2d0aa..a78f566f0e4c 100644 --- a/test_runner/regress/test_storage_controller.py +++ b/test_runner/regress/test_storage_controller.py @@ -1518,49 +1518,6 @@ def test_tenant_import(neon_env_builder: NeonEnvBuilder, shard_count, remote_sto workload.validate() -def retryable_node_operation(op, ps_id, max_attempts, backoff): - while max_attempts > 0: - try: - op(ps_id) - return - except StorageControllerApiException as e: - max_attempts -= 1 - log.info(f"Operation failed ({max_attempts} attempts left): {e}") - - if max_attempts == 0: - raise e - - time.sleep(backoff) - - -def poll_node_status(env, node_id, desired_scheduling_policy, max_attempts, backoff): - log.info(f"Polling {node_id} for {desired_scheduling_policy} scheduling policy") - while max_attempts > 0: - try: - status = env.storage_controller.node_status(node_id) - policy = status["scheduling"] - if policy == desired_scheduling_policy: - return - else: - max_attempts -= 1 - log.info(f"Status call returned {policy=} ({max_attempts} attempts left)") - - if max_attempts == 0: - raise AssertionError( - f"Status for {node_id=} did not reach {desired_scheduling_policy=}" - ) - - time.sleep(backoff) - except StorageControllerApiException as e: - max_attempts -= 1 - log.info(f"Status call failed ({max_attempts} retries left): {e}") - - if max_attempts == 0: - raise e - - time.sleep(backoff) - - def test_graceful_cluster_restart(neon_env_builder: NeonEnvBuilder): """ Graceful reststart of storage controller clusters use the drain and @@ -1601,10 +1558,10 @@ def assert_shard_counts_balanced(env: NeonEnv, shard_counts, total_shards): # Perform a graceful rolling restart for ps in env.pageservers: - retryable_node_operation( + env.storage_controller.retryable_node_operation( lambda ps_id: env.storage_controller.node_drain(ps_id), ps.id, max_attempts=3, backoff=2 ) - poll_node_status(env, ps.id, "PauseForRestart", max_attempts=6, backoff=5) + env.storage_controller.poll_node_status(ps.id, "PauseForRestart", max_attempts=6, backoff=5) shard_counts = get_node_shard_counts(env, tenant_ids) log.info(f"Shard counts after draining node {ps.id}: {shard_counts}") @@ -1614,12 +1571,12 @@ def assert_shard_counts_balanced(env: NeonEnv, shard_counts, total_shards): assert sum(shard_counts.values()) == total_shards ps.restart() - poll_node_status(env, ps.id, "Active", max_attempts=10, backoff=1) + env.storage_controller.poll_node_status(ps.id, "Active", max_attempts=10, backoff=1) - retryable_node_operation( + env.storage_controller.retryable_node_operation( lambda ps_id: env.storage_controller.node_fill(ps_id), ps.id, max_attempts=3, backoff=2 ) - poll_node_status(env, ps.id, "Active", max_attempts=6, backoff=5) + env.storage_controller.poll_node_status(ps.id, "Active", max_attempts=6, backoff=5) shard_counts = get_node_shard_counts(env, tenant_ids) log.info(f"Shard counts after filling node {ps.id}: {shard_counts}") @@ -1657,15 +1614,15 @@ def test_background_operation_cancellation(neon_env_builder: NeonEnvBuilder): ps_id_to_drain = env.pageservers[0].id - retryable_node_operation( + env.storage_controller.retryable_node_operation( lambda ps_id: env.storage_controller.node_drain(ps_id), ps_id_to_drain, max_attempts=3, backoff=2, ) - poll_node_status(env, ps_id_to_drain, "Draining", max_attempts=6, backoff=2) + env.storage_controller.poll_node_status(ps_id_to_drain, "Draining", max_attempts=6, backoff=2) env.storage_controller.cancel_node_drain(ps_id_to_drain) - poll_node_status(env, ps_id_to_drain, "Active", max_attempts=6, backoff=2) + env.storage_controller.poll_node_status(ps_id_to_drain, "Active", max_attempts=6, backoff=2) From e03c3c9893acbc6052184a5be8cc6b9f893a4d4e Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Thu, 4 Jul 2024 09:03:03 +0100 Subject: [PATCH 033/194] proxy: cache certain non-retriable console errors for a short time (#8201) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem If there's a quota error, it makes sense to cache it for a short window of time. Many clients do not handle database connection errors gracefully, so just spam retry 🤡 ## Summary of changes Updates the node_info cache to support storing console errors. Store console errors if they cannot be retried (using our own heuristic. should only trigger for quota exceeded errors). --- proxy/src/cache/common.rs | 7 +++ proxy/src/cache/timed_lru.rs | 38 ++++++++++++- proxy/src/console/messages.rs | 62 +++++++++++---------- proxy/src/console/provider.rs | 6 +- proxy/src/console/provider/neon.rs | 89 ++++++++++++++++++++++-------- proxy/src/proxy/tests.rs | 4 +- 6 files changed, 146 insertions(+), 60 deletions(-) diff --git a/proxy/src/cache/common.rs b/proxy/src/cache/common.rs index bc1c37512bce..4e393fddb2aa 100644 --- a/proxy/src/cache/common.rs +++ b/proxy/src/cache/common.rs @@ -53,6 +53,13 @@ impl Cached { ) } + pub fn map(self, f: impl FnOnce(V) -> U) -> Cached { + Cached { + token: self.token, + value: f(self.value), + } + } + /// Drop this entry from a cache if it's still there. pub fn invalidate(self) -> V { if let Some((cache, info)) = &self.token { diff --git a/proxy/src/cache/timed_lru.rs b/proxy/src/cache/timed_lru.rs index 3b21381bb971..c5c4f6a1ed09 100644 --- a/proxy/src/cache/timed_lru.rs +++ b/proxy/src/cache/timed_lru.rs @@ -65,6 +65,8 @@ impl Cache for TimedLru { struct Entry { created_at: Instant, expires_at: Instant, + ttl: Duration, + update_ttl_on_retrieval: bool, value: T, } @@ -122,7 +124,6 @@ impl TimedLru { Q: Hash + Eq + ?Sized, { let now = Instant::now(); - let deadline = now.checked_add(self.ttl).expect("time overflow"); // Do costly things before taking the lock. let mut cache = self.cache.lock(); @@ -142,7 +143,8 @@ impl TimedLru { let (created_at, expires_at) = (entry.created_at, entry.expires_at); // Update the deadline and the entry's position in the LRU list. - if self.update_ttl_on_retrieval { + let deadline = now.checked_add(raw_entry.get().ttl).expect("time overflow"); + if raw_entry.get().update_ttl_on_retrieval { raw_entry.get_mut().expires_at = deadline; } raw_entry.to_back(); @@ -162,12 +164,27 @@ impl TimedLru { /// existed, return the previous value and its creation timestamp. #[tracing::instrument(level = "debug", fields(cache = self.name), skip_all)] fn insert_raw(&self, key: K, value: V) -> (Instant, Option) { + self.insert_raw_ttl(key, value, self.ttl, self.update_ttl_on_retrieval) + } + + /// Insert an entry to the cache. If an entry with the same key already + /// existed, return the previous value and its creation timestamp. + #[tracing::instrument(level = "debug", fields(cache = self.name), skip_all)] + fn insert_raw_ttl( + &self, + key: K, + value: V, + ttl: Duration, + update: bool, + ) -> (Instant, Option) { let created_at = Instant::now(); - let expires_at = created_at.checked_add(self.ttl).expect("time overflow"); + let expires_at = created_at.checked_add(ttl).expect("time overflow"); let entry = Entry { created_at, expires_at, + ttl, + update_ttl_on_retrieval: update, value, }; @@ -190,6 +207,21 @@ impl TimedLru { } impl TimedLru { + pub fn insert_ttl(&self, key: K, value: V, ttl: Duration) { + self.insert_raw_ttl(key, value, ttl, false); + } + + pub fn insert_unit(&self, key: K, value: V) -> (Option, Cached<&Self, ()>) { + let (created_at, old) = self.insert_raw(key.clone(), value); + + let cached = Cached { + token: Some((self, LookupInfo { created_at, key })), + value: (), + }; + + (old, cached) + } + pub fn insert(&self, key: K, value: V) -> (Option, Cached<&Self>) { let (created_at, old) = self.insert_raw(key.clone(), value.clone()); diff --git a/proxy/src/console/messages.rs b/proxy/src/console/messages.rs index d28d13ba692b..9abf24ab7ffa 100644 --- a/proxy/src/console/messages.rs +++ b/proxy/src/console/messages.rs @@ -9,7 +9,7 @@ use crate::proxy::retry::CouldRetry; /// Generic error response with human-readable description. /// Note that we can't always present it to user as is. -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Clone)] pub struct ConsoleError { pub error: Box, #[serde(skip)] @@ -82,41 +82,19 @@ impl CouldRetry for ConsoleError { .details .error_info .map_or(Reason::Unknown, |e| e.reason); - match reason { - // not a transitive error - Reason::RoleProtected => false, - // on retry, it will still not be found - Reason::ResourceNotFound - | Reason::ProjectNotFound - | Reason::EndpointNotFound - | Reason::BranchNotFound => false, - // we were asked to go away - Reason::RateLimitExceeded - | Reason::NonDefaultBranchComputeTimeExceeded - | Reason::ActiveTimeQuotaExceeded - | Reason::ComputeTimeQuotaExceeded - | Reason::WrittenDataQuotaExceeded - | Reason::DataTransferQuotaExceeded - | Reason::LogicalSizeQuotaExceeded => false, - // transitive error. control plane is currently busy - // but might be ready soon - Reason::RunningOperations => true, - Reason::ConcurrencyLimitReached => true, - Reason::LockAlreadyTaken => true, - // unknown error. better not retry it. - Reason::Unknown => false, - } + + reason.can_retry() } } -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Clone)] pub struct Status { pub code: Box, pub message: Box, pub details: Details, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Clone)] pub struct Details { pub error_info: Option, pub retry_info: Option, @@ -199,6 +177,34 @@ impl Reason { | Reason::BranchNotFound ) } + + pub fn can_retry(&self) -> bool { + match self { + // do not retry role protected errors + // not a transitive error + Reason::RoleProtected => false, + // on retry, it will still not be found + Reason::ResourceNotFound + | Reason::ProjectNotFound + | Reason::EndpointNotFound + | Reason::BranchNotFound => false, + // we were asked to go away + Reason::RateLimitExceeded + | Reason::NonDefaultBranchComputeTimeExceeded + | Reason::ActiveTimeQuotaExceeded + | Reason::ComputeTimeQuotaExceeded + | Reason::WrittenDataQuotaExceeded + | Reason::DataTransferQuotaExceeded + | Reason::LogicalSizeQuotaExceeded => false, + // transitive error. control plane is currently busy + // but might be ready soon + Reason::RunningOperations + | Reason::ConcurrencyLimitReached + | Reason::LockAlreadyTaken => true, + // unknown error. better not retry it. + Reason::Unknown => false, + } + } } #[derive(Copy, Clone, Debug, Deserialize)] @@ -206,7 +212,7 @@ pub struct RetryInfo { pub retry_delay_ms: u64, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Clone)] pub struct UserFacingMessage { pub message: Box, } diff --git a/proxy/src/console/provider.rs b/proxy/src/console/provider.rs index bec55a83435f..7a9637066fb1 100644 --- a/proxy/src/console/provider.rs +++ b/proxy/src/console/provider.rs @@ -2,7 +2,7 @@ pub mod mock; pub mod neon; -use super::messages::MetricsAuxInfo; +use super::messages::{ConsoleError, MetricsAuxInfo}; use crate::{ auth::{ backend::{ComputeCredentialKeys, ComputeUserInfo}, @@ -317,8 +317,8 @@ impl NodeInfo { } } -pub type NodeInfoCache = TimedLru; -pub type CachedNodeInfo = Cached<&'static NodeInfoCache>; +pub type NodeInfoCache = TimedLru>>; +pub type CachedNodeInfo = Cached<&'static NodeInfoCache, NodeInfo>; pub type CachedRoleSecret = Cached<&'static ProjectInfoCacheImpl, Option>; pub type CachedAllowedIps = Cached<&'static ProjectInfoCacheImpl, Arc>>; diff --git a/proxy/src/console/provider/neon.rs b/proxy/src/console/provider/neon.rs index 41bd2f49567e..a6e67be22f13 100644 --- a/proxy/src/console/provider/neon.rs +++ b/proxy/src/console/provider/neon.rs @@ -9,7 +9,7 @@ use super::{ use crate::{ auth::backend::ComputeUserInfo, compute, - console::messages::ColdStartInfo, + console::messages::{ColdStartInfo, Reason}, http, metrics::{CacheOutcome, Metrics}, rate_limiter::EndpointRateLimiter, @@ -17,10 +17,10 @@ use crate::{ }; use crate::{cache::Cached, context::RequestMonitoring}; use futures::TryFutureExt; -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; use tokio::time::Instant; use tokio_postgres::config::SslMode; -use tracing::{error, info, info_span, warn, Instrument}; +use tracing::{debug, error, info, info_span, warn, Instrument}; pub struct Api { endpoint: http::Endpoint, @@ -273,26 +273,34 @@ impl super::Api for Api { ) -> Result { let key = user_info.endpoint_cache_key(); + macro_rules! check_cache { + () => { + if let Some(cached) = self.caches.node_info.get(&key) { + let (cached, info) = cached.take_value(); + let info = info.map_err(|c| { + info!(key = &*key, "found cached wake_compute error"); + WakeComputeError::ApiError(ApiError::Console(*c)) + })?; + + debug!(key = &*key, "found cached compute node info"); + ctx.set_project(info.aux.clone()); + return Ok(cached.map(|()| info)); + } + }; + } + // Every time we do a wakeup http request, the compute node will stay up // for some time (highly depends on the console's scale-to-zero policy); // The connection info remains the same during that period of time, // which means that we might cache it to reduce the load and latency. - if let Some(cached) = self.caches.node_info.get(&key) { - info!(key = &*key, "found cached compute node info"); - ctx.set_project(cached.aux.clone()); - return Ok(cached); - } + check_cache!(); let permit = self.locks.get_permit(&key).await?; // after getting back a permit - it's possible the cache was filled // double check if permit.should_check_cache() { - if let Some(cached) = self.caches.node_info.get(&key) { - info!(key = &*key, "found cached compute node info"); - ctx.set_project(cached.aux.clone()); - return Ok(cached); - } + check_cache!(); } // check rate limit @@ -300,23 +308,56 @@ impl super::Api for Api { .wake_compute_endpoint_rate_limiter .check(user_info.endpoint.normalize_intern(), 1) { - info!(key = &*key, "found cached compute node info"); return Err(WakeComputeError::TooManyConnections); } - let mut node = permit.release_result(self.do_wake_compute(ctx, user_info).await)?; - ctx.set_project(node.aux.clone()); - let cold_start_info = node.aux.cold_start_info; - info!("woken up a compute node"); + let node = permit.release_result(self.do_wake_compute(ctx, user_info).await); + match node { + Ok(node) => { + ctx.set_project(node.aux.clone()); + debug!(key = &*key, "created a cache entry for woken compute node"); - // store the cached node as 'warm' - node.aux.cold_start_info = ColdStartInfo::WarmCached; - let (_, mut cached) = self.caches.node_info.insert(key.clone(), node); - cached.aux.cold_start_info = cold_start_info; + let mut stored_node = node.clone(); + // store the cached node as 'warm_cached' + stored_node.aux.cold_start_info = ColdStartInfo::WarmCached; - info!(key = &*key, "created a cache entry for compute node info"); + let (_, cached) = self.caches.node_info.insert_unit(key, Ok(stored_node)); - Ok(cached) + Ok(cached.map(|()| node)) + } + Err(err) => match err { + WakeComputeError::ApiError(ApiError::Console(err)) => { + let Some(status) = &err.status else { + return Err(WakeComputeError::ApiError(ApiError::Console(err))); + }; + + let reason = status + .details + .error_info + .map_or(Reason::Unknown, |x| x.reason); + + // if we can retry this error, do not cache it. + if reason.can_retry() { + return Err(WakeComputeError::ApiError(ApiError::Console(err))); + } + + // at this point, we should only have quota errors. + debug!( + key = &*key, + "created a cache entry for the wake compute error" + ); + + self.caches.node_info.insert_ttl( + key, + Err(Box::new(err.clone())), + Duration::from_secs(30), + ); + + Err(WakeComputeError::ApiError(ApiError::Console(err))) + } + err => return Err(err), + }, + } } } diff --git a/proxy/src/proxy/tests.rs b/proxy/src/proxy/tests.rs index 8119f39fae6b..5186a9e1b0f1 100644 --- a/proxy/src/proxy/tests.rs +++ b/proxy/src/proxy/tests.rs @@ -540,8 +540,8 @@ fn helper_create_cached_node_info(cache: &'static NodeInfoCache) -> CachedNodeIn }, allow_self_signed_compute: false, }; - let (_, node) = cache.insert("key".into(), node); - node + let (_, node2) = cache.insert_unit("key".into(), Ok(node.clone())); + node2.map(|()| node) } fn helper_create_connect_info( From 5b69b32dc5fa1500fda12e53471809d5e6082f6f Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Thu, 4 Jul 2024 09:20:01 +0100 Subject: [PATCH 034/194] CI(build-and-test): add conclusion job (#8246) ## Problem Currently, if you need to rename a job and the job is listed in [branch protection rules](https://github.com/neondatabase/neon/settings/branch_protection_rules), the PR won't be allowed to merge. ## Summary of changes - Add `conclusion` job that fails if any of its dependencies don't finish successfully --- .github/workflows/build_and_test.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 5ac8c6ec2744..9b75d0bf3c47 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1368,3 +1368,31 @@ jobs: with: from-tag: ${{ needs.build-build-tools-image.outputs.image-tag }} secrets: inherit + + # This job simplifies setting branch protection rules (in GitHub UI) + # by allowing to set only this job instead of listing many others. + # It also makes it easier to rename or parametrise jobs (using matrix) + # which requires changes in branch protection rules + # + # Note, that we can't add external check (like `neon-cloud-e2e`) we still need to use GitHub UI for that. + # + # https://github.com/neondatabase/neon/settings/branch_protection_rules + conclusion: + if: always() + # Format `needs` differently to make the list more readable. + # Usually we do `needs: [...]` + needs: + - check-codestyle-python + - check-codestyle-rust + - regress-tests + - test-images + runs-on: ubuntu-22.04 + steps: + # The list of possible results: + # https://docs.github.com/en/actions/learn-github-actions/contexts#needs-context + - name: Fail the job if any of the dependencies do not succeed + run: exit 1 + if: | + contains(needs.*.result, 'failure') + || contains(needs.*.result, 'cancelled') + || contains(needs.*.result, 'skipped') From a46253766bf59d65c0b24f1e626787316e23ca80 Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Thu, 4 Jul 2024 13:22:33 +0100 Subject: [PATCH 035/194] pageserver: increase rate limit duration for layer visit log (#8263) ## Problem I'd like to keep this in the tree since it might be useful in prod as well. It's a bit too noisy as is and missing the lsn. ## Summary of changes Add an lsn field and and increase the rate limit duration. --- pageserver/src/tenant/timeline.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 54bbdef56e56..bbf0d0a4bf68 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -1284,15 +1284,14 @@ impl Timeline { if avg >= Self::VEC_GET_LAYERS_VISITED_WARN_THRESH { use utils::rate_limit::RateLimit; static LOGGED: Lazy> = - Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10)))); + Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(60)))); let mut rate_limit = LOGGED.lock().unwrap(); rate_limit.call(|| { tracing::info!( - tenant_id = %self.tenant_shard_id.tenant_id, - shard_id = %self.tenant_shard_id.shard_slug(), - timeline_id = %self.timeline_id, - "Vectored read for {} visited {} layers on average per key and {} in total. {}/{} pages were returned", - keyspace, avg, layers_visited, results.len(), keyspace.total_raw_size()); + shard_id = %self.tenant_shard_id.shard_slug(), + lsn = %lsn, + "Vectored read for {} visited {} layers on average per key and {} in total. {}/{} pages were returned", + keyspace, avg, layers_visited, results.len(), keyspace.total_raw_size()); }); } From a004d27fcae6b263a0878b24794514e8f5273dac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Thu, 4 Jul 2024 15:04:08 +0200 Subject: [PATCH 036/194] Use bool param for round_trip_test_compressed (#8252) As per @koivunej 's request in https://github.com/neondatabase/neon/pull/8238#discussion_r1663892091 , use a runtime param instead of monomorphizing the function based on the value. Part of https://github.com/neondatabase/neon/issues/5431 --- pageserver/src/tenant/blob_io.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pageserver/src/tenant/blob_io.rs b/pageserver/src/tenant/blob_io.rs index de74066b81bc..1a6a5702f19b 100644 --- a/pageserver/src/tenant/blob_io.rs +++ b/pageserver/src/tenant/blob_io.rs @@ -393,11 +393,12 @@ mod tests { use rand::{Rng, SeedableRng}; async fn round_trip_test(blobs: &[Vec]) -> Result<(), Error> { - round_trip_test_compressed::(blobs).await + round_trip_test_compressed::(blobs, false).await } - async fn round_trip_test_compressed( + async fn round_trip_test_compressed( blobs: &[Vec], + compression: bool, ) -> Result<(), Error> { let temp_dir = camino_tempfile::tempdir()?; let pathbuf = temp_dir.path().join("file"); @@ -409,7 +410,7 @@ mod tests { let file = VirtualFile::create(pathbuf.as_path(), &ctx).await?; let mut wtr = BlobWriter::::new(file, 0); for blob in blobs.iter() { - let (_, res) = if COMPRESSION { + let (_, res) = if compression { wtr.write_blob_maybe_compressed( blob.clone(), &ctx, @@ -432,7 +433,7 @@ mod tests { let file = VirtualFile::open(pathbuf.as_path(), &ctx).await?; let rdr = BlockReaderRef::VirtualFile(&file); - let rdr = BlockCursor::new_with_compression(rdr, COMPRESSION); + let rdr = BlockCursor::new_with_compression(rdr, compression); for (idx, (blob, offset)) in blobs.iter().zip(offsets.iter()).enumerate() { let blob_read = rdr.read_blob(*offset, &ctx).await?; assert_eq!( @@ -466,8 +467,8 @@ mod tests { ]; round_trip_test::(blobs).await?; round_trip_test::(blobs).await?; - round_trip_test_compressed::(blobs).await?; - round_trip_test_compressed::(blobs).await?; + round_trip_test_compressed::(blobs, true).await?; + round_trip_test_compressed::(blobs, true).await?; Ok(()) } @@ -483,8 +484,8 @@ mod tests { ]; round_trip_test::(blobs).await?; round_trip_test::(blobs).await?; - round_trip_test_compressed::(blobs).await?; - round_trip_test_compressed::(blobs).await?; + round_trip_test_compressed::(blobs, true).await?; + round_trip_test_compressed::(blobs, true).await?; Ok(()) } From bf9fc7706190ecd7cbd04fc56864086ced717327 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Thu, 4 Jul 2024 14:58:01 +0100 Subject: [PATCH 037/194] CI(pg-clients): unify workflow with build-and-test (#8160) ## Problem `pg-clients` workflow looks different from the main `build-and-test` workflow for historical reasons (it was my very first task at Neon, and back then I wasn't really familiar with the rest of the CI pipelines). This PR unifies `pg-clients` workflow with `build-and-test` ## Summary of changes - Rename `pg_clients.yml` to `pg-clients.yml` - Run the workflow on changes in relevant files - Create Allure report for tests - Send slack notifications to `#on-call-qa-staging-stream` channel (instead of `#on-call-staging-stream`) - Update Client libraries once we're here --- .github/workflows/build_and_test.yml | 2 +- .github/workflows/pg-clients.yml | 115 ++++++++ .github/workflows/pg_clients.yml | 98 ------- Dockerfile.build-tools | 22 +- test_runner/pg_clients/java/jdbc/Dockerfile | 2 +- .../pg_clients/python/pg8000/requirements.txt | 2 +- .../pg_clients/rust/tokio-postgres/Cargo.lock | 273 +++++++++--------- .../pg_clients/rust/tokio-postgres/Cargo.toml | 4 +- .../pg_clients/rust/tokio-postgres/Dockerfile | 2 +- .../swift/PostgresClientKitExample/Dockerfile | 4 +- .../PostgresClientKitExample/Package.resolved | 12 +- .../PostgresClientKitExample/Package.swift | 2 +- .../swift/PostgresNIOExample/Dockerfile | 4 +- .../swift/PostgresNIOExample/Package.resolved | 25 +- .../swift/PostgresNIOExample/Package.swift | 4 +- .../typescript/postgresql-client/Dockerfile | 2 +- .../postgresql-client/package-lock.json | 12 +- .../typescript/postgresql-client/package.json | 2 +- .../typescript/serverless-driver/Dockerfile | 2 +- .../serverless-driver/package-lock.json | 134 +++++---- .../typescript/serverless-driver/package.json | 2 +- 21 files changed, 398 insertions(+), 327 deletions(-) create mode 100644 .github/workflows/pg-clients.yml delete mode 100644 .github/workflows/pg_clients.yml diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 9b75d0bf3c47..a3246987e2b8 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -30,7 +30,7 @@ jobs: if: ${{ !contains(github.event.pull_request.labels.*.name, 'run-no-ci') }} uses: ./.github/workflows/check-permissions.yml with: - github-event-name: ${{ github.event_name}} + github-event-name: ${{ github.event_name }} cancel-previous-e2e-tests: needs: [ check-permissions ] diff --git a/.github/workflows/pg-clients.yml b/.github/workflows/pg-clients.yml new file mode 100644 index 000000000000..e21e45c929c8 --- /dev/null +++ b/.github/workflows/pg-clients.yml @@ -0,0 +1,115 @@ +name: Test Postgres client libraries + +on: + schedule: + # * is a special character in YAML so you have to quote this string + # ┌───────────── minute (0 - 59) + # │ ┌───────────── hour (0 - 23) + # │ │ ┌───────────── day of the month (1 - 31) + # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) + # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) + - cron: '23 02 * * *' # run once a day, timezone is utc + pull_request: + paths: + - '.github/workflows/pg-clients.yml' + - 'test_runner/pg_clients/**' + - 'poetry.lock' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +defaults: + run: + shell: bash -euxo pipefail {0} + +env: + DEFAULT_PG_VERSION: 16 + PLATFORM: neon-captest-new + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }} + AWS_DEFAULT_REGION: eu-central-1 + +jobs: + check-permissions: + if: ${{ !contains(github.event.pull_request.labels.*.name, 'run-no-ci') }} + uses: ./.github/workflows/check-permissions.yml + with: + github-event-name: ${{ github.event_name }} + + check-build-tools-image: + needs: [ check-permissions ] + uses: ./.github/workflows/check-build-tools-image.yml + + build-build-tools-image: + needs: [ check-build-tools-image ] + uses: ./.github/workflows/build-build-tools-image.yml + with: + image-tag: ${{ needs.check-build-tools-image.outputs.image-tag }} + secrets: inherit + + test-postgres-client-libs: + needs: [ build-build-tools-image ] + runs-on: ubuntu-22.04 + + container: + image: ${{ needs.build-build-tools-image.outputs.image }} + credentials: + username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} + password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + options: --init --user root + + steps: + - uses: actions/checkout@v4 + + - name: Download Neon artifact + uses: ./.github/actions/download + with: + name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact + path: /tmp/neon/ + prefix: latest + + - name: Create Neon Project + id: create-neon-project + uses: ./.github/actions/neon-project-create + with: + api_key: ${{ secrets.NEON_STAGING_API_KEY }} + postgres_version: ${{ env.DEFAULT_PG_VERSION }} + + - name: Run tests + uses: ./.github/actions/run-python-test-set + with: + build_type: remote + test_selection: pg_clients + run_in_parallel: false + extra_params: -m remote_cluster + pg_version: ${{ env.DEFAULT_PG_VERSION }} + env: + BENCHMARK_CONNSTR: ${{ steps.create-neon-project.outputs.dsn }} + + - name: Delete Neon Project + if: always() + uses: ./.github/actions/neon-project-delete + with: + project_id: ${{ steps.create-neon-project.outputs.project_id }} + api_key: ${{ secrets.NEON_STAGING_API_KEY }} + + - name: Create Allure report + if: ${{ !cancelled() }} + id: create-allure-report + uses: ./.github/actions/allure-report-generate + with: + store-test-results-into-db: true + env: + REGRESS_TEST_RESULT_CONNSTR_NEW: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }} + + - name: Post to a Slack channel + if: github.event.schedule && failure() + uses: slackapi/slack-github-action@v1 + with: + channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream + slack-message: | + Testing Postgres clients: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ job.status }}> (<${{ steps.create-allure-report.outputs.report-url }}|test report>) + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.github/workflows/pg_clients.yml b/.github/workflows/pg_clients.yml deleted file mode 100644 index dd09abddb848..000000000000 --- a/.github/workflows/pg_clients.yml +++ /dev/null @@ -1,98 +0,0 @@ -name: Test Postgres client libraries - -on: - schedule: - # * is a special character in YAML so you have to quote this string - # ┌───────────── minute (0 - 59) - # │ ┌───────────── hour (0 - 23) - # │ │ ┌───────────── day of the month (1 - 31) - # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) - # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) - - cron: '23 02 * * *' # run once a day, timezone is utc - - workflow_dispatch: - -concurrency: - # Allow only one workflow per any non-`main` branch. - group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }} - cancel-in-progress: true - -jobs: - test-postgres-client-libs: - # TODO: switch to gen2 runner, requires docker - runs-on: ubuntu-22.04 - - env: - DEFAULT_PG_VERSION: 14 - TEST_OUTPUT: /tmp/test_output - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - - name: Install Poetry - uses: snok/install-poetry@v1 - - - name: Cache poetry deps - uses: actions/cache@v4 - with: - path: ~/.cache/pypoetry/virtualenvs - key: v2-${{ runner.os }}-${{ runner.arch }}-python-deps-ubunutu-latest-${{ hashFiles('poetry.lock') }} - - - name: Install Python deps - shell: bash -euxo pipefail {0} - run: ./scripts/pysync - - - name: Create Neon Project - id: create-neon-project - uses: ./.github/actions/neon-project-create - with: - api_key: ${{ secrets.NEON_STAGING_API_KEY }} - postgres_version: ${{ env.DEFAULT_PG_VERSION }} - - - name: Run pytest - env: - REMOTE_ENV: 1 - BENCHMARK_CONNSTR: ${{ steps.create-neon-project.outputs.dsn }} - POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install - shell: bash -euxo pipefail {0} - run: | - # Test framework expects we have psql binary; - # but since we don't really need it in this test, let's mock it - mkdir -p "$POSTGRES_DISTRIB_DIR/v${DEFAULT_PG_VERSION}/bin" && touch "$POSTGRES_DISTRIB_DIR/v${DEFAULT_PG_VERSION}/bin/psql"; - ./scripts/pytest \ - --junitxml=$TEST_OUTPUT/junit.xml \ - --tb=short \ - --verbose \ - -m "remote_cluster" \ - -rA "test_runner/pg_clients" - - - name: Delete Neon Project - if: ${{ always() }} - uses: ./.github/actions/neon-project-delete - with: - project_id: ${{ steps.create-neon-project.outputs.project_id }} - api_key: ${{ secrets.NEON_STAGING_API_KEY }} - - # We use GitHub's action upload-artifact because `ubuntu-latest` doesn't have configured AWS CLI. - # It will be fixed after switching to gen2 runner - - name: Upload python test logs - if: always() - uses: actions/upload-artifact@v4 - with: - retention-days: 7 - name: python-test-pg_clients-${{ runner.os }}-${{ runner.arch }}-stage-logs - path: ${{ env.TEST_OUTPUT }} - - - name: Post to a Slack channel - if: ${{ github.event.schedule && failure() }} - uses: slackapi/slack-github-action@v1 - with: - channel-id: "C033QLM5P7D" # dev-staging-stream - slack-message: "Testing Postgres clients: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/Dockerfile.build-tools b/Dockerfile.build-tools index 30314376efdb..4826b7914e42 100644 --- a/Dockerfile.build-tools +++ b/Dockerfile.build-tools @@ -1,5 +1,13 @@ FROM debian:bullseye-slim +# Use ARG as a build-time environment variable here to allow. +# It's not supposed to be set outside. +# Alternatively it can be obtained using the following command +# ``` +# . /etc/os-release && echo "${VERSION_CODENAME}" +# ``` +ARG DEBIAN_VERSION_CODENAME=bullseye + # Add nonroot user RUN useradd -ms /bin/bash nonroot -b /home SHELL ["/bin/bash", "-c"] @@ -66,12 +74,24 @@ RUN curl -sL "https://github.com/peak/s5cmd/releases/download/v${S5CMD_VERSION}/ # LLVM ENV LLVM_VERSION=18 RUN curl -fsSL 'https://apt.llvm.org/llvm-snapshot.gpg.key' | apt-key add - \ - && echo "deb http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-${LLVM_VERSION} main" > /etc/apt/sources.list.d/llvm.stable.list \ + && echo "deb http://apt.llvm.org/${DEBIAN_VERSION_CODENAME}/ llvm-toolchain-${DEBIAN_VERSION_CODENAME}-${LLVM_VERSION} main" > /etc/apt/sources.list.d/llvm.stable.list \ && apt update \ && apt install -y clang-${LLVM_VERSION} llvm-${LLVM_VERSION} \ && bash -c 'for f in /usr/bin/clang*-${LLVM_VERSION} /usr/bin/llvm*-${LLVM_VERSION}; do ln -s "${f}" "${f%-${LLVM_VERSION}}"; done' \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +# Install docker +RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian ${DEBIAN_VERSION_CODENAME} stable" > /etc/apt/sources.list.d/docker.list \ + && apt update \ + && apt install -y docker-ce docker-ce-cli \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Configure sudo & docker +RUN usermod -aG sudo nonroot && \ + echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \ + usermod -aG docker nonroot + # AWS CLI RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "awscliv2.zip" \ && unzip -q awscliv2.zip \ diff --git a/test_runner/pg_clients/java/jdbc/Dockerfile b/test_runner/pg_clients/java/jdbc/Dockerfile index 7e074e07b836..7c2b1b40e091 100644 --- a/test_runner/pg_clients/java/jdbc/Dockerfile +++ b/test_runner/pg_clients/java/jdbc/Dockerfile @@ -1,4 +1,4 @@ -FROM openjdk:21 +FROM openjdk:22 WORKDIR /source COPY . . diff --git a/test_runner/pg_clients/python/pg8000/requirements.txt b/test_runner/pg_clients/python/pg8000/requirements.txt index e086a937e68b..099a4ade2c4d 100644 --- a/test_runner/pg_clients/python/pg8000/requirements.txt +++ b/test_runner/pg_clients/python/pg8000/requirements.txt @@ -1,2 +1,2 @@ -pg8000==1.30.5 +pg8000==1.31.2 scramp>=1.4.3 diff --git a/test_runner/pg_clients/rust/tokio-postgres/Cargo.lock b/test_runner/pg_clients/rust/tokio-postgres/Cargo.lock index a4a2426b97ec..32c1c52eea44 100644 --- a/test_runner/pg_clients/rust/tokio-postgres/Cargo.lock +++ b/test_runner/pg_clients/rust/tokio-postgres/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" dependencies = [ "gimli", ] @@ -19,9 +19,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "async-trait" -version = "0.1.77" +version = "0.1.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", @@ -30,15 +30,15 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "backtrace" -version = "0.3.69" +version = "0.3.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" dependencies = [ "addr2line", "cc", @@ -63,9 +63,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.2" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "block-buffer" @@ -78,9 +78,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.15.3" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea184aa71bb362a1157c896979544cc23974e08fd265f29ea96b59f0b4a555b" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "byteorder" @@ -90,15 +90,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "cc" -version = "1.0.89" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ba8f7aaa012f30d5b2861462f6708eccd49c3c39863fe083a308035f63d723" +checksum = "ac367972e516d45567c7eafc73d24e1c193dcf200a8d94e9db7b3d38b349572d" [[package]] name = "cfg-if" @@ -154,9 +154,9 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", "windows-sys 0.52.0", @@ -170,15 +170,9 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] name = "fastrand" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" - -[[package]] -name = "finl_unicode" -version = "1.2.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "foreign-types" @@ -296,9 +290,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.12" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", @@ -307,9 +301,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" [[package]] name = "hmac" @@ -329,29 +323,23 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - [[package]] name = "libc" -version = "0.2.153" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "linux-raw-sys" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -375,15 +363,15 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.1" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "miniz_oxide" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" dependencies = [ "adler", ] @@ -401,11 +389,10 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" dependencies = [ - "lazy_static", "libc", "log", "openssl", @@ -419,9 +406,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.2" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434" dependencies = [ "memchr", ] @@ -438,7 +425,7 @@ version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.6.0", "cfg-if", "foreign-types", "libc", @@ -466,9 +453,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.101" +version = "0.9.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff" +checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" dependencies = [ "cc", "libc", @@ -478,9 +465,9 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -488,15 +475,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.2", "smallvec", - "windows-targets 0.48.5", + "windows-targets 0.52.5", ] [[package]] @@ -525,9 +512,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pin-utils" @@ -591,18 +578,18 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.78" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -646,6 +633,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" +dependencies = [ + "bitflags 2.6.0", +] + [[package]] name = "rust-neon-example" version = "0.1.0" @@ -658,17 +654,17 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustix" -version = "0.38.31" +version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", @@ -692,11 +688,11 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "security-framework" -version = "2.9.2" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.6.0", "core-foundation", "core-foundation-sys", "libc", @@ -705,9 +701,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.9.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" dependencies = [ "core-foundation-sys", "libc", @@ -741,15 +737,15 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.13.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", "windows-sys 0.52.0", @@ -757,26 +753,26 @@ dependencies = [ [[package]] name = "stringprep" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" dependencies = [ - "finl_unicode", "unicode-bidi", "unicode-normalization", + "unicode-properties", ] [[package]] name = "subtle" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.52" +version = "2.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07" +checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" dependencies = [ "proc-macro2", "quote", @@ -797,9 +793,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "c55115c6fbe2d2bef26eb09ad74bde02d8255476fc0c7b515ef09fbb35742d82" dependencies = [ "tinyvec_macros", ] @@ -812,9 +808,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.36.0" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", "bytes", @@ -828,9 +824,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", @@ -875,35 +871,15 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ "bytes", "futures-core", "futures-sink", "pin-project-lite", "tokio", - "tracing", -] - -[[package]] -name = "tracing" -version = "0.1.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" -dependencies = [ - "pin-project-lite", - "tracing-core", -] - -[[package]] -name = "tracing-core" -version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" -dependencies = [ - "once_cell", ] [[package]] @@ -933,6 +909,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4259d9d4425d9f0661581b804cb85fe66a4c631cadd8f490d1c13a35d5d9291" + [[package]] name = "vcpkg" version = "0.2.15" @@ -1023,11 +1005,11 @@ dependencies = [ [[package]] name = "whoami" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fec781d48b41f8163426ed18e8fc2864c12937df9ce54c88ede7bd47270893e" +checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" dependencies = [ - "redox_syscall", + "redox_syscall 0.4.1", "wasite", "web-sys", ] @@ -1047,7 +1029,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.4", + "windows-targets 0.52.5", ] [[package]] @@ -1067,17 +1049,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm 0.52.4", - "windows_aarch64_msvc 0.52.4", - "windows_i686_gnu 0.52.4", - "windows_i686_msvc 0.52.4", - "windows_x86_64_gnu 0.52.4", - "windows_x86_64_gnullvm 0.52.4", - "windows_x86_64_msvc 0.52.4", + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", ] [[package]] @@ -1088,9 +1071,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" [[package]] name = "windows_aarch64_msvc" @@ -1100,9 +1083,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" [[package]] name = "windows_i686_gnu" @@ -1112,9 +1095,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.4" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" [[package]] name = "windows_i686_msvc" @@ -1124,9 +1113,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" [[package]] name = "windows_x86_64_gnu" @@ -1136,9 +1125,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" [[package]] name = "windows_x86_64_gnullvm" @@ -1148,9 +1137,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" [[package]] name = "windows_x86_64_msvc" @@ -1160,6 +1149,6 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" diff --git a/test_runner/pg_clients/rust/tokio-postgres/Cargo.toml b/test_runner/pg_clients/rust/tokio-postgres/Cargo.toml index 0f420e5b0643..27d01810bd52 100644 --- a/test_runner/pg_clients/rust/tokio-postgres/Cargo.toml +++ b/test_runner/pg_clients/rust/tokio-postgres/Cargo.toml @@ -7,9 +7,9 @@ publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -native-tls = "0.2.11" +native-tls = "0.2.12" postgres-native-tls = "0.5.0" -tokio = { version = "1.36", features=["rt", "macros"] } +tokio = { version = "1.38", features=["rt", "macros"] } tokio-postgres = "0.7.10" diff --git a/test_runner/pg_clients/rust/tokio-postgres/Dockerfile b/test_runner/pg_clients/rust/tokio-postgres/Dockerfile index 8611e66cbb67..3e214de785b3 100644 --- a/test_runner/pg_clients/rust/tokio-postgres/Dockerfile +++ b/test_runner/pg_clients/rust/tokio-postgres/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.76 +FROM rust:1.79 WORKDIR /source COPY . . diff --git a/test_runner/pg_clients/swift/PostgresClientKitExample/Dockerfile b/test_runner/pg_clients/swift/PostgresClientKitExample/Dockerfile index 04028388207c..6006e61ee22e 100644 --- a/test_runner/pg_clients/swift/PostgresClientKitExample/Dockerfile +++ b/test_runner/pg_clients/swift/PostgresClientKitExample/Dockerfile @@ -1,11 +1,11 @@ -FROM swift:5.9 AS build +FROM swift:5.10 AS build RUN apt-get -q update && apt-get -q install -y libssl-dev WORKDIR /source COPY . . RUN swift build --configuration release -FROM swift:5.9 +FROM swift:5.10 WORKDIR /app COPY --from=build /source/.build/release . CMD ["/app/PostgresClientKitExample"] diff --git a/test_runner/pg_clients/swift/PostgresClientKitExample/Package.resolved b/test_runner/pg_clients/swift/PostgresClientKitExample/Package.resolved index 767443a9ddcc..6e8613095f15 100644 --- a/test_runner/pg_clients/swift/PostgresClientKitExample/Package.resolved +++ b/test_runner/pg_clients/swift/PostgresClientKitExample/Package.resolved @@ -1,4 +1,5 @@ { + "originHash" : "8eff8c577ba246ce7824d3434839acefced2b1a1d2b1ad700554502538a50558", "pins" : [ { "identity" : "bluesocket", @@ -18,15 +19,6 @@ "version" : "2.0.2" } }, - { - "identity" : "openssl", - "kind" : "remoteSourceControl", - "location" : "https://github.com/Kitura/OpenSSL.git", - "state" : { - "revision" : "5dc8cb4f971135c17343e3c6df4f28904a0600e2", - "version" : "2.3.1" - } - }, { "identity" : "postgresclientkit", "kind" : "remoteSourceControl", @@ -37,5 +29,5 @@ } } ], - "version" : 2 + "version" : 3 } diff --git a/test_runner/pg_clients/swift/PostgresClientKitExample/Package.swift b/test_runner/pg_clients/swift/PostgresClientKitExample/Package.swift index 48320dd02314..a66d09c542f9 100644 --- a/test_runner/pg_clients/swift/PostgresClientKitExample/Package.swift +++ b/test_runner/pg_clients/swift/PostgresClientKitExample/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version:5.8 +// swift-tools-version:5.10 import PackageDescription let package = Package( diff --git a/test_runner/pg_clients/swift/PostgresNIOExample/Dockerfile b/test_runner/pg_clients/swift/PostgresNIOExample/Dockerfile index 9130e0973f8e..d6815fbb5fa2 100644 --- a/test_runner/pg_clients/swift/PostgresNIOExample/Dockerfile +++ b/test_runner/pg_clients/swift/PostgresNIOExample/Dockerfile @@ -1,10 +1,10 @@ -FROM swift:5.9 AS build +FROM swift:5.10 AS build WORKDIR /source COPY . . RUN swift build --configuration release -FROM swift:5.9 +FROM swift:5.10 WORKDIR /app COPY --from=build /source/.build/release . CMD ["/app/PostgresNIOExample"] diff --git a/test_runner/pg_clients/swift/PostgresNIOExample/Package.resolved b/test_runner/pg_clients/swift/PostgresNIOExample/Package.resolved index 023e03a7b1a0..0e5dfdafcb0a 100644 --- a/test_runner/pg_clients/swift/PostgresNIOExample/Package.resolved +++ b/test_runner/pg_clients/swift/PostgresNIOExample/Package.resolved @@ -1,12 +1,22 @@ { + "originHash" : "11b5dcece349a3e56a7a9a7d0af6d0f5b83dff321b43124a01b158ed7aac5302", "pins" : [ { "identity" : "postgres-nio", "kind" : "remoteSourceControl", "location" : "https://github.com/vapor/postgres-nio.git", "state" : { - "revision" : "69ccfdf4c80144d845e3b439961b7ec6cd7ae33f", - "version" : "1.20.2" + "revision" : "5c268768890b062803a49f1358becc478f954265", + "version" : "1.21.5" + } + }, + { + "identity" : "swift-async-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-async-algorithms.git", + "state" : { + "revision" : "da4e36f86544cdf733a40d59b3a2267e3a7bbf36", + "version" : "1.0.0" } }, { @@ -81,6 +91,15 @@ "version" : "1.20.1" } }, + { + "identity" : "swift-service-lifecycle", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swift-server/swift-service-lifecycle.git", + "state" : { + "revision" : "d58e6bf2b1ae2884cf204a8b5bcaaa7aae3c1ff0", + "version" : "2.6.0" + } + }, { "identity" : "swift-system", "kind" : "remoteSourceControl", @@ -91,5 +110,5 @@ } } ], - "version" : 2 + "version" : 3 } diff --git a/test_runner/pg_clients/swift/PostgresNIOExample/Package.swift b/test_runner/pg_clients/swift/PostgresNIOExample/Package.swift index 637eb4bc9ddb..20bb10f76c37 100644 --- a/test_runner/pg_clients/swift/PostgresNIOExample/Package.swift +++ b/test_runner/pg_clients/swift/PostgresNIOExample/Package.swift @@ -1,10 +1,10 @@ -// swift-tools-version:5.9 +// swift-tools-version:5.10 import PackageDescription let package = Package( name: "PostgresNIOExample", dependencies: [ - .package(url: "https://github.com/vapor/postgres-nio.git", from: "1.20.2") + .package(url: "https://github.com/vapor/postgres-nio.git", from: "1.21.5") ], targets: [ .executableTarget( diff --git a/test_runner/pg_clients/typescript/postgresql-client/Dockerfile b/test_runner/pg_clients/typescript/postgresql-client/Dockerfile index 004b383749f9..45e8753f7eec 100644 --- a/test_runner/pg_clients/typescript/postgresql-client/Dockerfile +++ b/test_runner/pg_clients/typescript/postgresql-client/Dockerfile @@ -1,4 +1,4 @@ -FROM node:21 +FROM node:22 WORKDIR /source COPY . . diff --git a/test_runner/pg_clients/typescript/postgresql-client/package-lock.json b/test_runner/pg_clients/typescript/postgresql-client/package-lock.json index b4f8587eacef..19311808b6b8 100644 --- a/test_runner/pg_clients/typescript/postgresql-client/package-lock.json +++ b/test_runner/pg_clients/typescript/postgresql-client/package-lock.json @@ -5,7 +5,7 @@ "packages": { "": { "dependencies": { - "postgresql-client": "2.10.5" + "postgresql-client": "2.11.0" } }, "node_modules/doublylinked": { @@ -42,9 +42,10 @@ } }, "node_modules/postgresql-client": { - "version": "2.10.5", - "resolved": "https://registry.npmjs.org/postgresql-client/-/postgresql-client-2.10.5.tgz", - "integrity": "sha512-R3EC16pUdbgrzk1J2MQLj7jY2TepWurJHoK90nOeLZj1XTpL/+wL1VCneTmclRVKDuKVjFHr+FASV47KrLpAbw==", + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/postgresql-client/-/postgresql-client-2.11.0.tgz", + "integrity": "sha512-QSPHcWVaiBG+JyASaDojOXvhRmsc2n8j2COdIjUDENFAtFls16Zy240asY2ENzZRQJUMAA8vpR8w4SAdI8jdbw==", + "license": "MIT", "dependencies": { "doublylinked": "^2.5.4", "lightning-pool": "^4.2.2", @@ -55,8 +56,7 @@ "putil-varhelpers": "^1.6.5" }, "engines": { - "node": ">=16.0", - "npm": ">=7.0.0" + "node": ">=16.0" } }, "node_modules/power-tasks": { diff --git a/test_runner/pg_clients/typescript/postgresql-client/package.json b/test_runner/pg_clients/typescript/postgresql-client/package.json index 07ec100d0d22..d2bba23d2912 100644 --- a/test_runner/pg_clients/typescript/postgresql-client/package.json +++ b/test_runner/pg_clients/typescript/postgresql-client/package.json @@ -1,6 +1,6 @@ { "type": "module", "dependencies": { - "postgresql-client": "2.10.5" + "postgresql-client": "2.11.0" } } diff --git a/test_runner/pg_clients/typescript/serverless-driver/Dockerfile b/test_runner/pg_clients/typescript/serverless-driver/Dockerfile index 004b383749f9..45e8753f7eec 100644 --- a/test_runner/pg_clients/typescript/serverless-driver/Dockerfile +++ b/test_runner/pg_clients/typescript/serverless-driver/Dockerfile @@ -1,4 +1,4 @@ -FROM node:21 +FROM node:22 WORKDIR /source COPY . . diff --git a/test_runner/pg_clients/typescript/serverless-driver/package-lock.json b/test_runner/pg_clients/typescript/serverless-driver/package-lock.json index f3b456f1edc7..7f3f7f2e84e7 100644 --- a/test_runner/pg_clients/typescript/serverless-driver/package-lock.json +++ b/test_runner/pg_clients/typescript/serverless-driver/package-lock.json @@ -5,96 +5,138 @@ "packages": { "": { "dependencies": { - "@neondatabase/serverless": "0.9.0", + "@neondatabase/serverless": "0.9.4", "ws": "8.17.1" } }, "node_modules/@neondatabase/serverless": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/@neondatabase/serverless/-/serverless-0.9.0.tgz", - "integrity": "sha512-mmJnUAzlzvxNSZuuhI6kgJjH+JgFdBMYUWxihtq/nj0Tjt+Y5UU3W+SvRFoucnd5NObYkuLYQzk+zV5DGFKGJg==", + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/@neondatabase/serverless/-/serverless-0.9.4.tgz", + "integrity": "sha512-D0AXgJh6xkf+XTlsO7iwE2Q1w8981E1cLCPAALMU2YKtkF/1SF6BiAzYARZFYo175ON+b1RNIy9TdSFHm5nteg==", + "license": "MIT", "dependencies": { - "@types/pg": "8.6.6" + "@types/pg": "8.11.6" } }, "node_modules/@types/node": { - "version": "18.16.3", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.16.3.tgz", - "integrity": "sha512-OPs5WnnT1xkCBiuQrZA4+YAV4HEJejmHneyraIaxsbev5yCEr6KMwINNFP9wQeFIw8FWcoTqF3vQsa5CDaI+8Q==" + "version": "20.14.9", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.9.tgz", + "integrity": "sha512-06OCtnTXtWOZBJlRApleWndH4JsRVs1pDCc8dLSQp+7PpUpX3ePdHyeNSFTeSe7FtKyQkrlPvHwJOW3SLd8Oyg==", + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } }, "node_modules/@types/pg": { - "version": "8.6.6", - "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.6.6.tgz", - "integrity": "sha512-O2xNmXebtwVekJDD+02udOncjVcMZQuTEQEMpKJ0ZRf5E7/9JJX3izhKUcUifBkyKpljyUM6BTgy2trmviKlpw==", + "version": "8.11.6", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.11.6.tgz", + "integrity": "sha512-/2WmmBXHLsfRqzfHW7BNZ8SbYzE8OSk7i3WjFYvfgRHj7S1xj+16Je5fUKv3lVdVzk/zn9TXOqf+avFCFIE0yQ==", + "license": "MIT", "dependencies": { "@types/node": "*", "pg-protocol": "*", - "pg-types": "^2.2.0" + "pg-types": "^4.0.1" } }, + "node_modules/obuf": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz", + "integrity": "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==", + "license": "MIT" + }, "node_modules/pg-int8": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", "engines": { "node": ">=4.0.0" } }, + "node_modules/pg-numeric": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pg-numeric/-/pg-numeric-1.0.2.tgz", + "integrity": "sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==", + "license": "ISC", + "engines": { + "node": ">=4" + } + }, "node_modules/pg-protocol": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz", - "integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==" + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.1.tgz", + "integrity": "sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==", + "license": "MIT" }, "node_modules/pg-types": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", - "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-4.0.2.tgz", + "integrity": "sha512-cRL3JpS3lKMGsKaWndugWQoLOCoP+Cic8oseVcbr0qhPzYD5DWXK+RZ9LY9wxRf7RQia4SCwQlXk0q6FCPrVng==", + "license": "MIT", "dependencies": { "pg-int8": "1.0.1", - "postgres-array": "~2.0.0", - "postgres-bytea": "~1.0.0", - "postgres-date": "~1.0.4", - "postgres-interval": "^1.1.0" + "pg-numeric": "1.0.2", + "postgres-array": "~3.0.1", + "postgres-bytea": "~3.0.0", + "postgres-date": "~2.1.0", + "postgres-interval": "^3.0.0", + "postgres-range": "^1.1.1" }, "engines": { - "node": ">=4" + "node": ">=10" } }, "node_modules/postgres-array": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", - "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-3.0.2.tgz", + "integrity": "sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==", + "license": "MIT", "engines": { - "node": ">=4" + "node": ">=12" } }, "node_modules/postgres-bytea": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", - "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-3.0.0.tgz", + "integrity": "sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==", + "license": "MIT", + "dependencies": { + "obuf": "~1.1.2" + }, "engines": { - "node": ">=0.10.0" + "node": ">= 6" } }, "node_modules/postgres-date": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", - "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-2.1.0.tgz", + "integrity": "sha512-K7Juri8gtgXVcDfZttFKVmhglp7epKb1K4pgrkLxehjqkrgPhfG6OO8LHLkfaqkbpjNRnra018XwAr1yQFWGcA==", + "license": "MIT", "engines": { - "node": ">=0.10.0" + "node": ">=12" } }, "node_modules/postgres-interval": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", - "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", - "dependencies": { - "xtend": "^4.0.0" - }, + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-3.0.0.tgz", + "integrity": "sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==", + "license": "MIT", "engines": { - "node": ">=0.10.0" + "node": ">=12" } }, + "node_modules/postgres-range": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/postgres-range/-/postgres-range-1.1.4.tgz", + "integrity": "sha512-i/hbxIE9803Alj/6ytL7UHQxRvZkI9O4Sy+J3HGc4F4oo/2eQAjTSNJ0bfxyse3bH0nuVesCk+3IRLaMtG3H6w==", + "license": "MIT" + }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "license": "MIT" + }, "node_modules/ws": { "version": "8.17.1", "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz", @@ -114,14 +156,6 @@ "optional": true } } - }, - "node_modules/xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", - "engines": { - "node": ">=0.4" - } } } } diff --git a/test_runner/pg_clients/typescript/serverless-driver/package.json b/test_runner/pg_clients/typescript/serverless-driver/package.json index 3ae7a8a6cfcd..f791d184c5f8 100644 --- a/test_runner/pg_clients/typescript/serverless-driver/package.json +++ b/test_runner/pg_clients/typescript/serverless-driver/package.json @@ -1,7 +1,7 @@ { "type": "module", "dependencies": { - "@neondatabase/serverless": "0.9.0", + "@neondatabase/serverless": "0.9.4", "ws": "8.17.1" } } From c9e6dd45d343ffcb502023857a814e7500a6d3f3 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 4 Jul 2024 15:05:41 +0100 Subject: [PATCH 038/194] pageserver: downgrade stale generation messages to INFO (#8256) ## Problem When generations were new, these messages were an important way of noticing if something unexpected was going on. We found some real issues when investigating tests that unexpectedly tripped them. At time has gone on, this code is now pretty battle-tested, and as we do more live migrations etc, it's fairly normal to see the occasional message from a node with a stale generation. At this point the cognitive load on developers to selectively allow-list these logs outweighs the benefit of having them at warn severity. Closes: https://github.com/neondatabase/neon/issues/8080 ## Summary of changes - Downgrade "Dropped remote consistent LSN updates" and "Dropping stale deletions" messages to INFO - Remove all the allow-list entries for these logs. --- pageserver/src/deletion_queue/validator.rs | 4 ++-- test_runner/fixtures/pageserver/many_tenants.py | 4 ---- .../interactive/test_many_small_tenants.py | 4 ---- .../pagebench/test_large_slru_basebackup.py | 4 ---- ...server_max_throughput_getpage_at_latest_lsn.py | 4 ---- .../performance/test_storage_controller_scale.py | 8 -------- test_runner/regress/test_attach_tenant_config.py | 6 ------ test_runner/regress/test_change_pageserver.py | 5 ----- test_runner/regress/test_layers_from_future.py | 3 --- .../regress/test_pageserver_generations.py | 13 ------------- test_runner/regress/test_pageserver_secondary.py | 3 --- test_runner/regress/test_remote_storage.py | 7 ------- test_runner/regress/test_sharding.py | 4 ---- test_runner/regress/test_storage_controller.py | 15 --------------- test_runner/regress/test_tenant_conf.py | 4 ---- test_runner/regress/test_tenant_detach.py | 12 ------------ test_runner/regress/test_tenant_relocation.py | 2 -- test_runner/regress/test_tenants.py | 4 ---- 18 files changed, 2 insertions(+), 104 deletions(-) diff --git a/pageserver/src/deletion_queue/validator.rs b/pageserver/src/deletion_queue/validator.rs index bf06c78e673f..d215fd2b7d2d 100644 --- a/pageserver/src/deletion_queue/validator.rs +++ b/pageserver/src/deletion_queue/validator.rs @@ -190,7 +190,7 @@ where } } else { // If we failed validation, then do not apply any of the projected updates - warn!("Dropped remote consistent LSN updates for tenant {tenant_id} in stale generation {:?}", tenant_lsn_state.generation); + info!("Dropped remote consistent LSN updates for tenant {tenant_id} in stale generation {:?}", tenant_lsn_state.generation); metrics::DELETION_QUEUE.dropped_lsn_updates.inc(); } } @@ -225,7 +225,7 @@ where && (tenant.generation == *validated_generation); if !this_list_valid { - warn!("Dropping stale deletions for tenant {tenant_id} in generation {:?}, objects may be leaked", tenant.generation); + info!("Dropping stale deletions for tenant {tenant_id} in generation {:?}, objects may be leaked", tenant.generation); metrics::DELETION_QUEUE.keys_dropped.inc_by(tenant.len() as u64); mutated = true; } else { diff --git a/test_runner/fixtures/pageserver/many_tenants.py b/test_runner/fixtures/pageserver/many_tenants.py index 8730d8ef751d..c437258c6f87 100644 --- a/test_runner/fixtures/pageserver/many_tenants.py +++ b/test_runner/fixtures/pageserver/many_tenants.py @@ -42,10 +42,6 @@ def single_timeline( log.info("detach template tenant form pageserver") env.pageserver.tenant_detach(template_tenant) - env.pageserver.allowed_errors.append( - # tenant detach causes this because the underlying attach-hook removes the tenant from storage controller entirely - ".*Dropped remote consistent LSN updates.*", - ) log.info(f"duplicating template tenant {ncopies} times in S3") tenants = fixtures.pageserver.remote_storage.duplicate_tenant(env, template_tenant, ncopies) diff --git a/test_runner/performance/pageserver/interactive/test_many_small_tenants.py b/test_runner/performance/pageserver/interactive/test_many_small_tenants.py index 0ff9c8fdaa98..33848b06d35c 100644 --- a/test_runner/performance/pageserver/interactive/test_many_small_tenants.py +++ b/test_runner/performance/pageserver/interactive/test_many_small_tenants.py @@ -55,10 +55,6 @@ def setup_template(env: NeonEnv): } template_tenant, template_timeline = env.neon_cli.create_tenant(set_default=True) env.pageserver.tenant_detach(template_tenant) - env.pageserver.allowed_errors.append( - # tenant detach causes this because the underlying attach-hook removes the tenant from storage controller entirely - ".*Dropped remote consistent LSN updates.*", - ) env.pageserver.tenant_attach(template_tenant, config) ep = env.endpoints.create_start("main", tenant_id=template_tenant) ep.safe_psql("create table foo(b text)") diff --git a/test_runner/performance/pageserver/pagebench/test_large_slru_basebackup.py b/test_runner/performance/pageserver/pagebench/test_large_slru_basebackup.py index b66db4d0ab72..b41ae601975f 100644 --- a/test_runner/performance/pageserver/pagebench/test_large_slru_basebackup.py +++ b/test_runner/performance/pageserver/pagebench/test_large_slru_basebackup.py @@ -86,10 +86,6 @@ def setup_tenant_template(env: NeonEnv, n_txns: int): template_tenant, template_timeline = env.neon_cli.create_tenant(set_default=True) env.pageserver.tenant_detach(template_tenant) - env.pageserver.allowed_errors.append( - # tenant detach causes this because the underlying attach-hook removes the tenant from storage controller entirely - ".*Dropped remote consistent LSN updates.*", - ) env.pageserver.tenant_attach(template_tenant, config) ps_http = env.pageserver.http_client() diff --git a/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py b/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py index a8f48fe675c6..60861cf939b8 100644 --- a/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py +++ b/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py @@ -164,10 +164,6 @@ def setup_tenant_template(env: NeonEnv, pg_bin: PgBin, scale: int): } template_tenant, template_timeline = env.neon_cli.create_tenant(set_default=True) env.pageserver.tenant_detach(template_tenant) - env.pageserver.allowed_errors.append( - # tenant detach causes this because the underlying attach-hook removes the tenant from storage controller entirely - ".*Dropped remote consistent LSN updates.*", - ) env.pageserver.tenant_attach(template_tenant, config) ps_http = env.pageserver.http_client() with env.endpoints.create_start("main", tenant_id=template_tenant) as ep: diff --git a/test_runner/performance/test_storage_controller_scale.py b/test_runner/performance/test_storage_controller_scale.py index d65a66b01081..3a6113706fa9 100644 --- a/test_runner/performance/test_storage_controller_scale.py +++ b/test_runner/performance/test_storage_controller_scale.py @@ -132,14 +132,6 @@ def test_storage_controller_many_tenants( ) for ps in env.pageservers: - # This can happen because when we do a loop over all pageservers and mark them offline/active, - # reconcilers might get cancelled, and the next reconcile can follow a not-so-elegant path of - # bumping generation before other attachments are detached. - # - # We could clean this up by making reconcilers respect the .observed of their predecessor, if - # we spawn with a wait for the predecessor. - ps.allowed_errors.append(".*Dropped remote consistent LSN updates.*") - # Storage controller is allowed to drop pageserver requests when the cancellation token # for a Reconciler fires. ps.allowed_errors.append(".*request was dropped before completing.*") diff --git a/test_runner/regress/test_attach_tenant_config.py b/test_runner/regress/test_attach_tenant_config.py index e117c2140f5e..f2ee2b70aac6 100644 --- a/test_runner/regress/test_attach_tenant_config.py +++ b/test_runner/regress/test_attach_tenant_config.py @@ -21,8 +21,6 @@ def positive_env(neon_env_builder: NeonEnvBuilder) -> NeonEnv: [ # eviction might be the first one after an attach to access the layers ".*unexpectedly on-demand downloading remote layer .* for task kind Eviction", - # detach can happen before we get to validate the generation number - ".*deletion backend: Dropped remote consistent LSN updates for tenant.*", ] ) assert isinstance(env.pageserver_remote_storage, LocalFsStorage) @@ -58,10 +56,6 @@ def negative_env(neon_env_builder: NeonEnvBuilder) -> Generator[NegativeTests, N env.pageserver.allowed_errors.extend( [ - # This fixture detaches the tenant, and tests using it will tend to re-attach it - # shortly after. There may be un-processed deletion_queue validations from the - # initial attachment - ".*Dropped remote consistent LSN updates.*", # This fixture is for tests that will intentionally generate 400 responses ".*Error processing HTTP request: Bad request", ] diff --git a/test_runner/regress/test_change_pageserver.py b/test_runner/regress/test_change_pageserver.py index 97ab69049d00..4d2cdb8e320a 100644 --- a/test_runner/regress/test_change_pageserver.py +++ b/test_runner/regress/test_change_pageserver.py @@ -14,11 +14,6 @@ def test_change_pageserver(neon_env_builder: NeonEnvBuilder): ) env = neon_env_builder.init_start() - for pageserver in env.pageservers: - # This test dual-attaches a tenant, one of the pageservers will therefore - # be running with a stale generation. - pageserver.allowed_errors.append(".*Dropped remote consistent LSN updates.*") - env.neon_cli.create_branch("test_change_pageserver") endpoint = env.endpoints.create_start("test_change_pageserver") diff --git a/test_runner/regress/test_layers_from_future.py b/test_runner/regress/test_layers_from_future.py index 54d3b2d515c5..3b2218dd9b09 100644 --- a/test_runner/regress/test_layers_from_future.py +++ b/test_runner/regress/test_layers_from_future.py @@ -39,9 +39,6 @@ def test_issue_5878(neon_env_builder: NeonEnvBuilder): env = neon_env_builder.init_configs() env.start() - env.pageserver.allowed_errors.extend( - [".*Dropped remote consistent LSN updates.*", ".*Dropping stale deletions.*"] - ) ps_http = env.pageserver.http_client() diff --git a/test_runner/regress/test_pageserver_generations.py b/test_runner/regress/test_pageserver_generations.py index 696af24e5c0a..7ce38c5c3c82 100644 --- a/test_runner/regress/test_pageserver_generations.py +++ b/test_runner/regress/test_pageserver_generations.py @@ -249,10 +249,6 @@ def test_deferred_deletion(neon_env_builder: NeonEnvBuilder): assert timeline["remote_consistent_lsn"] == timeline["remote_consistent_lsn_visible"] assert get_deletion_queue_dropped_lsn_updates(ps_http) == 0 - main_pageserver.allowed_errors.extend( - [".*Dropped remote consistent LSN updates.*", ".*Dropping stale deletions.*"] - ) - # Now advance the generation in the control plane: subsequent validations # from the running pageserver will fail. No more deletions should happen. env.storage_controller.attach_hook_issue(env.initial_tenant, other_pageserver.id) @@ -397,8 +393,6 @@ def assert_deletions_submitted(n: int) -> None: # validated before restart. assert get_deletion_queue_executed(ps_http) == before_restart_depth else: - main_pageserver.allowed_errors.extend([".*Dropping stale deletions.*"]) - # If we lost the attachment, we should have dropped our pre-restart deletions. assert get_deletion_queue_dropped(ps_http) == before_restart_depth @@ -553,13 +547,6 @@ def test_multi_attach( tenant_id = env.initial_tenant timeline_id = env.initial_timeline - # We will intentionally create situations where stale deletions happen from non-latest-generation - # nodes when the tenant is multiply-attached - for ps in env.pageservers: - ps.allowed_errors.extend( - [".*Dropped remote consistent LSN updates.*", ".*Dropping stale deletions.*"] - ) - # Initially, the tenant will be attached to the first pageserver (first is default in our test harness) wait_until(10, 0.2, lambda: assert_tenant_state(http_clients[0], tenant_id, "Active")) _detail = http_clients[0].timeline_detail(tenant_id, timeline_id) diff --git a/test_runner/regress/test_pageserver_secondary.py b/test_runner/regress/test_pageserver_secondary.py index 8431840dc069..4c828b86b053 100644 --- a/test_runner/regress/test_pageserver_secondary.py +++ b/test_runner/regress/test_pageserver_secondary.py @@ -83,9 +83,6 @@ def test_location_conf_churn(neon_env_builder: NeonEnvBuilder, seed: int): for ps in env.pageservers: ps.allowed_errors.extend( [ - # We will make no effort to avoid stale attachments - ".*Dropped remote consistent LSN updates.*", - ".*Dropping stale deletions.*", # page_service_conn_main{peer_addr=[::1]:41176}: query handler for 'pagestream 3b19aec5038c796f64b430b30a555121 d07776761d44050b8aab511df1657d83' failed: Tenant 3b19aec5038c796f64b430b30a555121 not found ".*query handler.*Tenant.*not found.*", # page_service_conn_main{peer_addr=[::1]:45552}: query handler for 'pagestream 414ede7ad50f775a8e7d9ba0e43b9efc a43884be16f44b3626482b6981b2c745' failed: Tenant 414ede7ad50f775a8e7d9ba0e43b9efc is not active diff --git a/test_runner/regress/test_remote_storage.py b/test_runner/regress/test_remote_storage.py index b26bd3422f30..fac7fe9deef6 100644 --- a/test_runner/regress/test_remote_storage.py +++ b/test_runner/regress/test_remote_storage.py @@ -355,13 +355,6 @@ def churn_while_failpoints_active(result): env.pageserver.stop(immediate=True) env.endpoints.stop_all() - # We are about to forcibly drop local dirs. Storage controller will increment generation in re-attach before - # we later increment when actually attaching it again, leading to skipping a generation and potentially getting - # these warnings if there was a durable but un-executed deletion list at time of restart. - env.pageserver.allowed_errors.extend( - [".*Dropped remote consistent LSN updates.*", ".*Dropping stale deletions.*"] - ) - dir_to_clear = env.pageserver.tenant_dir() shutil.rmtree(dir_to_clear) os.mkdir(dir_to_clear) diff --git a/test_runner/regress/test_sharding.py b/test_runner/regress/test_sharding.py index 8267d3f36c0b..d414f986e655 100644 --- a/test_runner/regress/test_sharding.py +++ b/test_runner/regress/test_sharding.py @@ -1144,10 +1144,6 @@ def test_sharding_split_failures( ) for ps in env.pageservers: - # When we do node failures and abandon a shard, it will de-facto have old generation and - # thereby be unable to publish remote consistent LSN updates - ps.allowed_errors.append(".*Dropped remote consistent LSN updates.*") - # If we're using a failure that will panic the storage controller, all background # upcalls from the pageserver can fail ps.allowed_errors.append(".*calling control plane generation validation API failed.*") diff --git a/test_runner/regress/test_storage_controller.py b/test_runner/regress/test_storage_controller.py index a78f566f0e4c..d37f7aae3dfd 100644 --- a/test_runner/regress/test_storage_controller.py +++ b/test_runner/regress/test_storage_controller.py @@ -60,11 +60,6 @@ def test_storage_controller_smoke( neon_env_builder.num_pageservers = 3 env = neon_env_builder.init_configs() - for pageserver in env.pageservers: - # This test detaches tenants during migration, which can race with deletion queue operations, - # during detach we only do an advisory flush, we don't wait for it. - pageserver.allowed_errors.extend([".*Dropped remote consistent LSN updates.*"]) - # Start services by hand so that we can skip a pageserver (this will start + register later) env.broker.try_start() env.storage_controller.start() @@ -484,9 +479,6 @@ def handler(request: Request): # Start running env = neon_env_builder.init_start() - # We will to an unclean migration, which will result in deletion queue warnings - env.pageservers[0].allowed_errors.append(".*Dropped remote consistent LSN updates for tenant.*") - # Initial notification from tenant creation assert len(notifications) == 1 expect: Dict[str, Union[List[Dict[str, int]], str, None, int]] = { @@ -1054,13 +1046,6 @@ def tenants_placed(): online_node_ids = set(range(1, len(env.pageservers) + 1)) - offline_node_ids for node_id in offline_node_ids: - env.get_pageserver(node_id).allowed_errors.append( - # In the case of the failpoint failure, the impacted pageserver - # still believes it has the tenant attached since location - # config calls into it will fail due to being marked offline. - ".*Dropped remote consistent LSN updates.*", - ) - if len(offline_node_ids) > 1: env.get_pageserver(node_id).allowed_errors.append( ".*Scheduling error when marking pageserver.*offline.*", diff --git a/test_runner/regress/test_tenant_conf.py b/test_runner/regress/test_tenant_conf.py index 2cbb036c0d7c..80fb2b55b8b2 100644 --- a/test_runner/regress/test_tenant_conf.py +++ b/test_runner/regress/test_tenant_conf.py @@ -320,10 +320,6 @@ def test_creating_tenant_conf_after_attach(neon_env_builder: NeonEnvBuilder): assert not config_path.exists(), "detach did not remove config file" - # The re-attach's increment of the generation number may invalidate deletion queue - # updates in flight from the previous attachment. - env.pageserver.allowed_errors.append(".*Dropped remote consistent LSN updates.*") - env.pageserver.tenant_attach(tenant_id) wait_until( number_of_iterations=5, diff --git a/test_runner/regress/test_tenant_detach.py b/test_runner/regress/test_tenant_detach.py index 2056840558e6..b165588636c7 100644 --- a/test_runner/regress/test_tenant_detach.py +++ b/test_runner/regress/test_tenant_detach.py @@ -76,10 +76,6 @@ def test_tenant_reattach(neon_env_builder: NeonEnvBuilder, mode: str): env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) - # Our re-attach may race with the deletion queue processing LSN updates - # from the original attachment. - env.pageserver.allowed_errors.append(".*Dropped remote consistent LSN updates.*") - with env.endpoints.create_start("main", tenant_id=tenant_id) as endpoint: with endpoint.cursor() as cur: cur.execute("CREATE TABLE t(key int primary key, value text)") @@ -349,10 +345,6 @@ def test_detach_while_attaching( env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) - # Our re-attach may race with the deletion queue processing LSN updates - # from the original attachment. - env.pageserver.allowed_errors.append(".*Dropped remote consistent LSN updates.*") - # Create table, and insert some rows. Make it big enough that it doesn't fit in # shared_buffers, otherwise the SELECT after restart will just return answer # from shared_buffers without hitting the page server, which defeats the point @@ -422,10 +414,6 @@ def test_detach_while_activating( env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) - # Our re-attach may race with the deletion queue processing LSN updates - # from the original attachment. - env.pageserver.allowed_errors.append(".*Dropped remote consistent LSN updates.*") - data_id = 1 data_secret = "very secret secret" insert_test_data(pageserver_http, tenant_id, timeline_id, data_id, data_secret, endpoint) diff --git a/test_runner/regress/test_tenant_relocation.py b/test_runner/regress/test_tenant_relocation.py index 9fe732e28806..43e9a0d36e80 100644 --- a/test_runner/regress/test_tenant_relocation.py +++ b/test_runner/regress/test_tenant_relocation.py @@ -203,8 +203,6 @@ def test_tenant_relocation( [ # Needed for detach polling on the original pageserver f".*NotFound: tenant {tenant_id}.*", - # We will dual-attach in this test, so stale generations are expected - ".*Dropped remote consistent LSN updates.*", ] ) diff --git a/test_runner/regress/test_tenants.py b/test_runner/regress/test_tenants.py index 3705406c2ff9..04b3fdd80fa5 100644 --- a/test_runner/regress/test_tenants.py +++ b/test_runner/regress/test_tenants.py @@ -386,10 +386,6 @@ def test_create_churn_during_restart(neon_env_builder: NeonEnvBuilder): # generation nubmers out of order. env.pageserver.allowed_errors.append(".*Generation .+ is less than existing .+") - # Our multiple creation requests will advance generation quickly, and when we skip - # a generation number we can generate these warnings - env.pageserver.allowed_errors.append(".*Dropped remote consistent LSN updates for tenant .+") - # Timeline::flush_and_shutdown cannot tell if it is hitting a failure because of # an incomplete attach, or some other problem. In the field this should be rare, # so we allow it to log at WARN, even if it is occasionally a false positive. From e579bc0819998f234277f2f29d10f2a444154753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Thu, 4 Jul 2024 17:07:16 +0200 Subject: [PATCH 039/194] Add find-large-objects subcommand to scrubber (#8257) Adds a find-large-objects subcommand to the scrubber to allow listing layer objects larger than a specific size. To be used like: ``` AWS_PROFILE=dev REGION=us-east-2 BUCKET=neon-dev-storage-us-east-2 cargo run -p storage_scrubber -- find-large-objects --min-size 250000000 --ignore-deltas ``` Part of #5431 --- storage_scrubber/src/checks.rs | 2 +- storage_scrubber/src/find_large_objects.rs | 97 ++++++++++++++++++++++ storage_scrubber/src/lib.rs | 1 + storage_scrubber/src/main.rs | 18 ++++ 4 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 storage_scrubber/src/find_large_objects.rs diff --git a/storage_scrubber/src/checks.rs b/storage_scrubber/src/checks.rs index 4eb8580e32cf..f687b24320ce 100644 --- a/storage_scrubber/src/checks.rs +++ b/storage_scrubber/src/checks.rs @@ -259,7 +259,7 @@ pub(crate) enum BlobDataParseResult { Incorrect(Vec), } -fn parse_layer_object_name(name: &str) -> Result<(LayerName, Generation), String> { +pub(crate) fn parse_layer_object_name(name: &str) -> Result<(LayerName, Generation), String> { match name.rsplit_once('-') { // FIXME: this is gross, just use a regex? Some((layer_filename, gen)) if gen.len() == 8 => { diff --git a/storage_scrubber/src/find_large_objects.rs b/storage_scrubber/src/find_large_objects.rs new file mode 100644 index 000000000000..24668b65169a --- /dev/null +++ b/storage_scrubber/src/find_large_objects.rs @@ -0,0 +1,97 @@ +use futures::StreamExt; +use pageserver::tenant::storage_layer::LayerName; +use serde::{Deserialize, Serialize}; + +use crate::{ + checks::parse_layer_object_name, init_remote, list_objects_with_retries, + metadata_stream::stream_tenants, BucketConfig, NodeKind, +}; + +#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq)] +enum LargeObjectKind { + DeltaLayer, + ImageLayer, + Other, +} + +impl LargeObjectKind { + fn from_key(key: &str) -> Self { + let fname = key.split('/').last().unwrap(); + + let Ok((layer_name, _generation)) = parse_layer_object_name(fname) else { + return LargeObjectKind::Other; + }; + + match layer_name { + LayerName::Image(_) => LargeObjectKind::ImageLayer, + LayerName::Delta(_) => LargeObjectKind::DeltaLayer, + } + } +} + +#[derive(Serialize, Deserialize)] +pub struct LargeObject { + pub key: String, + pub size: u64, + kind: LargeObjectKind, +} + +#[derive(Serialize, Deserialize)] +pub struct LargeObjectListing { + pub objects: Vec, +} + +pub async fn find_large_objects( + bucket_config: BucketConfig, + min_size: u64, + ignore_deltas: bool, +) -> anyhow::Result { + let (s3_client, target) = init_remote(bucket_config.clone(), NodeKind::Pageserver)?; + let mut tenants = std::pin::pin!(stream_tenants(&s3_client, &target)); + let mut objects = Vec::new(); + let mut tenant_ctr = 0u64; + let mut object_ctr = 0u64; + while let Some(tenant_shard_id) = tenants.next().await { + let tenant_shard_id = tenant_shard_id?; + let mut tenant_root = target.tenant_root(&tenant_shard_id); + // We want the objects and not just common prefixes + tenant_root.delimiter.clear(); + let mut continuation_token = None; + loop { + let fetch_response = + list_objects_with_retries(&s3_client, &tenant_root, continuation_token.clone()) + .await?; + for obj in fetch_response.contents().iter().filter(|o| { + if let Some(obj_size) = o.size { + min_size as i64 <= obj_size + } else { + false + } + }) { + let key = obj.key().expect("couldn't get key").to_owned(); + let kind = LargeObjectKind::from_key(&key); + if ignore_deltas && kind == LargeObjectKind::DeltaLayer { + continue; + } + objects.push(LargeObject { + key, + size: obj.size.unwrap() as u64, + kind, + }) + } + object_ctr += fetch_response.contents().len() as u64; + match fetch_response.next_continuation_token { + Some(new_token) => continuation_token = Some(new_token), + None => break, + } + } + + tenant_ctr += 1; + if tenant_ctr % 50 == 0 { + tracing::info!( + "Scanned {tenant_ctr} shards. objects={object_ctr}, found={}, current={tenant_shard_id}.", objects.len() + ); + } + } + Ok(LargeObjectListing { objects }) +} diff --git a/storage_scrubber/src/lib.rs b/storage_scrubber/src/lib.rs index 64273432fc0c..6adaa5d38f6b 100644 --- a/storage_scrubber/src/lib.rs +++ b/storage_scrubber/src/lib.rs @@ -2,6 +2,7 @@ #![deny(clippy::undocumented_unsafe_blocks)] pub mod checks; pub mod cloud_admin_api; +pub mod find_large_objects; pub mod garbage; pub mod metadata_stream; pub mod pageserver_physical_gc; diff --git a/storage_scrubber/src/main.rs b/storage_scrubber/src/main.rs index 222bd10ed248..10699edd3c94 100644 --- a/storage_scrubber/src/main.rs +++ b/storage_scrubber/src/main.rs @@ -1,6 +1,7 @@ use anyhow::bail; use camino::Utf8PathBuf; use pageserver_api::shard::TenantShardId; +use storage_scrubber::find_large_objects; use storage_scrubber::garbage::{find_garbage, purge_garbage, PurgeMode}; use storage_scrubber::pageserver_physical_gc::GcMode; use storage_scrubber::scan_pageserver_metadata::scan_metadata; @@ -72,6 +73,12 @@ enum Command { #[arg(short, long, default_value_t = GcMode::IndicesOnly)] mode: GcMode, }, + FindLargeObjects { + #[arg(long = "min-size")] + min_size: u64, + #[arg(short, long, default_value_t = false)] + ignore_deltas: bool, + }, } #[tokio::main] @@ -86,6 +93,7 @@ async fn main() -> anyhow::Result<()> { Command::PurgeGarbage { .. } => "purge-garbage", Command::TenantSnapshot { .. } => "tenant-snapshot", Command::PageserverPhysicalGc { .. } => "pageserver-physical-gc", + Command::FindLargeObjects { .. } => "find-large-objects", }; let _guard = init_logging(&format!( "{}_{}_{}_{}.log", @@ -199,5 +207,15 @@ async fn main() -> anyhow::Result<()> { println!("{}", serde_json::to_string(&summary).unwrap()); Ok(()) } + Command::FindLargeObjects { + min_size, + ignore_deltas, + } => { + let summary = + find_large_objects::find_large_objects(bucket_config, min_size, ignore_deltas) + .await?; + println!("{}", serde_json::to_string(&summary).unwrap()); + Ok(()) + } } } From 19accfee4e677ed8fabc4dd1f370389038978499 Mon Sep 17 00:00:00 2001 From: Yuchen Liang <70461588+yliang412@users.noreply.github.com> Date: Thu, 4 Jul 2024 11:09:05 -0400 Subject: [PATCH 040/194] feat(pageserver): integrate lsn lease into synthetic size (#8220) Part of #7497, closes #8071. (accidentally closed #8208, reopened here) ## Problem After the changes in #8084, we need synthetic size to also account for leased LSNs so that users do not get free retention by running a small ephemeral endpoint for a long time. ## Summary of changes This PR integrates LSN leases into the synthetic size calculation. We model leases as read-only branches started at the leased LSN (except it does not have a timeline id). Other changes: - Add new unit tests testing whether a lease behaves like a read-only branch. - Change `/size_debug` response to include lease point in the SVG visualization. - Fix `/lsn_lease` HTTP API to do proper parsing for POST. Signed-off-by: Yuchen Liang Co-authored-by: Joonas Koivunen Co-authored-by: Christian Schwarz --- libs/pageserver_api/src/models.rs | 5 ++ libs/tenant_size_model/src/calculation.rs | 4 +- libs/tenant_size_model/src/svg.rs | 36 ++++++++-- pageserver/src/http/openapi_spec.yml | 22 +++--- pageserver/src/http/routes.rs | 18 +++-- pageserver/src/tenant/size.rs | 85 ++++++++++++++++++++-- pageserver/src/tenant/timeline.rs | 9 +++ test_runner/fixtures/pageserver/http.py | 16 +++++ test_runner/regress/test_tenant_size.py | 88 +++++++++++++++++++++++ 9 files changed, 256 insertions(+), 27 deletions(-) diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index 92289537613d..ad65602f54d9 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -228,6 +228,11 @@ pub struct TimelineCreateRequest { pub pg_version: Option, } +#[derive(Serialize, Deserialize, Clone)] +pub struct LsnLeaseRequest { + pub lsn: Lsn, +} + #[derive(Serialize, Deserialize)] pub struct TenantShardSplitRequest { pub new_shard_count: u8, diff --git a/libs/tenant_size_model/src/calculation.rs b/libs/tenant_size_model/src/calculation.rs index f05997ee6547..be005622199d 100644 --- a/libs/tenant_size_model/src/calculation.rs +++ b/libs/tenant_size_model/src/calculation.rs @@ -34,10 +34,10 @@ struct SegmentSize { } struct SizeAlternatives { - // cheapest alternative if parent is available. + /// cheapest alternative if parent is available. incremental: SegmentSize, - // cheapest alternative if parent node is not available + /// cheapest alternative if parent node is not available non_incremental: Option, } diff --git a/libs/tenant_size_model/src/svg.rs b/libs/tenant_size_model/src/svg.rs index f26d3aa79d1a..0de2890bb414 100644 --- a/libs/tenant_size_model/src/svg.rs +++ b/libs/tenant_size_model/src/svg.rs @@ -3,10 +3,17 @@ use std::fmt::Write; const SVG_WIDTH: f32 = 500.0; +/// Different branch kind for SVG drawing. +#[derive(PartialEq)] +pub enum SvgBranchKind { + Timeline, + Lease, +} + struct SvgDraw<'a> { storage: &'a StorageModel, branches: &'a [String], - seg_to_branch: &'a [usize], + seg_to_branch: &'a [(usize, SvgBranchKind)], sizes: &'a [SegmentSizeResult], // layout @@ -42,13 +49,18 @@ fn draw_legend(result: &mut String) -> anyhow::Result<()> { "" )?; writeln!(result, "WAL not retained")?; + writeln!( + result, + "" + )?; + writeln!(result, "LSN lease")?; Ok(()) } pub fn draw_svg( storage: &StorageModel, branches: &[String], - seg_to_branch: &[usize], + seg_to_branch: &[(usize, SvgBranchKind)], sizes: &SizeResult, ) -> anyhow::Result { let mut draw = SvgDraw { @@ -100,7 +112,7 @@ impl<'a> SvgDraw<'a> { // Layout the timelines on Y dimension. // TODO - let mut y = 100.0; + let mut y = 120.0; let mut branch_y_coordinates = Vec::new(); for _branch in self.branches { branch_y_coordinates.push(y); @@ -109,7 +121,7 @@ impl<'a> SvgDraw<'a> { // Calculate coordinates for each point let seg_coordinates = std::iter::zip(segments, self.seg_to_branch) - .map(|(seg, branch_id)| { + .map(|(seg, (branch_id, _))| { let x = (seg.lsn - min_lsn) as f32 / xscale; let y = branch_y_coordinates[*branch_id]; (x, y) @@ -175,6 +187,22 @@ impl<'a> SvgDraw<'a> { // draw a snapshot point if it's needed let (coord_x, coord_y) = self.seg_coordinates[seg_id]; + + let (_, kind) = &self.seg_to_branch[seg_id]; + if kind == &SvgBranchKind::Lease { + let (x1, y1) = (coord_x, coord_y - 10.0); + let (x2, y2) = (coord_x, coord_y + 10.0); + + let style = "stroke-width=\"3\" stroke=\"blue\""; + + writeln!( + result, + "", + )?; + writeln!(result, " leased lsn at {}", seg.lsn)?; + writeln!(result, "")?; + } + if self.sizes[seg_id].method == SegmentMethod::SnapshotHere { writeln!( result, diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index 58ff6e3f83cc..5ba329f05ece 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -265,15 +265,19 @@ paths: type: string format: hex post: - description: Obtain lease for the given LSN - parameters: - - name: lsn - in: query - required: true - schema: - type: string - format: hex - description: A LSN to obtain the lease for + description: Obtains a lease for the given LSN. + requestBody: + content: + application/json: + schema: + type: object + required: + - lsn + properties: + lsn: + description: A LSN to obtain the lease for. + type: string + format: hex responses: "200": description: OK diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 6a6f17604dee..893302b7d6d9 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -22,6 +22,7 @@ use pageserver_api::models::ListAuxFilesRequest; use pageserver_api::models::LocationConfig; use pageserver_api::models::LocationConfigListResponse; use pageserver_api::models::LsnLease; +use pageserver_api::models::LsnLeaseRequest; use pageserver_api::models::ShardParameters; use pageserver_api::models::TenantDetails; use pageserver_api::models::TenantLocationConfigResponse; @@ -42,7 +43,7 @@ use pageserver_api::shard::TenantShardId; use remote_storage::DownloadError; use remote_storage::GenericRemoteStorage; use remote_storage::TimeTravelError; -use tenant_size_model::{SizeResult, StorageModel}; +use tenant_size_model::{svg::SvgBranchKind, SizeResult, StorageModel}; use tokio_util::sync::CancellationToken; use tracing::*; use utils::auth::JwtAuth; @@ -1195,10 +1196,15 @@ fn synthetic_size_html_response( timeline_map.insert(ti.timeline_id, index); timeline_ids.push(ti.timeline_id.to_string()); } - let seg_to_branch: Vec = inputs + let seg_to_branch: Vec<(usize, SvgBranchKind)> = inputs .segments .iter() - .map(|seg| *timeline_map.get(&seg.timeline_id).unwrap()) + .map(|seg| { + ( + *timeline_map.get(&seg.timeline_id).unwrap(), + seg.kind.into(), + ) + }) .collect(); let svg = @@ -1531,15 +1537,13 @@ async fn handle_tenant_break( // Obtains an lsn lease on the given timeline. async fn lsn_lease_handler( - request: Request, + mut request: Request, _cancel: CancellationToken, ) -> Result, ApiError> { let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?; let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; check_permission(&request, Some(tenant_shard_id.tenant_id))?; - - let lsn: Lsn = parse_query_param(&request, "lsn")? - .ok_or_else(|| ApiError::BadRequest(anyhow!("missing 'lsn' query parameter")))?; + let lsn = json_request::(&mut request).await?.lsn; let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download); diff --git a/pageserver/src/tenant/size.rs b/pageserver/src/tenant/size.rs index b2338b620ebf..23354417e788 100644 --- a/pageserver/src/tenant/size.rs +++ b/pageserver/src/tenant/size.rs @@ -3,6 +3,7 @@ use std::collections::hash_map::Entry; use std::collections::{HashMap, HashSet}; use std::sync::Arc; +use tenant_size_model::svg::SvgBranchKind; use tokio::sync::oneshot::error::RecvError; use tokio::sync::Semaphore; use tokio_util::sync::CancellationToken; @@ -87,6 +88,9 @@ impl SegmentMeta { LsnKind::BranchPoint => true, LsnKind::GcCutOff => true, LsnKind::BranchEnd => false, + LsnKind::LeasePoint => true, + LsnKind::LeaseStart => false, + LsnKind::LeaseEnd => false, } } } @@ -103,6 +107,21 @@ pub enum LsnKind { GcCutOff, /// Last record LSN BranchEnd, + /// A LSN lease is granted here. + LeasePoint, + /// A lease starts from here. + LeaseStart, + /// Last record LSN for the lease (should have the same LSN as the previous [`LsnKind::LeaseStart`]). + LeaseEnd, +} + +impl From for SvgBranchKind { + fn from(kind: LsnKind) -> Self { + match kind { + LsnKind::LeasePoint | LsnKind::LeaseStart | LsnKind::LeaseEnd => SvgBranchKind::Lease, + _ => SvgBranchKind::Timeline, + } + } } /// Collect all relevant LSNs to the inputs. These will only be helpful in the serialized form as @@ -124,6 +143,9 @@ pub struct TimelineInputs { /// Cutoff point calculated from the user-supplied 'max_retention_period' retention_param_cutoff: Option, + + /// Lease points on the timeline + lease_points: Vec, } /// Gathers the inputs for the tenant sizing model. @@ -234,6 +256,13 @@ pub(super) async fn gather_inputs( None }; + let lease_points = gc_info + .leases + .keys() + .filter(|&&lsn| lsn > ancestor_lsn) + .copied() + .collect::>(); + // next_gc_cutoff in parent branch are not of interest (right now at least), nor do we // want to query any logical size before initdb_lsn. let branch_start_lsn = cmp::max(ancestor_lsn, timeline.initdb_lsn); @@ -248,6 +277,8 @@ pub(super) async fn gather_inputs( .map(|lsn| (lsn, LsnKind::BranchPoint)) .collect::>(); + lsns.extend(lease_points.iter().map(|&lsn| (lsn, LsnKind::LeasePoint))); + drop(gc_info); // Add branch points we collected earlier, just in case there were any that were @@ -296,6 +327,7 @@ pub(super) async fn gather_inputs( if kind == LsnKind::BranchPoint { branchpoint_segments.insert((timeline_id, lsn), segments.len()); } + segments.push(SegmentMeta { segment: Segment { parent: Some(parent), @@ -306,7 +338,45 @@ pub(super) async fn gather_inputs( timeline_id: timeline.timeline_id, kind, }); - parent += 1; + + parent = segments.len() - 1; + + if kind == LsnKind::LeasePoint { + // Needs `LeaseStart` and `LeaseEnd` as well to model lease as a read-only branch that never writes data + // (i.e. it's lsn has not advanced from ancestor_lsn), and therefore the three segments have the same LSN + // value. Without the other two segments, the calculation code would not count the leased LSN as a point + // to be retained. + // Did not use `BranchStart` or `BranchEnd` so we can differentiate branches and leases during debug. + // + // Alt Design: rewrite the entire calculation code to be independent of timeline id. Both leases and + // branch points can be given a synthetic id so we can unite them. + let mut lease_parent = parent; + + // Start of a lease. + segments.push(SegmentMeta { + segment: Segment { + parent: Some(lease_parent), + lsn: lsn.0, + size: None, // Filled in later, if necessary + needed: lsn > next_gc_cutoff, // only needed if the point is within rentention. + }, + timeline_id: timeline.timeline_id, + kind: LsnKind::LeaseStart, + }); + lease_parent += 1; + + // End of the lease. + segments.push(SegmentMeta { + segment: Segment { + parent: Some(lease_parent), + lsn: lsn.0, + size: None, // Filled in later, if necessary + needed: true, // everything at the lease LSN must be readable => is needed + }, + timeline_id: timeline.timeline_id, + kind: LsnKind::LeaseEnd, + }); + } } // Current end of the timeline @@ -332,6 +402,7 @@ pub(super) async fn gather_inputs( pitr_cutoff, next_gc_cutoff, retention_param_cutoff, + lease_points, }); } @@ -674,7 +745,8 @@ fn verify_size_for_multiple_branches() { "horizon_cutoff": "0/2210CD0", "pitr_cutoff": "0/2210CD0", "next_gc_cutoff": "0/2210CD0", - "retention_param_cutoff": null + "retention_param_cutoff": null, + "lease_points": [] }, { "timeline_id": "454626700469f0a9914949b9d018e876", @@ -684,7 +756,8 @@ fn verify_size_for_multiple_branches() { "horizon_cutoff": "0/1817770", "pitr_cutoff": "0/1817770", "next_gc_cutoff": "0/1817770", - "retention_param_cutoff": null + "retention_param_cutoff": null, + "lease_points": [] }, { "timeline_id": "cb5e3cbe60a4afc00d01880e1a37047f", @@ -694,7 +767,8 @@ fn verify_size_for_multiple_branches() { "horizon_cutoff": "0/18B3D98", "pitr_cutoff": "0/18B3D98", "next_gc_cutoff": "0/18B3D98", - "retention_param_cutoff": null + "retention_param_cutoff": null, + "lease_points": [] } ] } @@ -749,7 +823,8 @@ fn verify_size_for_one_branch() { "horizon_cutoff": "47/240A5860", "pitr_cutoff": "47/240A5860", "next_gc_cutoff": "47/240A5860", - "retention_param_cutoff": "0/0" + "retention_param_cutoff": "0/0", + "lease_points": [] } ] }"#; diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index bbf0d0a4bf68..42e55ab2695c 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -14,6 +14,7 @@ use anyhow::{anyhow, bail, ensure, Context, Result}; use arc_swap::ArcSwap; use bytes::Bytes; use camino::Utf8Path; +use chrono::{DateTime, Utc}; use enumset::EnumSet; use fail::fail_point; use once_cell::sync::Lazy; @@ -1590,7 +1591,13 @@ impl Timeline { let existing_lease = occupied.get_mut(); if valid_until > existing_lease.valid_until { existing_lease.valid_until = valid_until; + let dt: DateTime = valid_until.into(); + info!("lease extended to {}", dt); + } else { + let dt: DateTime = existing_lease.valid_until.into(); + info!("existing lease covers greater length, valid until {}", dt); } + existing_lease.clone() } else { // Reject already GC-ed LSN (lsn < latest_gc_cutoff) @@ -1599,6 +1606,8 @@ impl Timeline { bail!("tried to request a page version that was garbage collected. requested at {} gc cutoff {}", lsn, *latest_gc_cutoff_lsn); } + let dt: DateTime = valid_until.into(); + info!("lease created, valid until {}", dt); entry.or_insert(LsnLease { valid_until }).clone() } }; diff --git a/test_runner/fixtures/pageserver/http.py b/test_runner/fixtures/pageserver/http.py index 3da0be802116..03aee9e5c597 100644 --- a/test_runner/fixtures/pageserver/http.py +++ b/test_runner/fixtures/pageserver/http.py @@ -599,6 +599,22 @@ def timeline_get_lsn_by_timestamp( res_json = res.json() return res_json + def timeline_lsn_lease( + self, tenant_id: Union[TenantId, TenantShardId], timeline_id: TimelineId, lsn: Lsn + ): + data = { + "lsn": str(lsn), + } + + log.info(f"Requesting lsn lease for {lsn=}, {tenant_id=}, {timeline_id=}") + res = self.post( + f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/lsn_lease", + json=data, + ) + self.verbose_error(res) + res_json = res.json() + return res_json + def timeline_get_timestamp_of_lsn( self, tenant_id: Union[TenantId, TenantShardId], timeline_id: TimelineId, lsn: Lsn ): diff --git a/test_runner/regress/test_tenant_size.py b/test_runner/regress/test_tenant_size.py index 6c85ddebbcfb..70e8fe67d595 100644 --- a/test_runner/regress/test_tenant_size.py +++ b/test_runner/regress/test_tenant_size.py @@ -10,6 +10,7 @@ Endpoint, NeonEnv, NeonEnvBuilder, + flush_ep_to_pageserver, wait_for_last_flush_lsn, wait_for_wal_insert_lsn, ) @@ -710,3 +711,90 @@ def mask_model_inputs(x): return newlist else: return x + + +@pytest.mark.parametrize("zero_gc", [True, False]) +def test_lsn_lease_size(neon_env_builder: NeonEnvBuilder, test_output_dir: Path, zero_gc: bool): + """ + Compare a LSN lease to a read-only branch for synthetic size calculation. + They should have the same effect. + """ + + conf = { + "pitr_interval": "0s" if zero_gc else "3600s", + "gc_period": "0s", + } + + env = neon_env_builder.init_start(initial_tenant_conf=conf) + + ro_branch_res = insert_with_action( + env, env.initial_tenant, env.initial_timeline, test_output_dir, action="branch" + ) + + tenant, timeline = env.neon_cli.create_tenant(conf=conf) + lease_res = insert_with_action(env, tenant, timeline, test_output_dir, action="lease") + + assert_size_approx_equal(lease_res, ro_branch_res) + + +def insert_with_action( + env: NeonEnv, + tenant: TenantId, + timeline: TimelineId, + test_output_dir: Path, + action: str, +) -> int: + """ + Inserts some data on the timeline, perform an action, and insert more data on the same timeline. + Returns the size at the end of the insertion. + + Valid actions: + - "lease": Acquires a lease. + - "branch": Creates a child branch but never writes to it. + """ + + client = env.pageserver.http_client() + with env.endpoints.create_start("main", tenant_id=tenant) as ep: + initial_size = client.tenant_size(tenant) + log.info(f"initial size: {initial_size}") + + with ep.cursor() as cur: + cur.execute( + "CREATE TABLE t0 AS SELECT i::bigint n FROM generate_series(0, 1000000) s(i)" + ) + last_flush_lsn = wait_for_last_flush_lsn(env, ep, tenant, timeline) + + if action == "lease": + res = client.timeline_lsn_lease(tenant, timeline, last_flush_lsn) + log.info(f"result from lsn_lease api: {res}") + elif action == "branch": + ro_branch = env.neon_cli.create_branch( + "ro_branch", tenant_id=tenant, ancestor_start_lsn=last_flush_lsn + ) + log.info(f"{ro_branch=} created") + else: + raise AssertionError("Invalid action type, only `lease` and `branch`are accepted") + + with ep.cursor() as cur: + cur.execute( + "CREATE TABLE t1 AS SELECT i::bigint n FROM generate_series(0, 1000000) s(i)" + ) + cur.execute( + "CREATE TABLE t2 AS SELECT i::bigint n FROM generate_series(0, 1000000) s(i)" + ) + cur.execute( + "CREATE TABLE t3 AS SELECT i::bigint n FROM generate_series(0, 1000000) s(i)" + ) + + last_flush_lsn = wait_for_last_flush_lsn(env, ep, tenant, timeline) + + # Avoid flakiness when calculating logical size. + flush_ep_to_pageserver(env, ep, tenant, timeline) + + size_after_action_and_insert = client.tenant_size(tenant) + log.info(f"{size_after_action_and_insert=}") + + size_debug_file = open(test_output_dir / f"size_debug_{action}.html", "w") + size_debug = client.tenant_size_debug(tenant) + size_debug_file.write(size_debug) + return size_after_action_and_insert From adde0ecfe03ff2e352650c2b807bcef4d8a2dc49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Thu, 4 Jul 2024 18:59:19 +0200 Subject: [PATCH 041/194] Flatten compression algorithm setting (#8265) This flattens the compression algorithm setting, removing the `Option<_>` wrapping layer and making handling of the setting easier. It also adds a specific setting for *disabled* compression with the continued ability to read copmressed data, giving us the option to more easily back out of a compression rollout, should the need arise, which was one of the limitations of #8238. Implements my suggestion from https://github.com/neondatabase/neon/pull/8238#issuecomment-2206181594 , inspired by Christian's review in https://github.com/neondatabase/neon/pull/8238#pullrequestreview-2156460268 . Part of #5431 --- libs/pageserver_api/src/models.rs | 15 ++++++++++++++- pageserver/src/config.rs | 11 ++++++----- pageserver/src/tenant/blob_io.rs | 18 +++++++++++++----- .../src/tenant/storage_layer/delta_layer.rs | 4 ++-- pageserver/src/tenant/storage_layer/layer.rs | 2 +- 5 files changed, 36 insertions(+), 14 deletions(-) diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index ad65602f54d9..ecc543917e56 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -450,9 +450,22 @@ pub enum CompactionAlgorithm { )] #[strum(serialize_all = "kebab-case")] pub enum ImageCompressionAlgorithm { + /// Disabled for writes, and never decompress during reading. + /// Never set this after you've enabled compression once! + DisabledNoDecompress, + // Disabled for writes, support decompressing during read path + Disabled, /// Zstandard compression. Level 0 means and None mean the same (default level). Levels can be negative as well. /// For details, see the [manual](http://facebook.github.io/zstd/zstd_manual.html). - Zstd { level: Option }, + Zstd { + level: Option, + }, +} + +impl ImageCompressionAlgorithm { + pub fn allow_decompression(&self) -> bool { + !matches!(self, ImageCompressionAlgorithm::DisabledNoDecompress) + } } #[derive(Eq, PartialEq, Debug, Clone, Serialize, Deserialize)] diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index fa7f7d8d97c0..b7c9af224404 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -91,7 +91,8 @@ pub mod defaults { pub const DEFAULT_MAX_VECTORED_READ_BYTES: usize = 128 * 1024; // 128 KiB - pub const DEFAULT_IMAGE_COMPRESSION: Option = None; + pub const DEFAULT_IMAGE_COMPRESSION: ImageCompressionAlgorithm = + ImageCompressionAlgorithm::DisabledNoDecompress; pub const DEFAULT_VALIDATE_VECTORED_GET: bool = true; @@ -288,7 +289,7 @@ pub struct PageServerConf { pub validate_vectored_get: bool, - pub image_compression: Option, + pub image_compression: ImageCompressionAlgorithm, /// How many bytes of ephemeral layer content will we allow per kilobyte of RAM. When this /// is exceeded, we start proactively closing ephemeral layers to limit the total amount @@ -402,7 +403,7 @@ struct PageServerConfigBuilder { validate_vectored_get: BuilderValue, - image_compression: BuilderValue>, + image_compression: BuilderValue, ephemeral_bytes_per_memory_kb: BuilderValue, @@ -680,7 +681,7 @@ impl PageServerConfigBuilder { self.validate_vectored_get = BuilderValue::Set(value); } - pub fn get_image_compression(&mut self, value: Option) { + pub fn get_image_compression(&mut self, value: ImageCompressionAlgorithm) { self.image_compression = BuilderValue::Set(value); } @@ -1028,7 +1029,7 @@ impl PageServerConf { builder.get_validate_vectored_get(parse_toml_bool("validate_vectored_get", item)?) } "image_compression" => { - builder.get_image_compression(Some(parse_toml_from_str("image_compression", item)?)) + builder.get_image_compression(parse_toml_from_str("image_compression", item)?) } "ephemeral_bytes_per_memory_kb" => { builder.get_ephemeral_bytes_per_memory_kb(parse_toml_u64("ephemeral_bytes_per_memory_kb", item)? as usize) diff --git a/pageserver/src/tenant/blob_io.rs b/pageserver/src/tenant/blob_io.rs index 1a6a5702f19b..0705182d5db2 100644 --- a/pageserver/src/tenant/blob_io.rs +++ b/pageserver/src/tenant/blob_io.rs @@ -273,7 +273,12 @@ impl BlobWriter { srcbuf: B, ctx: &RequestContext, ) -> (B::Buf, Result) { - self.write_blob_maybe_compressed(srcbuf, ctx, None).await + self.write_blob_maybe_compressed( + srcbuf, + ctx, + ImageCompressionAlgorithm::DisabledNoDecompress, + ) + .await } /// Write a blob of data. Returns the offset that it was written to, @@ -282,7 +287,7 @@ impl BlobWriter { &mut self, srcbuf: B, ctx: &RequestContext, - algorithm: Option, + algorithm: ImageCompressionAlgorithm, ) -> (B::Buf, Result) { let offset = self.offset; @@ -314,7 +319,7 @@ impl BlobWriter { ); } let (high_bit_mask, len_written, srcbuf) = match algorithm { - Some(ImageCompressionAlgorithm::Zstd { level }) => { + ImageCompressionAlgorithm::Zstd { level } => { let mut encoder = if let Some(level) = level { async_compression::tokio::write::ZstdEncoder::with_quality( Vec::new(), @@ -335,7 +340,10 @@ impl BlobWriter { (BYTE_UNCOMPRESSED, len, slice.into_inner()) } } - None => (BYTE_UNCOMPRESSED, len, srcbuf.slice_full().into_inner()), + ImageCompressionAlgorithm::Disabled + | ImageCompressionAlgorithm::DisabledNoDecompress => { + (BYTE_UNCOMPRESSED, len, srcbuf.slice_full().into_inner()) + } }; let mut len_buf = (len_written as u32).to_be_bytes(); assert_eq!(len_buf[0] & 0xf0, 0); @@ -414,7 +422,7 @@ mod tests { wtr.write_blob_maybe_compressed( blob.clone(), &ctx, - Some(ImageCompressionAlgorithm::Zstd { level: Some(1) }), + ImageCompressionAlgorithm::Zstd { level: Some(1) }, ) .await } else { diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index e6a4d6d5c45a..685f6dce60e7 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -49,7 +49,7 @@ use camino::{Utf8Path, Utf8PathBuf}; use futures::StreamExt; use itertools::Itertools; use pageserver_api::keyspace::KeySpace; -use pageserver_api::models::LayerAccessKind; +use pageserver_api::models::{ImageCompressionAlgorithm, LayerAccessKind}; use pageserver_api::shard::TenantShardId; use rand::{distributions::Alphanumeric, Rng}; use serde::{Deserialize, Serialize}; @@ -453,7 +453,7 @@ impl DeltaLayerWriterInner { ) -> (Vec, anyhow::Result<()>) { assert!(self.lsn_range.start <= lsn); // We don't want to use compression in delta layer creation - let compression = None; + let compression = ImageCompressionAlgorithm::DisabledNoDecompress; let (val, res) = self .blob_writer .write_blob_maybe_compressed(val, ctx, compression) diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index d1f5cc8f43a7..afd11780e77d 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -1685,7 +1685,7 @@ impl DownloadedLayer { lsn, summary, Some(owner.conf.max_vectored_read_bytes), - owner.conf.image_compression.is_some(), + owner.conf.image_compression.allow_decompression(), ctx, ) .await From 88b13d4552fb538ded52624c3daa0883ae272583 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 4 Jul 2024 22:03:58 +0300 Subject: [PATCH 042/194] implement rolling hyper-log-log algorithm (#8068) ## Problem See #7466 ## Summary of changes Implement algorithm descried in https://hal.science/hal-00465313/document Now new GUC is added: `neon.wss_max_duration` which specifies size of sliding window (in seconds). Default value is 1 hour. It is possible to request estimation of working set sizes (within this window using new function `approximate_working_set_size_seconds`. Old function `approximate_working_set_size` is preserved for backward compatibility. But its scope is also limited by `neon.wss_max_duration`. Version of Neon extension is changed to 1.4 ## Checklist before requesting a review - [ ] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. ## Checklist before merging - [ ] Do not forget to reformat commit message to not include the above checklist --------- Co-authored-by: Konstantin Knizhnik Co-authored-by: Matthias van de Meent --- pgxn/neon/Makefile | 3 +- pgxn/neon/file_cache.c | 42 ++-- pgxn/neon/hll.c | 193 ++++++++++++++++++ pgxn/neon/hll.h | 86 ++++++++ pgxn/neon/neon--1.3--1.4.sql | 9 + pgxn/neon/neon--1.4--1.3.sql | 1 + .../test_lfc_working_set_approximation.py | 44 ++++ test_runner/regress/test_neon_extension.py | 2 +- 8 files changed, 363 insertions(+), 17 deletions(-) create mode 100644 pgxn/neon/hll.c create mode 100644 pgxn/neon/hll.h create mode 100644 pgxn/neon/neon--1.3--1.4.sql create mode 100644 pgxn/neon/neon--1.4--1.3.sql diff --git a/pgxn/neon/Makefile b/pgxn/neon/Makefile index cd316dbb9141..3b755bb0420c 100644 --- a/pgxn/neon/Makefile +++ b/pgxn/neon/Makefile @@ -6,6 +6,7 @@ OBJS = \ $(WIN32RES) \ extension_server.o \ file_cache.o \ + hll.o \ libpagestore.o \ neon.o \ neon_utils.o \ @@ -22,7 +23,7 @@ SHLIB_LINK_INTERNAL = $(libpq) SHLIB_LINK = -lcurl EXTENSION = neon -DATA = neon--1.0.sql neon--1.0--1.1.sql neon--1.1--1.2.sql neon--1.2--1.3.sql neon--1.3--1.2.sql neon--1.2--1.1.sql neon--1.1--1.0.sql +DATA = neon--1.0.sql neon--1.0--1.1.sql neon--1.1--1.2.sql neon--1.2--1.3.sql neon--1.3--1.2.sql neon--1.2--1.1.sql neon--1.1--1.0.sql neon--1.3--1.4.sql neon--1.4--1.3.sql PGFILEDESC = "neon - cloud storage for PostgreSQL" EXTRA_CLEAN = \ diff --git a/pgxn/neon/file_cache.c b/pgxn/neon/file_cache.c index 25275ef31fe9..1894e8c72a5c 100644 --- a/pgxn/neon/file_cache.c +++ b/pgxn/neon/file_cache.c @@ -26,7 +26,6 @@ #include "miscadmin.h" #include "pagestore_client.h" #include "common/hashfn.h" -#include "lib/hyperloglog.h" #include "pgstat.h" #include "postmaster/bgworker.h" #include RELFILEINFO_HDR @@ -40,6 +39,8 @@ #include "utils/dynahash.h" #include "utils/guc.h" +#include "hll.h" + /* * Local file cache is used to temporary store relations pages in local file system. * All blocks of all relations are stored inside one file and addressed using shared hash map. @@ -62,7 +63,6 @@ #define BLOCKS_PER_CHUNK 128 /* 1Mb chunk */ #define MB ((uint64)1024*1024) -#define HYPER_LOG_LOG_BIT_WIDTH 10 #define SIZE_MB_TO_CHUNKS(size) ((uint32)((size) * MB / BLCKSZ / BLOCKS_PER_CHUNK)) typedef struct FileCacheEntry @@ -87,8 +87,7 @@ typedef struct FileCacheControl uint64 writes; dlist_head lru; /* double linked list for LRU replacement * algorithm */ - hyperLogLogState wss_estimation; /* estimation of wroking set size */ - uint8_t hyperloglog_hashes[(1 << HYPER_LOG_LOG_BIT_WIDTH) + 1]; + HyperLogLogState wss_estimation; /* estimation of working set size */ } FileCacheControl; static HTAB *lfc_hash; @@ -238,12 +237,7 @@ lfc_shmem_startup(void) dlist_init(&lfc_ctl->lru); /* Initialize hyper-log-log structure for estimating working set size */ - initHyperLogLog(&lfc_ctl->wss_estimation, HYPER_LOG_LOG_BIT_WIDTH); - - /* We need hashes in shared memory */ - pfree(lfc_ctl->wss_estimation.hashesArr); - memset(lfc_ctl->hyperloglog_hashes, 0, sizeof lfc_ctl->hyperloglog_hashes); - lfc_ctl->wss_estimation.hashesArr = lfc_ctl->hyperloglog_hashes; + initSHLL(&lfc_ctl->wss_estimation); /* Recreate file cache on restart */ fd = BasicOpenFile(lfc_path, O_RDWR | O_CREAT | O_TRUNC); @@ -545,7 +539,7 @@ lfc_read(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, /* Approximate working set */ tag.blockNum = blkno; - addHyperLogLog(&lfc_ctl->wss_estimation, hash_bytes((uint8_t const*)&tag, sizeof(tag))); + addSHLL(&lfc_ctl->wss_estimation, hash_bytes((uint8_t const*)&tag, sizeof(tag))); if (entry == NULL || (entry->bitmap[chunk_offs >> 5] & (1 << (chunk_offs & 31))) == 0) { @@ -986,20 +980,38 @@ local_cache_pages(PG_FUNCTION_ARGS) SRF_RETURN_DONE(funcctx); } +PG_FUNCTION_INFO_V1(approximate_working_set_size_seconds); + +Datum +approximate_working_set_size_seconds(PG_FUNCTION_ARGS) +{ + if (lfc_size_limit != 0) + { + int32 dc; + time_t duration = PG_ARGISNULL(0) ? (time_t)-1 : PG_GETARG_INT32(0); + LWLockAcquire(lfc_lock, LW_SHARED); + dc = (int32) estimateSHLL(&lfc_ctl->wss_estimation, duration); + LWLockRelease(lfc_lock); + PG_RETURN_INT32(dc); + } + PG_RETURN_NULL(); +} + PG_FUNCTION_INFO_V1(approximate_working_set_size); Datum approximate_working_set_size(PG_FUNCTION_ARGS) { - int32 dc = -1; if (lfc_size_limit != 0) { + int32 dc; bool reset = PG_GETARG_BOOL(0); LWLockAcquire(lfc_lock, reset ? LW_EXCLUSIVE : LW_SHARED); - dc = (int32) estimateHyperLogLog(&lfc_ctl->wss_estimation); + dc = (int32) estimateSHLL(&lfc_ctl->wss_estimation, (time_t)-1); if (reset) - memset(lfc_ctl->hyperloglog_hashes, 0, sizeof lfc_ctl->hyperloglog_hashes); + memset(lfc_ctl->wss_estimation.regs, 0, sizeof lfc_ctl->wss_estimation.regs); LWLockRelease(lfc_lock); + PG_RETURN_INT32(dc); } - PG_RETURN_INT32(dc); + PG_RETURN_NULL(); } diff --git a/pgxn/neon/hll.c b/pgxn/neon/hll.c new file mode 100644 index 000000000000..f8496b31259d --- /dev/null +++ b/pgxn/neon/hll.c @@ -0,0 +1,193 @@ +/*------------------------------------------------------------------------- + * + * hll.c + * Sliding HyperLogLog cardinality estimator + * + * Portions Copyright (c) 2014-2023, PostgreSQL Global Development Group + * + * Implements https://hal.science/hal-00465313/document + * + * Based on Hideaki Ohno's C++ implementation. This is probably not ideally + * suited to estimating the cardinality of very large sets; in particular, we + * have not attempted to further optimize the implementation as described in + * the Heule, Nunkesser and Hall paper "HyperLogLog in Practice: Algorithmic + * Engineering of a State of The Art Cardinality Estimation Algorithm". + * + * A sparse representation of HyperLogLog state is used, with fixed space + * overhead. + * + * The copyright terms of Ohno's original version (the MIT license) follow. + * + * IDENTIFICATION + * src/backend/lib/hyperloglog.c + * + *------------------------------------------------------------------------- + */ + +/* + * Copyright (c) 2013 Hideaki Ohno + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the 'Software'), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include + +#include "postgres.h" +#include "funcapi.h" +#include "port/pg_bitutils.h" +#include "utils/timestamp.h" +#include "hll.h" + + +#define POW_2_32 (4294967296.0) +#define NEG_POW_2_32 (-4294967296.0) + +#define ALPHA_MM ((0.7213 / (1.0 + 1.079 / HLL_N_REGISTERS)) * HLL_N_REGISTERS * HLL_N_REGISTERS) + +/* + * Worker for addHyperLogLog(). + * + * Calculates the position of the first set bit in first b bits of x argument + * starting from the first, reading from most significant to least significant + * bits. + * + * Example (when considering fist 10 bits of x): + * + * rho(x = 0b1000000000) returns 1 + * rho(x = 0b0010000000) returns 3 + * rho(x = 0b0000000000) returns b + 1 + * + * "The binary address determined by the first b bits of x" + * + * Return value "j" used to index bit pattern to watch. + */ +static inline uint8 +rho(uint32 x, uint8 b) +{ + uint8 j = 1; + + if (x == 0) + return b + 1; + + j = 32 - pg_leftmost_one_pos32(x); + + if (j > b) + return b + 1; + + return j; +} + +/* + * Initialize HyperLogLog track state + */ +void +initSHLL(HyperLogLogState *cState) +{ + memset(cState->regs, 0, sizeof(cState->regs)); +} + +/* + * Adds element to the estimator, from caller-supplied hash. + * + * It is critical that the hash value passed be an actual hash value, typically + * generated using hash_any(). The algorithm relies on a specific bit-pattern + * observable in conjunction with stochastic averaging. There must be a + * uniform distribution of bits in hash values for each distinct original value + * observed. + */ +void +addSHLL(HyperLogLogState *cState, uint32 hash) +{ + uint8 count; + uint32 index; + size_t i; + size_t j; + + TimestampTz now = GetCurrentTimestamp(); + /* Use the first "k" (registerWidth) bits as a zero based index */ + index = hash >> HLL_C_BITS; + + /* Compute the rank of the remaining 32 - "k" (registerWidth) bits */ + count = rho(hash << HLL_BIT_WIDTH, HLL_C_BITS); + + cState->regs[index][count] = now; +} + +static uint8 +getMaximum(const TimestampTz* reg, TimestampTz since) +{ + uint8 max = 0; + + for (size_t i = 0; i < HLL_C_BITS + 1; i++) + { + if (reg[i] >= since) + { + max = i; + } + } + + return max; +} + + +/* + * Estimates cardinality, based on elements added so far + */ +double +estimateSHLL(HyperLogLogState *cState, time_t duration) +{ + double result; + double sum = 0.0; + size_t i; + uint8 R[HLL_N_REGISTERS]; + /* 0 indicates uninitialized timestamp, so if we need to cover the whole range than starts with 1 */ + TimestampTz since = duration == (time_t)-1 ? 1 : GetCurrentTimestamp() - duration * USECS_PER_SEC; + + for (i = 0; i < HLL_N_REGISTERS; i++) + { + R[i] = getMaximum(cState->regs[i], since); + sum += 1.0 / pow(2.0, R[i]); + } + + /* result set to "raw" HyperLogLog estimate (E in the HyperLogLog paper) */ + result = ALPHA_MM / sum; + + if (result <= (5.0 / 2.0) * HLL_N_REGISTERS) + { + /* Small range correction */ + int zero_count = 0; + + for (i = 0; i < HLL_N_REGISTERS; i++) + { + zero_count += R[i] == 0; + } + + if (zero_count != 0) + result = HLL_N_REGISTERS * log((double) HLL_N_REGISTERS / + zero_count); + } + else if (result > (1.0 / 30.0) * POW_2_32) + { + /* Large range correction */ + result = NEG_POW_2_32 * log(1.0 - (result / POW_2_32)); + } + + return result; +} + diff --git a/pgxn/neon/hll.h b/pgxn/neon/hll.h new file mode 100644 index 000000000000..9256cb9afa2f --- /dev/null +++ b/pgxn/neon/hll.h @@ -0,0 +1,86 @@ +/*------------------------------------------------------------------------- + * + * hll.h + * Sliding HyperLogLog cardinality estimator + * + * Portions Copyright (c) 2014-2023, PostgreSQL Global Development Group + * + * Implements https://hal.science/hal-00465313/document + * + * Based on Hideaki Ohno's C++ implementation. This is probably not ideally + * suited to estimating the cardinality of very large sets; in particular, we + * have not attempted to further optimize the implementation as described in + * the Heule, Nunkesser and Hall paper "HyperLogLog in Practice: Algorithmic + * Engineering of a State of The Art Cardinality Estimation Algorithm". + * + * A sparse representation of HyperLogLog state is used, with fixed space + * overhead. + * + * The copyright terms of Ohno's original version (the MIT license) follow. + * + * IDENTIFICATION + * src/backend/lib/hyperloglog.c + * + *------------------------------------------------------------------------- + */ + +/* + * Copyright (c) 2013 Hideaki Ohno + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the 'Software'), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#ifndef HLL_H +#define HLL_H + +#define HLL_BIT_WIDTH 10 +#define HLL_C_BITS (32 - HLL_BIT_WIDTH) +#define HLL_N_REGISTERS (1 << HLL_BIT_WIDTH) + +/* + * HyperLogLog is an approximate technique for computing the number of distinct + * entries in a set. Importantly, it does this by using a fixed amount of + * memory. See the 2007 paper "HyperLogLog: the analysis of a near-optimal + * cardinality estimation algorithm" for more. + * + * Instead of a single counter for every bits register, we have a timestamp + * for every valid number of bits we can encounter. Every time we encounter + * a certain number of bits, we update the timestamp in those registers to + * the current timestamp. + * + * We can query the sketch's stored cardinality for the range of some timestamp + * up to now: For each register, we return the highest bits bucket that has a + * modified timestamp >= the query timestamp. This value is the number of bits + * for this register in the normal HLL calculation. + * + * The memory usage is 2^B * (C + 1) * sizeof(TimetampTz), or 184kiB. + * Usage could be halved if we decide to reduce the required time dimension + * precision; as 32 bits in second precision should be enough for statistics. + * However, that is not yet implemented. + */ +typedef struct HyperLogLogState +{ + TimestampTz regs[HLL_N_REGISTERS][HLL_C_BITS + 1]; +} HyperLogLogState; + +extern void initSHLL(HyperLogLogState *cState); +extern void addSHLL(HyperLogLogState *cState, uint32 hash); +extern double estimateSHLL(HyperLogLogState *cState, time_t dutration); + +#endif diff --git a/pgxn/neon/neon--1.3--1.4.sql b/pgxn/neon/neon--1.3--1.4.sql new file mode 100644 index 000000000000..042effe3461c --- /dev/null +++ b/pgxn/neon/neon--1.3--1.4.sql @@ -0,0 +1,9 @@ +\echo Use "ALTER EXTENSION neon UPDATE TO '1.4'" to load this file. \quit + +CREATE FUNCTION approximate_working_set_size_seconds(duration integer default null) +RETURNS integer +AS 'MODULE_PATHNAME', 'approximate_working_set_size_seconds' +LANGUAGE C PARALLEL SAFE; + +GRANT EXECUTE ON FUNCTION approximate_working_set_size_seconds(integer) TO pg_monitor; + diff --git a/pgxn/neon/neon--1.4--1.3.sql b/pgxn/neon/neon--1.4--1.3.sql new file mode 100644 index 000000000000..bea72d1a6b17 --- /dev/null +++ b/pgxn/neon/neon--1.4--1.3.sql @@ -0,0 +1 @@ +DROP FUNCTION IF EXISTS approximate_working_set_size_seconds(integer) CASCADE; diff --git a/test_runner/regress/test_lfc_working_set_approximation.py b/test_runner/regress/test_lfc_working_set_approximation.py index a6f05fe0f712..6465bdfd217d 100644 --- a/test_runner/regress/test_lfc_working_set_approximation.py +++ b/test_runner/regress/test_lfc_working_set_approximation.py @@ -1,3 +1,4 @@ +import time from pathlib import Path from fixtures.log_helper import log @@ -72,3 +73,46 @@ def test_lfc_working_set_approximation(neon_simple_env: NeonEnv): blocks = query_scalar(cur, "select approximate_working_set_size(true)") log.info(f"working set size after some index access of a few select pages only {blocks}") assert blocks < 10 + + +def test_sliding_working_set_approximation(neon_simple_env: NeonEnv): + env = neon_simple_env + + endpoint = env.endpoints.create_start( + branch_name="main", + config_lines=[ + "autovacuum = off", + "shared_buffers=1MB", + "neon.max_file_cache_size=256MB", + "neon.file_cache_size_limit=245MB", + ], + ) + conn = endpoint.connect() + cur = conn.cursor() + cur.execute("create extension neon version '1.4'") + cur.execute( + "create table t(pk integer primary key, count integer default 0, payload text default repeat('?', 128))" + ) + cur.execute("insert into t (pk) values (generate_series(1,1000000))") + time.sleep(2) + before_10k = time.monotonic() + cur.execute("select sum(count) from t where pk between 10000 and 20000") + time.sleep(2) + before_1k = time.monotonic() + cur.execute("select sum(count) from t where pk between 1000 and 2000") + after = time.monotonic() + + cur.execute(f"select approximate_working_set_size_seconds({int(after - before_1k + 1)})") + estimation_1k = cur.fetchall()[0][0] + log.info(f"Working set size for selecting 1k records {estimation_1k}") + + cur.execute(f"select approximate_working_set_size_seconds({int(after - before_10k + 1)})") + estimation_10k = cur.fetchall()[0][0] + log.info(f"Working set size for selecting 10k records {estimation_10k}") + + cur.execute("select pg_table_size('t')") + size = cur.fetchall()[0][0] // 8192 + log.info(f"Table size {size} blocks") + + assert estimation_1k >= 20 and estimation_1k <= 40 + assert estimation_10k >= 200 and estimation_10k <= 400 diff --git a/test_runner/regress/test_neon_extension.py b/test_runner/regress/test_neon_extension.py index 39b486502672..e83aaf91c60f 100644 --- a/test_runner/regress/test_neon_extension.py +++ b/test_runner/regress/test_neon_extension.py @@ -50,7 +50,7 @@ def test_neon_extension_compatibility(neon_env_builder: NeonEnvBuilder): # Ensure that the default version is also updated in the neon.control file assert cur.fetchone() == ("1.3",) cur.execute("SELECT * from neon.NEON_STAT_FILE_CACHE") - all_versions = ["1.3", "1.2", "1.1", "1.0"] + all_versions = ["1.4", "1.3", "1.2", "1.1", "1.0"] current_version = "1.3" for idx, begin_version in enumerate(all_versions): for target_version in all_versions[idx + 1 :]: From 711716c72506cdf05ce3a4cd755b007439de86e9 Mon Sep 17 00:00:00 2001 From: Peter Bendel Date: Thu, 4 Jul 2024 22:17:45 +0200 Subject: [PATCH 043/194] add checkout depth1 to workflow to access local github actions like generate allure report (#8259) ## Problem job step to create allure report fails https://github.com/neondatabase/neon/actions/runs/9781886710/job/27006997416#step:11:1 ## Summary of changes Shallow checkout of sources to get access to local github action needed in the job step ## Example run example run with this change https://github.com/neondatabase/neon/actions/runs/9790647724 do not merge this PR until the job is clean --------- Co-authored-by: Alexander Bayandin --- .github/workflows/periodic_pagebench.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/periodic_pagebench.yml b/.github/workflows/periodic_pagebench.yml index c0219599a2e7..a8baf6bf7aac 100644 --- a/.github/workflows/periodic_pagebench.yml +++ b/.github/workflows/periodic_pagebench.yml @@ -43,6 +43,10 @@ jobs: AWS_DEFAULT_REGION : "eu-central-1" AWS_INSTANCE_ID : "i-02a59a3bf86bc7e74" steps: + # we don't need the neon source code because we run everything remotely + # however we still need the local github actions to run the allure step below + - uses: actions/checkout@v4 + - name: Show my own (github runner) external IP address - usefull for IP allowlisting run: curl https://ifconfig.me @@ -116,6 +120,9 @@ jobs: cat "test_log_${GITHUB_RUN_ID}" - name: Create Allure report + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }} if: ${{ !cancelled() }} uses: ./.github/actions/allure-report-generate From e25ac31fc9d18d312ec83decb3ceed82cbbf6119 Mon Sep 17 00:00:00 2001 From: John Spray Date: Fri, 5 Jul 2024 10:09:15 +0100 Subject: [PATCH 044/194] tests: extend allow list in deletion test (#8268) ## Problem 1ea5d8b1327d2e93cbe11682f60a90e35d42d1ee tolerated this as an error message, but it can show up in logs as well. Example failure: https://neon-github-public-dev.s3.amazonaws.com/reports/pr-8201/9780147712/index.html#testresult/263422f5f5f292ea/retries ## Summary of changes - Tolerate "failed to delete 1 objects" in pageserver logs, this occurs occasionally when injected failures exhaust deletion's retries. --- test_runner/regress/test_tenant_delete.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test_runner/regress/test_tenant_delete.py b/test_runner/regress/test_tenant_delete.py index d3fba32a19e0..1d7c8b8e31f0 100644 --- a/test_runner/regress/test_tenant_delete.py +++ b/test_runner/regress/test_tenant_delete.py @@ -67,8 +67,9 @@ def test_tenant_delete_smoke( # first try to delete non existing tenant tenant_id = TenantId.generate() - env.pageserver.allowed_errors.append(".*NotFound.*") - env.pageserver.allowed_errors.append(".*simulated failure.*") + env.pageserver.allowed_errors.extend( + [".*NotFound.*", ".*simulated failure.*", ".*failed to delete .+ objects.*"] + ) # Check that deleting a non-existent tenant gives the expected result: this is a loop because we # may need to retry on some remote storage errors injected by the test harness From 6876f0d06616851a694ad36bfec11d83e71cc49a Mon Sep 17 00:00:00 2001 From: Peter Bendel Date: Fri, 5 Jul 2024 11:23:46 +0200 Subject: [PATCH 045/194] correct error handling for periodic pagebench runner status (#8274) ## Problem the following periodic pagebench run was failed but was still shown as successful https://github.com/neondatabase/neon/actions/runs/9798909458/job/27058179993#step:9:47 ## Summary of changes if the ec2 test runner reports a failure fail the job step and thus the workflow --------- Co-authored-by: Alexander Bayandin --- .github/workflows/periodic_pagebench.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/periodic_pagebench.yml b/.github/workflows/periodic_pagebench.yml index a8baf6bf7aac..ed4e6be71239 100644 --- a/.github/workflows/periodic_pagebench.yml +++ b/.github/workflows/periodic_pagebench.yml @@ -94,10 +94,12 @@ jobs: set +x status=$(echo $response | jq -r '.status') echo "Test status: $status" - if [[ "$status" == "failure" || "$status" == "success" || "$status" == "null" ]]; then + if [[ "$status" == "failure" ]]; then + echo "Test failed" + exit 1 # Fail the job step if status is failure + elif [[ "$status" == "success" || "$status" == "null" ]]; then break - fi - if [[ "$status" == "too_many_runs" ]]; then + elif [[ "$status" == "too_many_runs" ]]; then echo "Too many runs already running" echo "too_many_runs=true" >> "$GITHUB_OUTPUT" exit 1 @@ -107,6 +109,7 @@ jobs: done - name: Retrieve Test Logs + if: always() && steps.poll_step.outputs.too_many_runs != 'true' run: | curl -k -X 'GET' \ "${EC2_MACHINE_URL_US}/test_log/${GITHUB_RUN_ID}" \ @@ -115,6 +118,7 @@ jobs: --output "test_log_${GITHUB_RUN_ID}.gz" - name: Unzip Test Log and Print it into this job's log + if: always() && steps.poll_step.outputs.too_many_runs != 'true' run: | gzip -d "test_log_${GITHUB_RUN_ID}.gz" cat "test_log_${GITHUB_RUN_ID}" From 5aae80640b5d0fe20502c0c3b32dd6ffa02456b9 Mon Sep 17 00:00:00 2001 From: John Spray Date: Fri, 5 Jul 2024 10:34:16 +0100 Subject: [PATCH 046/194] tests: make location_conf_churn more robust (#8271) ## Problem This test directly manages locations on pageservers and configuration of an endpoint. However, it did not switch off the parts of the storage controller that attempt to do the same: occasionally, the test would fail in a strange way such as a compute failing to accept a reconfiguration request. ## Summary of changes - Wire up the storage controller's compute notification hook to a no-op handler - Configure the tenant's scheduling policy to Stop. --- .../regress/test_pageserver_secondary.py | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/test_runner/regress/test_pageserver_secondary.py b/test_runner/regress/test_pageserver_secondary.py index 4c828b86b053..0416078ebc67 100644 --- a/test_runner/regress/test_pageserver_secondary.py +++ b/test_runner/regress/test_pageserver_secondary.py @@ -16,6 +16,8 @@ from fixtures.remote_storage import LocalFsStorage, RemoteStorageKind, S3Storage, s3_storage from fixtures.utils import wait_until from fixtures.workload import Workload +from werkzeug.wrappers.request import Request +from werkzeug.wrappers.response import Response # A tenant configuration that is convenient for generating uploads and deletions # without a large amount of postgres traffic. @@ -59,7 +61,7 @@ def evict_random_layers( @pytest.mark.parametrize("seed", [1, 2, 3]) -def test_location_conf_churn(neon_env_builder: NeonEnvBuilder, seed: int): +def test_location_conf_churn(neon_env_builder: NeonEnvBuilder, make_httpserver, seed: int): """ Issue many location configuration changes, ensure that tenants remain readable & we don't get any unexpected errors. We should @@ -73,6 +75,20 @@ def test_location_conf_churn(neon_env_builder: NeonEnvBuilder, seed: int): neon_env_builder.enable_pageserver_remote_storage( remote_storage_kind=s3_storage(), ) + neon_env_builder.control_plane_compute_hook_api = ( + f"http://{make_httpserver.host}:{make_httpserver.port}/notify-attach" + ) + + def ignore_notify(request: Request): + # This test does all its own compute configuration (by passing explicit pageserver ID to Workload functions), + # so we send controller notifications to /dev/null to prevent it fighting the test for control of the compute. + log.info(f"Ignoring storage controller compute notification: {request.json}") + return Response(status=200) + + make_httpserver.expect_request("/notify-attach", method="PUT").respond_with_handler( + ignore_notify + ) + env = neon_env_builder.init_start(initial_tenant_conf=TENANT_CONF) pageservers = env.pageservers @@ -99,6 +115,15 @@ def test_location_conf_churn(neon_env_builder: NeonEnvBuilder, seed: int): workload.init(env.pageservers[0].id) workload.write_rows(256, env.pageservers[0].id) + # Discourage the storage controller from interfering with the changes we will make directly on the pageserver + env.storage_controller.tenant_policy_update( + tenant_id, + { + "scheduling": "Stop", + }, + ) + env.storage_controller.allowed_errors.append(".*Scheduling is disabled by policy Stop.*") + # We use a fixed seed to make the test reproducible: we want a randomly # chosen order, but not to change the order every time we run the test. rng = random.Random(seed) From 6849ae4810e9a678dfc301f7118c4ce152a0c484 Mon Sep 17 00:00:00 2001 From: John Spray Date: Fri, 5 Jul 2024 11:17:44 +0100 Subject: [PATCH 047/194] safekeeper: add separate `tombstones` map for deleted timelines (#8253) ## Problem Safekeepers left running for a long time use a lot of memory (up to the point of OOMing, on small nodes) for deleted timelines, because the `Timeline` struct is kept alive as a guard against recreating deleted timelines. Closes: https://github.com/neondatabase/neon/issues/6810 ## Summary of changes - Create separate tombstones that just record a ttid and when the timeline was deleted. - Add a periodic housekeeping task that cleans up tombstones older than a hardcoded TTL (24h) I think this also makes https://github.com/neondatabase/neon/pull/6766 un-needed, as the tombstone is also checked during deletion. I considered making the overall timeline map use an enum type containing active or deleted, but having a separate map of tombstones avoids bloating that map, so that calls like `get()` can still go straight to a timeline without having to walk a hashmap that also contains tombstones. --- safekeeper/src/bin/safekeeper.rs | 13 +++ safekeeper/src/timelines_global_map.rs | 105 +++++++++++++++++-------- 2 files changed, 87 insertions(+), 31 deletions(-) diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index d25b8722ac23..4d580e57ed7e 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -445,6 +445,19 @@ async fn start_safekeeper(conf: SafeKeeperConf) -> Result<()> { .map(|res| ("WAL service main".to_owned(), res)); tasks_handles.push(Box::pin(wal_service_handle)); + let timeline_housekeeping_handle = current_thread_rt + .as_ref() + .unwrap_or_else(|| WAL_SERVICE_RUNTIME.handle()) + .spawn(async move { + const TOMBSTONE_TTL: Duration = Duration::from_secs(3600 * 24); + loop { + tokio::time::sleep(TOMBSTONE_TTL).await; + GlobalTimelines::housekeeping(&TOMBSTONE_TTL); + } + }) + .map(|res| ("Timeline map housekeeping".to_owned(), res)); + tasks_handles.push(Box::pin(timeline_housekeeping_handle)); + if let Some(pg_listener_tenant_only) = pg_listener_tenant_only { let conf_ = conf.clone(); let wal_service_handle = current_thread_rt diff --git a/safekeeper/src/timelines_global_map.rs b/safekeeper/src/timelines_global_map.rs index 9ce1112cec43..f57da5c7cbf1 100644 --- a/safekeeper/src/timelines_global_map.rs +++ b/safekeeper/src/timelines_global_map.rs @@ -15,12 +15,19 @@ use std::collections::HashMap; use std::str::FromStr; use std::sync::atomic::Ordering; use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; use tracing::*; use utils::id::{TenantId, TenantTimelineId, TimelineId}; use utils::lsn::Lsn; struct GlobalTimelinesState { timelines: HashMap>, + + // A tombstone indicates this timeline used to exist has been deleted. These are used to prevent + // on-demand timeline creation from recreating deleted timelines. This is only soft-enforced, as + // this map is dropped on restart. + tombstones: HashMap, + conf: Option, broker_active_set: Arc, load_lock: Arc>, @@ -64,11 +71,17 @@ impl GlobalTimelinesState { .cloned() .ok_or(TimelineError::NotFound(*ttid)) } + + fn delete(&mut self, ttid: TenantTimelineId) { + self.timelines.remove(&ttid); + self.tombstones.insert(ttid, Instant::now()); + } } static TIMELINES_STATE: Lazy> = Lazy::new(|| { Mutex::new(GlobalTimelinesState { timelines: HashMap::new(), + tombstones: HashMap::new(), conf: None, broker_active_set: Arc::new(TimelinesSet::default()), load_lock: Arc::new(tokio::sync::Mutex::new(TimelineLoadLock)), @@ -198,11 +211,17 @@ impl GlobalTimelines { let tli = Arc::new(timeline); // TODO: prevent concurrent timeline creation/loading - TIMELINES_STATE - .lock() - .unwrap() - .timelines - .insert(ttid, tli.clone()); + { + let mut state = TIMELINES_STATE.lock().unwrap(); + + // We may be have been asked to load a timeline that was previously deleted (e.g. from `pull_timeline.rs`). We trust + // that the human doing this manual intervention knows what they are doing, and remove its tombstone. + if state.tombstones.remove(&ttid).is_some() { + warn!("Un-deleted timeline {ttid}"); + } + + state.timelines.insert(ttid, tli.clone()); + } tli.bootstrap(&conf, broker_active_set, partial_backup_rate_limiter); @@ -229,7 +248,7 @@ impl GlobalTimelines { /// Create a new timeline with the given id. If the timeline already exists, returns /// an existing timeline. - pub async fn create( + pub(crate) async fn create( ttid: TenantTimelineId, server_info: ServerInfo, commit_lsn: Lsn, @@ -241,6 +260,11 @@ impl GlobalTimelines { // Timeline already exists, return it. return Ok(timeline); } + + if state.tombstones.contains_key(&ttid) { + anyhow::bail!("Timeline {ttid} is deleted, refusing to recreate"); + } + state.get_dependencies() }; @@ -300,17 +324,19 @@ impl GlobalTimelines { /// Get a timeline from the global map. If it's not present, it doesn't exist on disk, /// or was corrupted and couldn't be loaded on startup. Returned timeline is always valid, /// i.e. loaded in memory and not cancelled. - pub fn get(ttid: TenantTimelineId) -> Result, TimelineError> { - let res = TIMELINES_STATE.lock().unwrap().get(&ttid); - - match res { + pub(crate) fn get(ttid: TenantTimelineId) -> Result, TimelineError> { + let tli_res = { + let state = TIMELINES_STATE.lock().unwrap(); + state.get(&ttid) + }; + match tli_res { Ok(tli) => { if tli.is_cancelled() { return Err(TimelineError::Cancelled(ttid)); } Ok(tli) } - _ => res, + _ => tli_res, } } @@ -339,12 +365,26 @@ impl GlobalTimelines { /// Cancels timeline, then deletes the corresponding data directory. /// If only_local, doesn't remove WAL segments in remote storage. - pub async fn delete( + pub(crate) async fn delete( ttid: &TenantTimelineId, only_local: bool, ) -> Result { - let tli_res = TIMELINES_STATE.lock().unwrap().get(ttid); - match tli_res { + let tli_res = { + let state = TIMELINES_STATE.lock().unwrap(); + + if state.tombstones.contains_key(ttid) { + // Presence of a tombstone guarantees that a previous deletion has completed and there is no work to do. + info!("Timeline {ttid} was already deleted"); + return Ok(TimelineDeleteForceResult { + dir_existed: false, + was_active: false, + }); + } + + state.get(ttid) + }; + + let result = match tli_res { Ok(timeline) => { let was_active = timeline.broker_active.load(Ordering::Relaxed); @@ -354,11 +394,6 @@ impl GlobalTimelines { info!("deleting timeline {}, only_local={}", ttid, only_local); let dir_existed = timeline.delete(&mut shared_state, only_local).await?; - // Remove timeline from the map. - // FIXME: re-enable it once we fix the issue with recreation of deleted timelines - // https://github.com/neondatabase/neon/issues/3146 - // TIMELINES_STATE.lock().unwrap().timelines.remove(ttid); - Ok(TimelineDeleteForceResult { dir_existed, was_active, // TODO: we probably should remove this field @@ -374,7 +409,14 @@ impl GlobalTimelines { was_active: false, }) } - } + }; + + // Finalize deletion, by dropping Timeline objects and storing smaller tombstones. The tombstones + // are used to prevent still-running computes from re-creating the same timeline when they send data, + // and to speed up repeated deletion calls by avoiding re-listing objects. + TIMELINES_STATE.lock().unwrap().delete(*ttid); + + result } /// Deactivates and deletes all timelines for the tenant. Returns map of all timelines which @@ -420,19 +462,20 @@ impl GlobalTimelines { tenant_id, ))?; - // FIXME: we temporarily disabled removing timelines from the map, see `delete_force` - // let tlis_after_delete = Self::get_all_for_tenant(*tenant_id); - // if !tlis_after_delete.is_empty() { - // // Some timelines were created while we were deleting them, returning error - // // to the caller, so it can retry later. - // bail!( - // "failed to delete all timelines for tenant {}: some timelines were created while we were deleting them", - // tenant_id - // ); - // } - Ok(deleted) } + + pub fn housekeeping(tombstone_ttl: &Duration) { + let mut state = TIMELINES_STATE.lock().unwrap(); + + // We keep tombstones long enough to have a good chance of preventing rogue computes from re-creating deleted + // timelines. If a compute kept running for longer than this TTL (or across a safekeeper restart) then they + // may recreate a deleted timeline. + let now = Instant::now(); + state + .tombstones + .retain(|_, v| now.duration_since(*v) < *tombstone_ttl); + } } #[derive(Clone, Copy, Serialize)] From 7dd2e447d3aa44b8e3e55a6f4cca39c295dc80e7 Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Fri, 5 Jul 2024 14:02:02 +0100 Subject: [PATCH 048/194] pageserver: add time based image layer creation check (#8247) ## Problem Assume a timeline with the following workload: very slow ingest of updates to a small number of keys that fit within the same partition (as decided by `KeySpace::partition`). These tenants will create small L0 layers since due to time based rolling, and, consequently, the L1 layers will also be small. Currently, by default, we need to ingest 512 MiB of WAL before checking if an image layer is required. This scheme works fine under the assumption that L1s are roughly of checkpoint distance size, but as the first paragraph explained, that's not the case for all workloads. ## Summary of changes Check if new image layers are required at least once every checkpoint timeout interval. --- pageserver/src/tenant/timeline.rs | 71 ++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 16 deletions(-) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 42e55ab2695c..92baf1073aae 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -365,6 +365,7 @@ pub struct Timeline { repartition_threshold: u64, last_image_layer_creation_check_at: AtomicLsn, + last_image_layer_creation_check_instant: std::sync::Mutex>, /// Current logical size of the "datadir", at the last LSN. current_logical_size: LogicalSize, @@ -2384,6 +2385,7 @@ impl Timeline { )), repartition_threshold: 0, last_image_layer_creation_check_at: AtomicLsn::new(0), + last_image_layer_creation_check_instant: Mutex::new(None), last_received_wal: Mutex::new(None), rel_size_cache: RwLock::new(RelSizeCache { @@ -4464,6 +4466,58 @@ impl Timeline { } } + /// Predicate function which indicates whether we should check if new image layers + /// are required. Since checking if new image layers are required is expensive in + /// terms of CPU, we only do it in the following cases: + /// 1. If the timeline has ingested sufficient WAL to justify the cost + /// 2. If enough time has passed since the last check + /// 2.1. For large tenants, we wish to perform the check more often since they + /// suffer from the lack of image layers + /// 2.2. For small tenants (that can mostly fit in RAM), we use a much longer interval + fn should_check_if_image_layers_required(self: &Arc, lsn: Lsn) -> bool { + const LARGE_TENANT_THRESHOLD: u64 = 2 * 1024 * 1024 * 1024; + + let last_checks_at = self.last_image_layer_creation_check_at.load(); + let distance = lsn + .checked_sub(last_checks_at) + .expect("Attempt to compact with LSN going backwards"); + let min_distance = + self.get_image_layer_creation_check_threshold() as u64 * self.get_checkpoint_distance(); + + let distance_based_decision = distance.0 >= min_distance; + + let mut time_based_decision = false; + let mut last_check_instant = self.last_image_layer_creation_check_instant.lock().unwrap(); + if let CurrentLogicalSize::Exact(logical_size) = self.current_logical_size.current_size() { + let check_required_after = if Into::::into(&logical_size) >= LARGE_TENANT_THRESHOLD + { + self.get_checkpoint_timeout() + } else { + Duration::from_secs(3600 * 48) + }; + + time_based_decision = match *last_check_instant { + Some(last_check) => { + let elapsed = last_check.elapsed(); + elapsed >= check_required_after + } + None => true, + }; + } + + // Do the expensive delta layer counting only if this timeline has ingested sufficient + // WAL since the last check or a checkpoint timeout interval has elapsed since the last + // check. + let decision = distance_based_decision || time_based_decision; + + if decision { + self.last_image_layer_creation_check_at.store(lsn); + *last_check_instant = Some(Instant::now()); + } + + decision + } + #[tracing::instrument(skip_all, fields(%lsn, %mode))] async fn create_image_layers( self: &Arc, @@ -4486,22 +4540,7 @@ impl Timeline { // image layers <100000000..100000099> and <200000000..200000199> are not completely covering it. let mut start = Key::MIN; - let check_for_image_layers = { - let last_checks_at = self.last_image_layer_creation_check_at.load(); - let distance = lsn - .checked_sub(last_checks_at) - .expect("Attempt to compact with LSN going backwards"); - let min_distance = self.get_image_layer_creation_check_threshold() as u64 - * self.get_checkpoint_distance(); - - // Skip the expensive delta layer counting if this timeline has not ingested sufficient - // WAL since the last check. - distance.0 >= min_distance - }; - - if check_for_image_layers { - self.last_image_layer_creation_check_at.store(lsn); - } + let check_for_image_layers = self.should_check_if_image_layers_required(lsn); for partition in partitioning.parts.iter() { let img_range = start..partition.ranges.last().unwrap().end; From c9fd8d76937c2031fd4fea1cdf661d6cf4f00dc3 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Fri, 5 Jul 2024 15:12:01 +0100 Subject: [PATCH 049/194] =?UTF-8?q?SELECT=20=F0=9F=92=A3();=20(#8270)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem We want to be able to test how our infrastructure reacts on segfaults in Postgres (for example, we collect cores, and get some required logs/metrics, etc) ## Summary of changes - Add `trigger_segfauls` function to `neon_test_utils` to trigger a segfault in Postgres - Add `trigger_panic` function to `neon_test_utils` to trigger SIGABRT (by using `elog(PANIC, ...)) - Fix cleanup logic in regression tests in endpoint crashed --- pgxn/neon_test_utils/Makefile | 2 +- ...tils--1.2.sql => neon_test_utils--1.3.sql} | 18 +++++++++++++++ pgxn/neon_test_utils/neon_test_utils.control | 2 +- pgxn/neon_test_utils/neontest.c | 23 +++++++++++++++++++ test_runner/fixtures/neon_fixtures.py | 18 +++++++++++---- test_runner/regress/test_endpoint_crash.py | 23 +++++++++++++++++++ 6 files changed, 80 insertions(+), 6 deletions(-) rename pgxn/neon_test_utils/{neon_test_utils--1.2.sql => neon_test_utils--1.3.sql} (77%) create mode 100644 test_runner/regress/test_endpoint_crash.py diff --git a/pgxn/neon_test_utils/Makefile b/pgxn/neon_test_utils/Makefile index 13712724399d..252810b5b02e 100644 --- a/pgxn/neon_test_utils/Makefile +++ b/pgxn/neon_test_utils/Makefile @@ -7,7 +7,7 @@ OBJS = \ neontest.o EXTENSION = neon_test_utils -DATA = neon_test_utils--1.2.sql +DATA = neon_test_utils--1.3.sql PGFILEDESC = "neon_test_utils - helpers for neon testing and debugging" PG_CONFIG = pg_config diff --git a/pgxn/neon_test_utils/neon_test_utils--1.2.sql b/pgxn/neon_test_utils/neon_test_utils--1.3.sql similarity index 77% rename from pgxn/neon_test_utils/neon_test_utils--1.2.sql rename to pgxn/neon_test_utils/neon_test_utils--1.3.sql index f84a24ec8d48..3b8794a8cff4 100644 --- a/pgxn/neon_test_utils/neon_test_utils--1.2.sql +++ b/pgxn/neon_test_utils/neon_test_utils--1.3.sql @@ -45,3 +45,21 @@ CREATE FUNCTION neon_xlogflush(lsn pg_lsn DEFAULT NULL) RETURNS VOID AS 'MODULE_PATHNAME', 'neon_xlogflush' LANGUAGE C PARALLEL UNSAFE; + +CREATE FUNCTION trigger_panic() +RETURNS VOID +AS 'MODULE_PATHNAME', 'trigger_panic' +LANGUAGE C PARALLEL UNSAFE; + +CREATE FUNCTION trigger_segfault() +RETURNS VOID +AS 'MODULE_PATHNAME', 'trigger_segfault' +LANGUAGE C PARALLEL UNSAFE; + +-- Alias for `trigger_segfault`, just because `SELECT 💣()` looks fun +CREATE OR REPLACE FUNCTION 💣() RETURNS void +LANGUAGE plpgsql AS $$ +BEGIN + PERFORM trigger_segfault(); +END; +$$; diff --git a/pgxn/neon_test_utils/neon_test_utils.control b/pgxn/neon_test_utils/neon_test_utils.control index c7b9191ddc12..f22afd70c4fa 100644 --- a/pgxn/neon_test_utils/neon_test_utils.control +++ b/pgxn/neon_test_utils/neon_test_utils.control @@ -1,6 +1,6 @@ # neon_test_utils extension comment = 'helpers for neon testing and debugging' -default_version = '1.2' +default_version = '1.3' module_pathname = '$libdir/neon_test_utils' relocatable = true trusted = true diff --git a/pgxn/neon_test_utils/neontest.c b/pgxn/neon_test_utils/neontest.c index 071dc122edbd..650ef7405d64 100644 --- a/pgxn/neon_test_utils/neontest.c +++ b/pgxn/neon_test_utils/neontest.c @@ -42,6 +42,8 @@ PG_FUNCTION_INFO_V1(clear_buffer_cache); PG_FUNCTION_INFO_V1(get_raw_page_at_lsn); PG_FUNCTION_INFO_V1(get_raw_page_at_lsn_ex); PG_FUNCTION_INFO_V1(neon_xlogflush); +PG_FUNCTION_INFO_V1(trigger_panic); +PG_FUNCTION_INFO_V1(trigger_segfault); /* * Linkage to functions in neon module. @@ -489,3 +491,24 @@ neon_xlogflush(PG_FUNCTION_ARGS) XLogFlush(lsn); PG_RETURN_VOID(); } + +/* + * Function to trigger panic. + */ +Datum +trigger_panic(PG_FUNCTION_ARGS) +{ + elog(PANIC, "neon_test_utils: panic"); + PG_RETURN_VOID(); +} + +/* + * Function to trigger a segfault. + */ +Datum +trigger_segfault(PG_FUNCTION_ARGS) +{ + int *ptr = NULL; + *ptr = 42; + PG_RETURN_VOID(); +} diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index c002e11c1c08..5fb4d948175f 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -943,6 +943,8 @@ def __exit__( # if the test threw an exception, don't check for errors # as a failing assertion would cause the cleanup below to fail ps_assert_metric_no_errors=(exc_type is None), + # do not fail on endpoint errors to allow the rest of cleanup to proceed + fail_on_endpoint_errors=False, ) cleanup_error = None @@ -1214,11 +1216,11 @@ def start(self, timeout_in_seconds: Optional[int] = None): for f in futs: f.result() - def stop(self, immediate=False, ps_assert_metric_no_errors=False): + def stop(self, immediate=False, ps_assert_metric_no_errors=False, fail_on_endpoint_errors=True): """ After this method returns, there should be no child processes running. """ - self.endpoints.stop_all() + self.endpoints.stop_all(fail_on_endpoint_errors) # Stop storage controller before pageservers: we don't want it to spuriously # detect a pageserver "failure" during test teardown @@ -3899,9 +3901,17 @@ def create( pageserver_id=pageserver_id, ) - def stop_all(self) -> "EndpointFactory": + def stop_all(self, fail_on_error=True) -> "EndpointFactory": + exception = None for ep in self.endpoints: - ep.stop() + try: + ep.stop() + except Exception as e: + log.error(f"Failed to stop endpoint {ep.endpoint_id}: {e}") + exception = e + + if fail_on_error and exception is not None: + raise exception return self diff --git a/test_runner/regress/test_endpoint_crash.py b/test_runner/regress/test_endpoint_crash.py new file mode 100644 index 000000000000..ae3dded437a0 --- /dev/null +++ b/test_runner/regress/test_endpoint_crash.py @@ -0,0 +1,23 @@ +import pytest +from fixtures.neon_fixtures import NeonEnvBuilder + + +@pytest.mark.parametrize( + "sql_func", + [ + "trigger_panic", + "trigger_segfault", + "💣", # calls `trigger_segfault` internally + ], +) +def test_endpoint_crash(neon_env_builder: NeonEnvBuilder, sql_func: str): + """ + Test that triggering crash from neon_test_utils crashes the endpoint + """ + env = neon_env_builder.init_start() + env.neon_cli.create_branch("test_endpoint_crash") + endpoint = env.endpoints.create_start("test_endpoint_crash") + + endpoint.safe_psql("CREATE EXTENSION neon_test_utils;") + with pytest.raises(Exception, match="This probably means the server terminated abnormally"): + endpoint.safe_psql(f"SELECT {sql_func}();") From 13522fb722bdf09a920e8c99b6128490ccf9205b Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Fri, 5 Jul 2024 20:39:10 +0300 Subject: [PATCH 050/194] Increase timeout for wating subscriber caught-up (#8118) ## Problem test_subscriber_restart has quit large failure rate' https://neonprod.grafana.net/d/fddp4rvg7k2dcf/regression-test-failures?orgId=1&var-test_name=test_subscriber_restart&var-max_count=100&var-restrict=false I can be caused by too small timeout (5 seconds) to wait until changes are propagated. Related to #8097 ## Summary of changes Increase timeout to 30 seconds. ## Checklist before requesting a review - [ ] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. ## Checklist before merging - [ ] Do not forget to reformat commit message to not include the above checklist Co-authored-by: Konstantin Knizhnik --- test_runner/regress/test_subscriber_restart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_runner/regress/test_subscriber_restart.py b/test_runner/regress/test_subscriber_restart.py index d7f396262059..91caad722051 100644 --- a/test_runner/regress/test_subscriber_restart.py +++ b/test_runner/regress/test_subscriber_restart.py @@ -54,4 +54,4 @@ def insert_data(pub): pcur.execute(f"INSERT into t values ({n_records}, 0)") n_records += 1 with sub.cursor() as scur: - wait_until(10, 0.5, check_that_changes_propagated) + wait_until(60, 0.5, check_that_changes_propagated) From f0d29a0f3ea68159a02d07f7010416b89cacce56 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Fri, 5 Jul 2024 22:17:05 +0200 Subject: [PATCH 051/194] pageserver_live_connections: track as counter pair (#8227) Generally counter pairs are preferred over gauges. In this case, I found myself asking what the typical rate of accepted page_service connections on a pageserver is, and I couldn't answer it with the gauge metric. There are a few dashboards using this metric: https://github.com/search?q=repo%3Aneondatabase%2Fgrafana-dashboard-export%20pageserver_live_connections&type=code I'll convert them to use the new metric once this PR reaches prod. refs https://github.com/neondatabase/neon/issues/7427 --- pageserver/src/metrics.rs | 10 ++++++---- pageserver/src/page_service.rs | 13 ++++--------- .../timeline/walreceiver/walreceiver_connection.rs | 13 ++++--------- 3 files changed, 14 insertions(+), 22 deletions(-) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 9e9fe7fbb834..59b729363147 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -1456,10 +1456,12 @@ impl<'a, 'c> BasebackupQueryTimeOngoingRecording<'a, 'c> { } } -pub(crate) static LIVE_CONNECTIONS_COUNT: Lazy = Lazy::new(|| { - register_int_gauge_vec!( - "pageserver_live_connections", - "Number of live network connections", +pub(crate) static LIVE_CONNECTIONS: Lazy = Lazy::new(|| { + register_int_counter_pair_vec!( + "pageserver_live_connections_started", + "Number of network connections that we started handling", + "pageserver_live_connections_finished", + "Number of network connections that we finished handling", &["pageserver_connection_kind"] ) .expect("failed to define a metric") diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index a440ad63785b..07365b5eb85e 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -55,7 +55,7 @@ use crate::basebackup::BasebackupError; use crate::context::{DownloadBehavior, RequestContext}; use crate::import_datadir::import_wal_from_tar; use crate::metrics; -use crate::metrics::{ComputeCommandKind, COMPUTE_COMMANDS_COUNTERS, LIVE_CONNECTIONS_COUNT}; +use crate::metrics::{ComputeCommandKind, COMPUTE_COMMANDS_COUNTERS, LIVE_CONNECTIONS}; use crate::pgdatadir_mapping::Version; use crate::span::debug_assert_current_span_has_tenant_and_timeline_id; use crate::span::debug_assert_current_span_has_tenant_and_timeline_id_no_shard_id; @@ -215,14 +215,9 @@ async fn page_service_conn_main( auth_type: AuthType, connection_ctx: RequestContext, ) -> anyhow::Result<()> { - // Immediately increment the gauge, then create a job to decrement it on task exit. - // One of the pros of `defer!` is that this will *most probably* - // get called, even in presence of panics. - let gauge = LIVE_CONNECTIONS_COUNT.with_label_values(&["page_service"]); - gauge.inc(); - scopeguard::defer! { - gauge.dec(); - } + let _guard = LIVE_CONNECTIONS + .with_label_values(&["page_service"]) + .guard(); socket .set_nodelay(true) diff --git a/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs b/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs index c6ee6b90c4d1..a66900522af4 100644 --- a/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs +++ b/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs @@ -26,7 +26,7 @@ use tracing::{debug, error, info, trace, warn, Instrument}; use super::TaskStateUpdate; use crate::{ context::RequestContext, - metrics::{LIVE_CONNECTIONS_COUNT, WALRECEIVER_STARTED_CONNECTIONS, WAL_INGEST}, + metrics::{LIVE_CONNECTIONS, WALRECEIVER_STARTED_CONNECTIONS, WAL_INGEST}, task_mgr::TaskKind, task_mgr::WALRECEIVER_RUNTIME, tenant::{debug_assert_current_span_has_tenant_and_timeline_id, Timeline, WalReceiverInfo}, @@ -208,14 +208,9 @@ pub(super) async fn handle_walreceiver_connection( .instrument(tracing::info_span!("poller")), ); - // Immediately increment the gauge, then create a job to decrement it on task exit. - // One of the pros of `defer!` is that this will *most probably* - // get called, even in presence of panics. - let gauge = LIVE_CONNECTIONS_COUNT.with_label_values(&["wal_receiver"]); - gauge.inc(); - scopeguard::defer! { - gauge.dec(); - } + let _guard = LIVE_CONNECTIONS + .with_label_values(&["wal_receiver"]) + .guard(); let identify = identify_system(&replication_client).await?; info!("{identify:?}"); From b8d031cd0cff8bc155d962e35a781ed934999a58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Fri, 5 Jul 2024 22:18:05 +0200 Subject: [PATCH 052/194] Improve parsing of `ImageCompressionAlgorithm` (#8281) Improve parsing of the `ImageCompressionAlgorithm` enum to allow level customization like `zstd(1)`, as strum only takes `Default::default()`, i.e. `None` as the level. Part of #5431 --- libs/pageserver_api/src/models.rs | 64 +++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 12 deletions(-) diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index ecc543917e56..49c942938dfd 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -9,6 +9,7 @@ use std::{ collections::HashMap, io::{BufRead, Read}, num::{NonZeroU64, NonZeroUsize}, + str::FromStr, sync::atomic::AtomicUsize, time::{Duration, SystemTime}, }; @@ -437,18 +438,7 @@ pub enum CompactionAlgorithm { Tiered, } -#[derive( - Debug, - Clone, - Copy, - PartialEq, - Eq, - Serialize, - Deserialize, - strum_macros::FromRepr, - strum_macros::EnumString, -)] -#[strum(serialize_all = "kebab-case")] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum ImageCompressionAlgorithm { /// Disabled for writes, and never decompress during reading. /// Never set this after you've enabled compression once! @@ -468,6 +458,31 @@ impl ImageCompressionAlgorithm { } } +impl FromStr for ImageCompressionAlgorithm { + type Err = anyhow::Error; + fn from_str(s: &str) -> Result { + let mut components = s.split(['(', ')']); + let first = components + .next() + .ok_or_else(|| anyhow::anyhow!("empty string"))?; + match first { + "disabled-no-decompress" => Ok(ImageCompressionAlgorithm::DisabledNoDecompress), + "disabled" => Ok(ImageCompressionAlgorithm::Disabled), + "zstd" => { + let level = if let Some(v) = components.next() { + let v: i8 = v.parse()?; + Some(v) + } else { + None + }; + + Ok(ImageCompressionAlgorithm::Zstd { level }) + } + _ => anyhow::bail!("invalid specifier '{first}'"), + } + } +} + #[derive(Eq, PartialEq, Debug, Clone, Serialize, Deserialize)] pub struct CompactionAlgorithmSettings { pub kind: CompactionAlgorithm, @@ -1660,4 +1675,29 @@ mod tests { AuxFilePolicy::CrossValidation ); } + + #[test] + fn test_image_compression_algorithm_parsing() { + use ImageCompressionAlgorithm::*; + assert_eq!( + ImageCompressionAlgorithm::from_str("disabled").unwrap(), + Disabled + ); + assert_eq!( + ImageCompressionAlgorithm::from_str("disabled-no-decompress").unwrap(), + DisabledNoDecompress + ); + assert_eq!( + ImageCompressionAlgorithm::from_str("zstd").unwrap(), + Zstd { level: None } + ); + assert_eq!( + ImageCompressionAlgorithm::from_str("zstd(18)").unwrap(), + Zstd { level: Some(18) } + ); + assert_eq!( + ImageCompressionAlgorithm::from_str("zstd(-3)").unwrap(), + Zstd { level: Some(-3) } + ); + } } From 0a937b7f91646d942eb2717239578d96b8e854ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Fri, 5 Jul 2024 22:36:28 +0200 Subject: [PATCH 053/194] Add concurrency to the find-large-objects scrubber subcommand (#8291) The find-large-objects scrubber subcommand is quite fast if you run it in an environment with low latency to the S3 bucket (say an EC2 instance in the same region). However, the higher the latency gets, the slower the command becomes. Therefore, add a concurrency param and make it parallelized. This doesn't change that general relationship, but at least lets us do multiple requests in parallel and therefore hopefully faster. Running with concurrency of 64 (default): ``` 2024-07-05T17:30:22.882959Z INFO lazy_load_identity [...] [...] 2024-07-05T17:30:28.289853Z INFO Scanned 500 shards. [...] ``` With concurrency of 1, simulating state before this PR: ``` 2024-07-05T17:31:43.375153Z INFO lazy_load_identity [...] [...] 2024-07-05T17:33:51.987092Z INFO Scanned 500 shards. [...] ``` In other words, to list 500 shards, speed is increased from 2:08 minutes to 6 seconds. Follow-up of #8257, part of #5431 --- storage_scrubber/src/find_large_objects.rs | 97 +++++++++++++--------- storage_scrubber/src/main.rs | 13 ++- 2 files changed, 70 insertions(+), 40 deletions(-) diff --git a/storage_scrubber/src/find_large_objects.rs b/storage_scrubber/src/find_large_objects.rs index 24668b65169a..1422545f2fce 100644 --- a/storage_scrubber/src/find_large_objects.rs +++ b/storage_scrubber/src/find_large_objects.rs @@ -1,4 +1,4 @@ -use futures::StreamExt; +use futures::{StreamExt, TryStreamExt}; use pageserver::tenant::storage_layer::LayerName; use serde::{Deserialize, Serialize}; @@ -29,7 +29,7 @@ impl LargeObjectKind { } } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Clone)] pub struct LargeObject { pub key: String, pub size: u64, @@ -45,53 +45,76 @@ pub async fn find_large_objects( bucket_config: BucketConfig, min_size: u64, ignore_deltas: bool, + concurrency: usize, ) -> anyhow::Result { let (s3_client, target) = init_remote(bucket_config.clone(), NodeKind::Pageserver)?; - let mut tenants = std::pin::pin!(stream_tenants(&s3_client, &target)); - let mut objects = Vec::new(); - let mut tenant_ctr = 0u64; - let mut object_ctr = 0u64; - while let Some(tenant_shard_id) = tenants.next().await { - let tenant_shard_id = tenant_shard_id?; + let tenants = std::pin::pin!(stream_tenants(&s3_client, &target)); + + let objects_stream = tenants.map_ok(|tenant_shard_id| { let mut tenant_root = target.tenant_root(&tenant_shard_id); - // We want the objects and not just common prefixes - tenant_root.delimiter.clear(); - let mut continuation_token = None; - loop { - let fetch_response = - list_objects_with_retries(&s3_client, &tenant_root, continuation_token.clone()) - .await?; - for obj in fetch_response.contents().iter().filter(|o| { - if let Some(obj_size) = o.size { - min_size as i64 <= obj_size - } else { - false + let s3_client = s3_client.clone(); + async move { + let mut objects = Vec::new(); + let mut total_objects_ctr = 0u64; + // We want the objects and not just common prefixes + tenant_root.delimiter.clear(); + let mut continuation_token = None; + loop { + let fetch_response = + list_objects_with_retries(&s3_client, &tenant_root, continuation_token.clone()) + .await?; + for obj in fetch_response.contents().iter().filter(|o| { + if let Some(obj_size) = o.size { + min_size as i64 <= obj_size + } else { + false + } + }) { + let key = obj.key().expect("couldn't get key").to_owned(); + let kind = LargeObjectKind::from_key(&key); + if ignore_deltas && kind == LargeObjectKind::DeltaLayer { + continue; + } + objects.push(LargeObject { + key, + size: obj.size.unwrap() as u64, + kind, + }) } - }) { - let key = obj.key().expect("couldn't get key").to_owned(); - let kind = LargeObjectKind::from_key(&key); - if ignore_deltas && kind == LargeObjectKind::DeltaLayer { - continue; + total_objects_ctr += fetch_response.contents().len() as u64; + match fetch_response.next_continuation_token { + Some(new_token) => continuation_token = Some(new_token), + None => break, } - objects.push(LargeObject { - key, - size: obj.size.unwrap() as u64, - kind, - }) - } - object_ctr += fetch_response.contents().len() as u64; - match fetch_response.next_continuation_token { - Some(new_token) => continuation_token = Some(new_token), - None => break, } + + Ok((tenant_shard_id, objects, total_objects_ctr)) } + }); + let mut objects_stream = std::pin::pin!(objects_stream.try_buffer_unordered(concurrency)); + let mut objects = Vec::new(); + + let mut tenant_ctr = 0u64; + let mut object_ctr = 0u64; + while let Some(res) = objects_stream.next().await { + let (tenant_shard_id, objects_slice, total_objects_ctr) = res?; + objects.extend_from_slice(&objects_slice); + + object_ctr += total_objects_ctr; tenant_ctr += 1; - if tenant_ctr % 50 == 0 { + if tenant_ctr % 100 == 0 { tracing::info!( - "Scanned {tenant_ctr} shards. objects={object_ctr}, found={}, current={tenant_shard_id}.", objects.len() + "Scanned {tenant_ctr} shards. objects={object_ctr}, found={}, current={tenant_shard_id}.", + objects.len() ); } } + + let bucket_name = target.bucket_name(); + tracing::info!( + "Scan of {bucket_name} finished. Scanned {tenant_ctr} shards. objects={object_ctr}, found={}.", + objects.len() + ); Ok(LargeObjectListing { objects }) } diff --git a/storage_scrubber/src/main.rs b/storage_scrubber/src/main.rs index 10699edd3c94..16a26613d25b 100644 --- a/storage_scrubber/src/main.rs +++ b/storage_scrubber/src/main.rs @@ -78,6 +78,8 @@ enum Command { min_size: u64, #[arg(short, long, default_value_t = false)] ignore_deltas: bool, + #[arg(long = "concurrency", short = 'j', default_value_t = 64)] + concurrency: usize, }, } @@ -210,10 +212,15 @@ async fn main() -> anyhow::Result<()> { Command::FindLargeObjects { min_size, ignore_deltas, + concurrency, } => { - let summary = - find_large_objects::find_large_objects(bucket_config, min_size, ignore_deltas) - .await?; + let summary = find_large_objects::find_large_objects( + bucket_config, + min_size, + ignore_deltas, + concurrency, + ) + .await?; println!("{}", serde_json::to_string(&summary).unwrap()); Ok(()) } From 27fe7f8963e5227d24cdd56aab419fa973dba369 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Jul 2024 17:41:54 +0100 Subject: [PATCH 054/194] build(deps): bump certifi from 2023.7.22 to 2024.7.4 (#8301) --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7740388fb8be..bf16aaf55d26 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "aiohttp" @@ -734,13 +734,13 @@ typing-extensions = ">=4.1.0" [[package]] name = "certifi" -version = "2023.7.22" +version = "2024.7.4" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, - {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"}, + {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, + {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, ] [[package]] From 154ba5e1b440bda455c8962b53688268a2161d4b Mon Sep 17 00:00:00 2001 From: Alex Chi Z Date: Mon, 8 Jul 2024 09:05:49 -0400 Subject: [PATCH 055/194] fix(pageserver): ensure sparse keyspace is ordered (#8285) ## Problem Sparse keyspaces were constructed with ranges out of order: this didn't break things obviously, but meant that users of KeySpace functions that assume ordering would assert out. Closes https://github.com/neondatabase/neon/issues/8277 ## Summary of changes make sure the sparse keyspace has ordered keyspace parts --- pageserver/src/pgdatadir_mapping.rs | 52 +++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index 25d00d6dfd0d..fefd8d88ff21 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -919,6 +919,9 @@ impl Timeline { result.add_key(AUX_FILES_KEY); } + // Add extra keyspaces in the test cases. Some test cases write keys into the storage without + // creating directory keys. These test cases will add such keyspaces into `extra_test_dense_keyspace` + // and the keys will not be garbage-colllected. #[cfg(test)] { let guard = self.extra_test_dense_keyspace.load(); @@ -927,13 +930,48 @@ impl Timeline { } } - Ok(( - result.to_keyspace(), - /* AUX sparse key space */ - SparseKeySpace(KeySpace { - ranges: vec![repl_origin_key_range(), Key::metadata_aux_key_range()], - }), - )) + let dense_keyspace = result.to_keyspace(); + let sparse_keyspace = SparseKeySpace(KeySpace { + ranges: vec![Key::metadata_aux_key_range(), repl_origin_key_range()], + }); + + if cfg!(debug_assertions) { + // Verify if the sparse keyspaces are ordered and non-overlapping. + + // We do not use KeySpaceAccum for sparse_keyspace because we want to ensure each + // category of sparse keys are split into their own image/delta files. If there + // are overlapping keyspaces, they will be automatically merged by keyspace accum, + // and we want the developer to keep the keyspaces separated. + + let ranges = &sparse_keyspace.0.ranges; + + // TODO: use a single overlaps_with across the codebase + fn overlaps_with(a: &Range, b: &Range) -> bool { + !(a.end <= b.start || b.end <= a.start) + } + for i in 0..ranges.len() { + for j in 0..i { + if overlaps_with(&ranges[i], &ranges[j]) { + panic!( + "overlapping sparse keyspace: {}..{} and {}..{}", + ranges[i].start, ranges[i].end, ranges[j].start, ranges[j].end + ); + } + } + } + for i in 1..ranges.len() { + assert!( + ranges[i - 1].end <= ranges[i].start, + "unordered sparse keyspace: {}..{} and {}..{}", + ranges[i - 1].start, + ranges[i - 1].end, + ranges[i].start, + ranges[i].end + ); + } + } + + Ok((dense_keyspace, sparse_keyspace)) } /// Get cached size of relation if it not updated after specified LSN From 1121a1cbac0059369870d943bf144f0a221db65c Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 8 Jul 2024 14:10:42 +0100 Subject: [PATCH 056/194] pageserver: switch to jemalloc (#8307) ## Problem - Resident memory on long running pageserver processes tends to climb: memory fragmentation is suspected. - Total resident memory may be a limiting factor for running on smaller nodes. ## Summary of changes - As a low-energy experiment, switch the pageserver to use jemalloc (not a net-new dependency, proxy already use it) - Decide at end of week whether to revert before next release. --- Cargo.lock | 2 ++ pageserver/Cargo.toml | 1 + pageserver/src/bin/pageserver.rs | 3 +++ workspace_hack/Cargo.toml | 1 + 4 files changed, 7 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 6dae8e340348..716b6690d9f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3667,6 +3667,7 @@ dependencies = [ "sysinfo", "tenant_size_model", "thiserror", + "tikv-jemallocator", "tokio", "tokio-epoll-uring", "tokio-io-timeout", @@ -7468,6 +7469,7 @@ dependencies = [ "syn 1.0.109", "syn 2.0.52", "sync_wrapper", + "tikv-jemalloc-sys", "time", "time-macros", "tokio", diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index 4335f38f1e7f..0d9343d64382 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -62,6 +62,7 @@ sync_wrapper.workspace = true sysinfo.workspace = true tokio-tar.workspace = true thiserror.workspace = true +tikv-jemallocator.workspace = true tokio = { workspace = true, features = ["process", "sync", "fs", "rt", "io-util", "time"] } tokio-epoll-uring.workspace = true tokio-io-timeout.workspace = true diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 39d4e46c9663..2763352a213f 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -47,6 +47,9 @@ use utils::{ project_git_version!(GIT_VERSION); project_build_tag!(BUILD_TAG); +#[global_allocator] +static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + const PID_FILE_NAME: &str = "pageserver.pid"; const FEATURES: &[&str] = &[ diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index f43076171f21..e1b1806bc877 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -69,6 +69,7 @@ sha2 = { version = "0.10", features = ["asm"] } smallvec = { version = "1", default-features = false, features = ["const_new", "write"] } subtle = { version = "2" } sync_wrapper = { version = "0.1", default-features = false, features = ["futures"] } +tikv-jemalloc-sys = { version = "0.5" } time = { version = "0.3", features = ["macros", "serde-well-known"] } tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "test-util"] } tokio-rustls = { version = "0.24" } From 2a3410d1c3f4d1cfec3c3959311962872c8fdb87 Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Wed, 3 Jul 2024 14:57:17 -0500 Subject: [PATCH 057/194] Hide import behind TYPE_CHECKING No need to import it if we aren't type checking anything. --- test_runner/performance/test_logical_replication.py | 7 ++++++- test_runner/regress/test_physical_replication.py | 6 +++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/test_runner/performance/test_logical_replication.py b/test_runner/performance/test_logical_replication.py index 7d11facc2949..570bd11b6f15 100644 --- a/test_runner/performance/test_logical_replication.py +++ b/test_runner/performance/test_logical_replication.py @@ -1,8 +1,13 @@ +from __future__ import annotations + import time import pytest from fixtures.log_helper import log -from fixtures.neon_fixtures import AuxFileStore, NeonEnv, PgBin, logical_replication_sync +from fixtures.neon_fixtures import AuxFileStore, logical_replication_sync + +if TYPE_CHECKING: + from fixtures.neon_fixtures import NeonEnv, PgBin @pytest.mark.parametrize("pageserver_aux_file_policy", [AuxFileStore.V2]) diff --git a/test_runner/regress/test_physical_replication.py b/test_runner/regress/test_physical_replication.py index a1bff32eedd1..043aff686b09 100644 --- a/test_runner/regress/test_physical_replication.py +++ b/test_runner/regress/test_physical_replication.py @@ -1,7 +1,11 @@ +from __future__ import annotations + import random import time +from typing import TYPE_CHECKING -from fixtures.neon_fixtures import NeonEnv +if TYPE_CHECKING: + from fixtures.neon_fixtures import NeonEnv def test_physical_replication(neon_simple_env: NeonEnv): From f2ec5429542f4aa4d5be6c2f6551cde8727c2829 Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Wed, 3 Jul 2024 14:54:49 -0500 Subject: [PATCH 058/194] Add Neon HTTP API test fixture This is a Python binding to the Neon HTTP API. It isn't complete, but can be extended as necessary. Co-authored-by: Sasha Krassovsky --- test_runner/fixtures/neon_api.py | 263 ++++++++++++++++++++++++++ test_runner/fixtures/neon_fixtures.py | 21 ++ 2 files changed, 284 insertions(+) create mode 100644 test_runner/fixtures/neon_api.py diff --git a/test_runner/fixtures/neon_api.py b/test_runner/fixtures/neon_api.py new file mode 100644 index 000000000000..39baf5fab69f --- /dev/null +++ b/test_runner/fixtures/neon_api.py @@ -0,0 +1,263 @@ +from __future__ import annotations + +import time +from typing import TYPE_CHECKING, cast + +import requests + +if TYPE_CHECKING: + from typing import Any, Dict, Literal, Optional, Union + + from fixtures.pg_version import PgVersion + + +def connection_parameters_to_env(params: Dict[str, str]) -> Dict[str, str]: + return { + "PGHOST": params["host"], + "PGDATABASE": params["database"], + "PGUSER": params["role"], + "PGPASSWORD": params["password"], + } + + +class NeonAPI: + def __init__(self, neon_api_key: str, neon_api_base_url: str): + self.__neon_api_key = neon_api_key + self.__neon_api_base_url = neon_api_base_url.strip("/") + + def __request( + self, method: Union[str, bytes], endpoint: str, **kwargs: Any + ) -> requests.Response: + if "headers" not in kwargs: + kwargs["headers"] = {} + kwargs["headers"]["Authorization"] = f"Bearer {self.__neon_api_key}" + + return requests.request(method, f"{self.__neon_api_base_url}{endpoint}", **kwargs) + + def create_project( + self, + pg_version: Optional[PgVersion] = None, + name: Optional[str] = None, + branch_name: Optional[str] = None, + branch_role_name: Optional[str] = None, + branch_database_name: Optional[str] = None, + ) -> Dict[str, Any]: + data: Dict[str, Any] = { + "project": { + "branch": {}, + }, + } + if name: + data["project"]["name"] = name + if pg_version: + data["project"]["pg_version"] = int(pg_version) + if branch_name: + data["project"]["branch"]["name"] = branch_name + if branch_role_name: + data["project"]["branch"]["role_name"] = branch_role_name + if branch_database_name: + data["project"]["branch"]["database_name"] = branch_database_name + + resp = self.__request( + "POST", + "/projects", + headers={ + "Accept": "application/json", + "Content-Type": "application/json", + }, + json=data, + ) + + assert resp.status_code == 201 + + return cast("Dict[str, Any]", resp.json()) + + def get_project_details(self, project_id: str) -> Dict[str, Any]: + resp = self.__request( + "GET", + f"/projects/{project_id}", + headers={ + "Accept": "application/json", + "Content-Type": "application/json", + }, + ) + assert resp.status_code == 200 + return cast("Dict[str, Any]", resp.json()) + + def delete_project( + self, + project_id: str, + ) -> Dict[str, Any]: + resp = self.__request( + "DELETE", + f"/projects/{project_id}", + headers={ + "Accept": "application/json", + "Content-Type": "application/json", + }, + ) + + assert resp.status_code == 200 + + return cast("Dict[str, Any]", resp.json()) + + def start_endpoint( + self, + project_id: str, + endpoint_id: str, + ) -> Dict[str, Any]: + resp = self.__request( + "POST", + f"/projects/{project_id}/endpoints/{endpoint_id}/start", + headers={ + "Accept": "application/json", + }, + ) + + assert resp.status_code == 200 + + return cast("Dict[str, Any]", resp.json()) + + def suspend_endpoint( + self, + project_id: str, + endpoint_id: str, + ) -> Dict[str, Any]: + resp = self.__request( + "POST", + f"/projects/{project_id}/endpoints/{endpoint_id}/suspend", + headers={ + "Accept": "application/json", + }, + ) + + assert resp.status_code == 200 + + return cast("Dict[str, Any]", resp.json()) + + def restart_endpoint( + self, + project_id: str, + endpoint_id: str, + ) -> Dict[str, Any]: + resp = self.__request( + "POST", + f"/projects/{project_id}/endpoints/{endpoint_id}/restart", + headers={ + "Accept": "application/json", + }, + ) + + assert resp.status_code == 200 + + return cast("Dict[str, Any]", resp.json()) + + def create_endpoint( + self, + project_id: str, + branch_id: str, + endpoint_type: Literal["read_write", "read_only"], + settings: Dict[str, Any], + ) -> Dict[str, Any]: + data: Dict[str, Any] = { + "endpoint": { + "branch_id": branch_id, + }, + } + + if endpoint_type: + data["endpoint"]["type"] = endpoint_type + if settings: + data["endpoint"]["settings"] = settings + + resp = self.__request( + "POST", + f"/projects/{project_id}/endpoints", + headers={ + "Accept": "application/json", + "Content-Type": "application/json", + }, + json=data, + ) + + assert resp.status_code == 201 + + return cast("Dict[str, Any]", resp.json()) + + def get_connection_uri( + self, + project_id: str, + branch_id: Optional[str] = None, + endpoint_id: Optional[str] = None, + database_name: str = "neondb", + role_name: str = "neondb_owner", + pooled: bool = True, + ) -> Dict[str, Any]: + resp = self.__request( + "GET", + f"/projects/{project_id}/connection_uri", + params={ + "branch_id": branch_id, + "endpoint_id": endpoint_id, + "database_name": database_name, + "role_name": role_name, + "pooled": pooled, + }, + headers={ + "Accept": "application/json", + }, + ) + + assert resp.status_code == 200 + + return cast("Dict[str, Any]", resp.json()) + + def get_branches(self, project_id: str) -> Dict[str, Any]: + resp = self.__request( + "GET", + f"/projects/{project_id}/branches", + headers={ + "Accept": "application/json", + }, + ) + + assert resp.status_code == 200 + + return cast("Dict[str, Any]", resp.json()) + + def get_endpoints(self, project_id: str) -> Dict[str, Any]: + resp = self.__request( + "GET", + f"/projects/{project_id}/endpoints", + headers={ + "Accept": "application/json", + }, + ) + + assert resp.status_code == 200 + + return cast("Dict[str, Any]", resp.json()) + + def get_operations(self, project_id: str) -> Dict[str, Any]: + resp = self.__request( + "GET", + f"/projects/{project_id}/operations", + headers={ + "Accept": "application/json", + "Authorization": f"Bearer {self.__neon_api_key}", + }, + ) + + assert resp.status_code == 200 + + return cast("Dict[str, Any]", resp.json()) + + def wait_for_operation_to_finish(self, project_id: str): + has_running = True + while has_running: + has_running = False + operations = self.get_operations(project_id)["operations"] + for op in operations: + if op["status"] in {"scheduling", "running", "cancelling"}: + has_running = True + time.sleep(0.5) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 5fb4d948175f..ac2fcd8ade5c 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -87,6 +87,8 @@ ) from fixtures.utils import AuxFileStore as AuxFileStore # reexport +from .neon_api import NeonAPI + """ This file contains pytest fixtures. A fixture is a test resource that can be summoned by placing its name in the test's arguments. @@ -184,6 +186,25 @@ def versioned_pg_distrib_dir(pg_distrib_dir: Path, pg_version: PgVersion) -> Ite yield versioned_dir +@pytest.fixture(scope="session") +def neon_api_key() -> str: + api_key = os.getenv("NEON_API_KEY") + if not api_key: + raise AssertionError("Set the NEON_API_KEY environment variable") + + return api_key + + +@pytest.fixture(scope="session") +def neon_api_base_url() -> str: + return os.getenv("NEON_API_BASE_URL", "https://console-stage.neon.build/api/v2") + + +@pytest.fixture(scope="session") +def neon_api(neon_api_key: str, neon_api_base_url: str) -> NeonAPI: + return NeonAPI(neon_api_key, neon_api_base_url) + + def shareable_scope(fixture_name: str, config: Config) -> Literal["session", "function"]: """Return either session of function scope, depending on TEST_SHARED_FIXTURES envvar. From 118847cd41bcf5f84126f371f8e322d51eeed1f7 Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Wed, 3 Jul 2024 14:59:19 -0500 Subject: [PATCH 059/194] Log PG environment variables when a PgBin runs Useful for debugging situations like connecting to databases. Co-authored-by: Sasha Krassovsky --- test_runner/fixtures/neon_fixtures.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index ac2fcd8ade5c..532e7bcce535 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2883,6 +2883,13 @@ def _build_env(self, env_add: Optional[Env]) -> Env: env.update(env_add) return env + def _log_env(self, env: dict[str, str]) -> None: + env_s = {} + for k, v in env.items(): + if k.startswith("PG") and k != "PGPASSWORD": + env_s[k] = v + log.debug(f"Environment: {env_s}") + def run( self, command: List[str], @@ -2905,6 +2912,7 @@ def run( self._fixpath(command) log.info(f"Running command '{' '.join(command)}'") env = self._build_env(env) + self._log_env(env) subprocess.run(command, env=env, cwd=cwd, check=True) def run_capture( @@ -2925,6 +2933,7 @@ def run_capture( self._fixpath(command) log.info(f"Running command '{' '.join(command)}'") env = self._build_env(env) + self._log_env(env) base_path, _, _ = subprocess_capture( self.log_dir, command, From b54dd9af1575169ce008e6bc1e3f44d7ab22413f Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Wed, 3 Jul 2024 15:04:57 -0500 Subject: [PATCH 060/194] Add PgBin.run_nonblocking() Allows a process to run without blocking program execution, which can be useful for certain test scenarios. Co-authored-by: Sasha Krassovsky --- test_runner/fixtures/neon_fixtures.py | 32 ++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 532e7bcce535..cae2e422c198 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2890,14 +2890,14 @@ def _log_env(self, env: dict[str, str]) -> None: env_s[k] = v log.debug(f"Environment: {env_s}") - def run( + def run_nonblocking( self, command: List[str], env: Optional[Env] = None, cwd: Optional[Union[str, Path]] = None, - ): + ) -> subprocess.Popen[Any]: """ - Run one of the postgres binaries. + Run one of the postgres binaries, not waiting for it to finish The command should be in list form, e.g. ['pgbench', '-p', '55432'] @@ -2908,12 +2908,34 @@ def run( If you want stdout/stderr captured to files, use `run_capture` instead. """ - self._fixpath(command) log.info(f"Running command '{' '.join(command)}'") env = self._build_env(env) self._log_env(env) - subprocess.run(command, env=env, cwd=cwd, check=True) + return subprocess.Popen(command, env=env, cwd=cwd, stdout=subprocess.PIPE, text=True) + + def run( + self, + command: List[str], + env: Optional[Env] = None, + cwd: Optional[Union[str, Path]] = None, + ) -> None: + """ + Run one of the postgres binaries, waiting for it to finish + + The command should be in list form, e.g. ['pgbench', '-p', '55432'] + + All the necessary environment variables will be set. + + If the first argument (the command name) doesn't include a path (no '/' + characters present), then it will be edited to include the correct path. + + If you want stdout/stderr captured to files, use `run_capture` instead. + """ + proc = self.run_nonblocking(command, env, cwd) + proc.wait() + if proc.returncode != 0: + raise subprocess.CalledProcessError(proc.returncode, proc.args) def run_capture( self, From 1c57f6bac34c2e97a1929cd5e96af1156bdc240d Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Wed, 3 Jul 2024 15:22:42 -0500 Subject: [PATCH 061/194] Add long running replication tests These tests will help verify that replication, both physical and logical, works as expected in Neon. Co-authored-by: Sasha Krassovsky --- .../actions/run-python-test-set/action.yml | 1 + .github/workflows/benchmarking.yml | 72 ++++- .../performance/test_logical_replication.py | 295 ++++++++++++++++- .../performance/test_physical_replication.py | 296 ++++++++++++++++++ 4 files changed, 662 insertions(+), 2 deletions(-) create mode 100644 test_runner/performance/test_physical_replication.py diff --git a/.github/actions/run-python-test-set/action.yml b/.github/actions/run-python-test-set/action.yml index 7f843de1a55c..daaedf6d11d2 100644 --- a/.github/actions/run-python-test-set/action.yml +++ b/.github/actions/run-python-test-set/action.yml @@ -115,6 +115,7 @@ runs: export POSTGRES_DISTRIB_DIR=${POSTGRES_DISTRIB_DIR:-/tmp/neon/pg_install} export DEFAULT_PG_VERSION=${PG_VERSION#v} export LD_LIBRARY_PATH=${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/lib + export BENCHMARK_CONNSTR=${BENCHMARK_CONNSTR:-} if [ "${BUILD_TYPE}" = "remote" ]; then export REMOTE_ENV=1 diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index db04b5de7ddc..899cae2b8658 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -99,7 +99,14 @@ jobs: # Set --sparse-ordering option of pytest-order plugin # to ensure tests are running in order of appears in the file. # It's important for test_perf_pgbench.py::test_pgbench_remote_* tests - extra_params: -m remote_cluster --sparse-ordering --timeout 5400 --ignore test_runner/performance/test_perf_olap.py --ignore test_runner/performance/test_perf_pgvector_queries.py + extra_params: + -m remote_cluster + --sparse-ordering + --timeout 5400 + --ignore test_runner/performance/test_perf_olap.py + --ignore test_runner/performance/test_perf_pgvector_queries.py + --ignore test_runner/performance/test_logical_replication.py + --ignore test_runner/performance/test_physical_replication.py env: BENCHMARK_CONNSTR: ${{ steps.create-neon-project.outputs.dsn }} VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" @@ -125,6 +132,69 @@ jobs: env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + replication-tests: + env: + POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install + DEFAULT_PG_VERSION: 14 + TEST_OUTPUT: /tmp/test_output + BUILD_TYPE: remote + SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }} + PLATFORM: "neon-staging" + + runs-on: [ self-hosted, us-east-2, x64 ] + container: + image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:pinned + options: --init + + steps: + - uses: actions/checkout@v4 + + - name: Download Neon artifact + uses: ./.github/actions/download + with: + name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact + path: /tmp/neon/ + prefix: latest + + - name: Run benchmark + uses: ./.github/actions/run-python-test-set + with: + build_type: ${{ env.BUILD_TYPE }} + test_selection: performance/test_logical_replication.py + run_in_parallel: false + save_perf_report: ${{ env.SAVE_PERF_REPORT }} + extra_params: -m remote_cluster --timeout 5400 + env: + VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" + PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" + NEON_API_KEY: ${{ secrets.NEON_STAGING_API_KEY }} + + - name: Run benchmark + uses: ./.github/actions/run-python-test-set + with: + build_type: ${{ env.BUILD_TYPE }} + test_selection: performance/test_physical_replication.py + run_in_parallel: false + save_perf_report: ${{ env.SAVE_PERF_REPORT }} + extra_params: -m remote_cluster --timeout 5400 + env: + VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" + PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" + NEON_API_KEY: ${{ secrets.NEON_STAGING_API_KEY }} + + - name: Create Allure report + if: ${{ !cancelled() }} + uses: ./.github/actions/allure-report-generate + + - name: Post to a Slack channel + if: ${{ github.event.schedule && failure() }} + uses: slackapi/slack-github-action@v1 + with: + channel-id: "C033QLM5P7D" # dev-staging-stream + slack-message: "Periodic replication testing: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + generate-matrices: if: ${{ github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null }} # Create matrices for the benchmarking jobs, so we run benchmarks on rds only once a week (on Saturday) diff --git a/test_runner/performance/test_logical_replication.py b/test_runner/performance/test_logical_replication.py index 570bd11b6f15..5ab83dd31d0b 100644 --- a/test_runner/performance/test_logical_replication.py +++ b/test_runner/performance/test_logical_replication.py @@ -1,13 +1,24 @@ from __future__ import annotations import time +import traceback +from typing import TYPE_CHECKING +import psycopg2 +import psycopg2.extras import pytest +from fixtures.benchmark_fixture import MetricReport +from fixtures.common_types import Lsn from fixtures.log_helper import log +from fixtures.neon_api import connection_parameters_to_env from fixtures.neon_fixtures import AuxFileStore, logical_replication_sync +from fixtures.pg_version import PgVersion if TYPE_CHECKING: + from fixtures.benchmark_fixture import NeonBenchmarker + from fixtures.neon_api import NeonAPI from fixtures.neon_fixtures import NeonEnv, PgBin + from fixtures.pg_version import PgVersion @pytest.mark.parametrize("pageserver_aux_file_policy", [AuxFileStore.V2]) @@ -31,7 +42,6 @@ def test_logical_replication(neon_simple_env: NeonEnv, pg_bin: PgBin, vanilla_pg vanilla_pg.safe_psql("truncate table pgbench_history") connstr = endpoint.connstr().replace("'", "''") - print(f"connstr='{connstr}'") vanilla_pg.safe_psql(f"create subscription sub1 connection '{connstr}' publication pub1") # Wait logical replication channel to be established @@ -47,3 +57,286 @@ def test_logical_replication(neon_simple_env: NeonEnv, pg_bin: PgBin, vanilla_pg sum_master = endpoint.safe_psql("select sum(abalance) from pgbench_accounts")[0][0] sum_replica = vanilla_pg.safe_psql("select sum(abalance) from pgbench_accounts")[0][0] assert sum_master == sum_replica + + +def check_pgbench_still_running(pgbench, label=""): + rc = pgbench.poll() + if rc is not None: + raise RuntimeError(f"{label} pgbench terminated early with return code {rc}") + + +def measure_logical_replication_lag(sub_cur, pub_cur, timeout_sec=600): + start = time.time() + pub_cur.execute("SELECT pg_current_wal_flush_lsn()") + pub_lsn = Lsn(pub_cur.fetchall()[0][0]) + while (time.time() - start) < timeout_sec: + sub_cur.execute("SELECT latest_end_lsn FROM pg_catalog.pg_stat_subscription") + res = sub_cur.fetchall()[0][0] + if res: + log.info(f"subscriber_lsn={res}") + sub_lsn = Lsn(res) + log.info(f"Subscriber LSN={sub_lsn}, publisher LSN={pub_lsn}") + if sub_lsn >= pub_lsn: + return time.time() - start + time.sleep(0.5) + raise TimeoutError(f"Logical replication sync took more than {timeout_sec} sec") + + +@pytest.mark.remote_cluster +@pytest.mark.timeout(2 * 60 * 60) +def test_subscriber_lag( + pg_bin: PgBin, + neon_api: NeonAPI, + pg_version: PgVersion, + zenbenchmark: NeonBenchmarker, +): + """ + Creates a publisher and subscriber, runs pgbench inserts on publisher and pgbench selects + on subscriber. Periodically restarts subscriber while still running the inserts, and + measures how long sync takes after restart. + """ + test_duration_min = 60 + sync_interval_min = 5 + pgbench_duration = f"-T{test_duration_min * 60 * 2}" + + pub_project = neon_api.create_project(pg_version) + pub_project_id = pub_project["project"]["id"] + neon_api.wait_for_operation_to_finish(pub_project_id) + error_occurred = False + try: + sub_project = neon_api.create_project(pg_version) + sub_project_id = sub_project["project"]["id"] + sub_endpoint_id = sub_project["endpoints"][0]["id"] + neon_api.wait_for_operation_to_finish(sub_project_id) + try: + pub_env = connection_parameters_to_env( + pub_project["connection_uris"][0]["connection_parameters"] + ) + sub_env = connection_parameters_to_env( + sub_project["connection_uris"][0]["connection_parameters"] + ) + pub_connstr = pub_project["connection_uris"][0]["connection_uri"] + sub_connstr = sub_project["connection_uris"][0]["connection_uri"] + + pg_bin.run_capture(["pgbench", "-i", "-s100"], env=pub_env) + pg_bin.run_capture(["pgbench", "-i", "-s100"], env=sub_env) + + pub_conn = psycopg2.connect(pub_connstr) + sub_conn = psycopg2.connect(sub_connstr) + pub_conn.autocommit = True + sub_conn.autocommit = True + with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: + sub_cur.execute("truncate table pgbench_accounts") + sub_cur.execute("truncate table pgbench_history") + + pub_cur.execute( + "create publication pub1 for table pgbench_accounts, pgbench_history" + ) + sub_cur.execute( + f"create subscription sub1 connection '{pub_connstr}' publication pub1" + ) + + initial_sync_lag = measure_logical_replication_lag(sub_cur, pub_cur) + pub_conn.close() + sub_conn.close() + + zenbenchmark.record( + "initial_sync_lag", initial_sync_lag, "s", MetricReport.LOWER_IS_BETTER + ) + + pub_workload = pg_bin.run_nonblocking( + ["pgbench", "-c10", pgbench_duration, "-Mprepared"], env=pub_env + ) + try: + sub_workload = pg_bin.run_nonblocking( + ["pgbench", "-c10", pgbench_duration, "-S"], + env=sub_env, + ) + try: + start = time.time() + while time.time() - start < test_duration_min * 60: + time.sleep(sync_interval_min * 60) + check_pgbench_still_running(pub_workload, "pub") + check_pgbench_still_running(sub_workload, "sub") + + with psycopg2.connect(pub_connstr) as pub_conn, psycopg2.connect( + sub_connstr + ) as sub_conn: + with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: + lag = measure_logical_replication_lag(sub_cur, pub_cur) + + log.info(f"Replica lagged behind master by {lag} seconds") + zenbenchmark.record("replica_lag", lag, "s", MetricReport.LOWER_IS_BETTER) + sub_workload.terminate() + neon_api.restart_endpoint( + sub_project_id, + sub_endpoint_id, + ) + neon_api.wait_for_operation_to_finish(sub_project_id) + sub_workload = pg_bin.run_nonblocking( + ["pgbench", "-c10", pgbench_duration, "-S"], + env=sub_env, + ) + + # Measure storage to make sure replication information isn't bloating storage + sub_storage = neon_api.get_project_details(sub_project_id)["project"][ + "synthetic_storage_size" + ] + pub_storage = neon_api.get_project_details(pub_project_id)["project"][ + "synthetic_storage_size" + ] + zenbenchmark.record( + "sub_storage", sub_storage, "B", MetricReport.LOWER_IS_BETTER + ) + zenbenchmark.record( + "pub_storage", pub_storage, "B", MetricReport.LOWER_IS_BETTER + ) + + finally: + sub_workload.terminate() + finally: + pub_workload.terminate() + except Exception as e: + error_occurred = True + log.error(f"Caught exception {e}") + log.error(traceback.format_exc()) + finally: + if not error_occurred: + neon_api.delete_project(sub_project_id) + except Exception as e: + error_occurred = True + log.error(f"Caught exception {e}") + log.error(traceback.format_exc()) + finally: + assert not error_occurred + neon_api.delete_project(pub_project_id) + + +@pytest.mark.remote_cluster +@pytest.mark.timeout(2 * 60 * 60) +def test_publisher_restart( + pg_bin: PgBin, + neon_api: NeonAPI, + pg_version: PgVersion, + zenbenchmark: NeonBenchmarker, +): + """ + Creates a publisher and subscriber, runs pgbench inserts on publisher and pgbench selects + on subscriber. Periodically restarts publisher (to exercise on-demand WAL download), and + measures how long sync takes after restart. + """ + test_duration_min = 60 + sync_interval_min = 5 + pgbench_duration = f"-T{test_duration_min * 60 * 2}" + + pub_project = neon_api.create_project(pg_version) + pub_project_id = pub_project["project"]["id"] + pub_endpoint_id = pub_project["endpoints"][0]["id"] + neon_api.wait_for_operation_to_finish(pub_project_id) + error_occurred = False + try: + sub_project = neon_api.create_project(pg_version) + sub_project_id = sub_project["project"]["id"] + neon_api.wait_for_operation_to_finish(sub_project_id) + try: + pub_env = connection_parameters_to_env( + pub_project["connection_uris"][0]["connection_parameters"] + ) + sub_env = connection_parameters_to_env( + sub_project["connection_uris"][0]["connection_parameters"] + ) + pub_connstr = pub_project["connection_uris"][0]["connection_uri"] + sub_connstr = sub_project["connection_uris"][0]["connection_uri"] + + pg_bin.run_capture(["pgbench", "-i", "-s100"], env=pub_env) + pg_bin.run_capture(["pgbench", "-i", "-s100"], env=sub_env) + + pub_conn = psycopg2.connect(pub_connstr) + sub_conn = psycopg2.connect(sub_connstr) + pub_conn.autocommit = True + sub_conn.autocommit = True + with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: + sub_cur.execute("truncate table pgbench_accounts") + sub_cur.execute("truncate table pgbench_history") + + pub_cur.execute( + "create publication pub1 for table pgbench_accounts, pgbench_history" + ) + sub_cur.execute( + f"create subscription sub1 connection '{pub_connstr}' publication pub1" + ) + + initial_sync_lag = measure_logical_replication_lag(sub_cur, pub_cur) + pub_conn.close() + sub_conn.close() + + zenbenchmark.record( + "initial_sync_lag", initial_sync_lag, "s", MetricReport.LOWER_IS_BETTER + ) + + pub_workload = pg_bin.run_nonblocking( + ["pgbench", "-c10", pgbench_duration, "-Mprepared"], env=pub_env + ) + try: + sub_workload = pg_bin.run_nonblocking( + ["pgbench", "-c10", pgbench_duration, "-S"], + env=sub_env, + ) + try: + start = time.time() + while time.time() - start < test_duration_min * 60: + time.sleep(sync_interval_min * 60) + check_pgbench_still_running(pub_workload, "pub") + check_pgbench_still_running(sub_workload, "sub") + + pub_workload.terminate() + neon_api.restart_endpoint( + pub_project_id, + pub_endpoint_id, + ) + neon_api.wait_for_operation_to_finish(pub_project_id) + pub_workload = pg_bin.run_nonblocking( + ["pgbench", "-c10", pgbench_duration, "-Mprepared"], + env=pub_env, + ) + with psycopg2.connect(pub_connstr) as pub_conn, psycopg2.connect( + sub_connstr + ) as sub_conn: + with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: + lag = measure_logical_replication_lag(sub_cur, pub_cur) + + log.info(f"Replica lagged behind master by {lag} seconds") + zenbenchmark.record("replica_lag", lag, "s", MetricReport.LOWER_IS_BETTER) + + # Measure storage to make sure replication information isn't bloating storage + sub_storage = neon_api.get_project_details(sub_project_id)["project"][ + "synthetic_storage_size" + ] + pub_storage = neon_api.get_project_details(pub_project_id)["project"][ + "synthetic_storage_size" + ] + zenbenchmark.record( + "sub_storage", sub_storage, "B", MetricReport.LOWER_IS_BETTER + ) + zenbenchmark.record( + "pub_storage", pub_storage, "B", MetricReport.LOWER_IS_BETTER + ) + + finally: + sub_workload.terminate() + finally: + pub_workload.terminate() + except Exception as e: + error_occurred = True + log.error(f"Caught exception {e}") + log.error(traceback.format_exc()) + finally: + if not error_occurred: + neon_api.delete_project(sub_project_id) + except Exception as e: + error_occurred = True + log.error(f"Caught exception {e}") + log.error(traceback.format_exc()) + finally: + assert not error_occurred + neon_api.delete_project(pub_project_id) diff --git a/test_runner/performance/test_physical_replication.py b/test_runner/performance/test_physical_replication.py new file mode 100644 index 000000000000..7e1619721144 --- /dev/null +++ b/test_runner/performance/test_physical_replication.py @@ -0,0 +1,296 @@ +from __future__ import annotations + +import csv +import os +import subprocess +import time +import traceback +from pathlib import Path +from typing import TYPE_CHECKING + +import psycopg2 +import psycopg2.extras +import pytest +from fixtures.benchmark_fixture import MetricReport +from fixtures.common_types import Lsn +from fixtures.log_helper import log +from fixtures.neon_api import connection_parameters_to_env +from fixtures.pg_version import PgVersion + +if TYPE_CHECKING: + from typing import Any, List, Optional + + from fixtures.benchmark_fixture import NeonBenchmarker + from fixtures.neon_api import NeonAPI + from fixtures.neon_fixtures import PgBin + + +# Granularity of ~0.5 sec +def measure_replication_lag(master, replica, timeout_sec=600): + start = time.time() + master.execute("SELECT pg_current_wal_flush_lsn()") + master_lsn = Lsn(master.fetchall()[0][0]) + while (time.time() - start) < timeout_sec: + replica.execute("select pg_last_wal_replay_lsn()") + replica_lsn = replica.fetchall()[0][0] + if replica_lsn: + if Lsn(replica_lsn) >= master_lsn: + return time.time() - start + time.sleep(0.5) + raise TimeoutError(f"Replication sync took more than {timeout_sec} sec") + + +def check_pgbench_still_running(pgbench): + rc = pgbench.poll() + if rc is not None: + raise RuntimeError(f"Pgbench terminated early with return code {rc}") + + +@pytest.mark.remote_cluster +@pytest.mark.timeout(2 * 60 * 60) +def test_ro_replica_lag( + pg_bin: PgBin, + neon_api: NeonAPI, + pg_version: PgVersion, + zenbenchmark: NeonBenchmarker, +): + test_duration_min = 60 + sync_interval_min = 10 + + pgbench_duration = f"-T{test_duration_min * 60 * 2}" + + project = neon_api.create_project(pg_version) + project_id = project["project"]["id"] + neon_api.wait_for_operation_to_finish(project_id) + error_occurred = False + try: + branch_id = project["branch"]["id"] + master_connstr = project["connection_uris"][0]["connection_uri"] + master_env = connection_parameters_to_env( + project["connection_uris"][0]["connection_parameters"] + ) + + replica = neon_api.create_endpoint( + project_id, + branch_id, + endpoint_type="read_only", + settings={"pg_settings": {"hot_standby_feedback": "on"}}, + ) + replica_env = master_env.copy() + replica_env["PGHOST"] = replica["endpoint"]["host"] + neon_api.wait_for_operation_to_finish(project_id) + + replica_connstr = neon_api.get_connection_uri( + project_id, + endpoint_id=replica["endpoint"]["id"], + )["uri"] + + pg_bin.run_capture(["pgbench", "-i", "-s100"], env=master_env) + + master_workload = pg_bin.run_nonblocking( + ["pgbench", "-c10", pgbench_duration, "-Mprepared"], + env=master_env, + ) + try: + replica_workload = pg_bin.run_nonblocking( + ["pgbench", "-c10", pgbench_duration, "-S"], + env=replica_env, + ) + try: + start = time.time() + while time.time() - start < test_duration_min * 60: + check_pgbench_still_running(master_workload) + check_pgbench_still_running(replica_workload) + time.sleep(sync_interval_min * 60) + with psycopg2.connect(master_connstr) as conn_master, psycopg2.connect( + replica_connstr + ) as conn_replica: + with conn_master.cursor() as cur_master, conn_replica.cursor() as cur_replica: + lag = measure_replication_lag(cur_master, cur_replica) + log.info(f"Replica lagged behind master by {lag} seconds") + zenbenchmark.record("replica_lag", lag, "s", MetricReport.LOWER_IS_BETTER) + finally: + replica_workload.terminate() + finally: + master_workload.terminate() + except Exception as e: + error_occurred = True + log.error(f"Caught exception: {e}") + log.error(traceback.format_exc()) + finally: + assert not error_occurred # Fail the test if an error occurred + neon_api.delete_project(project_id) + + +def report_pgbench_aggregate_intervals( + output_dir: Path, + prefix: str, + zenbenchmark: NeonBenchmarker, +): + for filename in os.listdir(output_dir): + if filename.startswith(prefix): + # The file will be in the form _. + # So we first lop off the ., and then lop off the prefix and the _ + node = filename.split(".")[0][len(prefix) + 1 :] + with open(output_dir / filename) as f: + reader = csv.reader(f, delimiter=" ") + for line in reader: + num_transactions = int(line[1]) + if num_transactions == 0: + continue + sum_latency = int(line[2]) + sum_lag = int(line[3]) + zenbenchmark.record( + f"{node}_num_txns", num_transactions, "txns", MetricReport.HIGHER_IS_BETTER + ) + zenbenchmark.record( + f"{node}_avg_latency", + sum_latency / num_transactions, + "s", + MetricReport.LOWER_IS_BETTER, + ) + zenbenchmark.record( + f"{node}_avg_lag", + sum_lag / num_transactions, + "s", + MetricReport.LOWER_IS_BETTER, + ) + + +@pytest.mark.remote_cluster +@pytest.mark.timeout(2 * 60 * 60) +def test_replication_start_stop( + pg_bin: PgBin, + test_output_dir: Path, + neon_api: NeonAPI, + pg_version: PgVersion, + zenbenchmark: NeonBenchmarker, +): + """ + Cycles through different configurations of read replicas being enabled disabled. The whole time, + there's a pgbench read/write workload going on the master. For each replica, we either turn it + on or off, and see how long it takes to catch up after some set amount of time of replicating + the pgbench. + """ + + prefix = "pgbench_agg" + num_replicas = 2 + configuration_test_time_sec = 10 * 60 + pgbench_duration = f"-T{2 ** num_replicas * configuration_test_time_sec}" + error_occurred = False + + project = neon_api.create_project(pg_version) + project_id = project["project"]["id"] + neon_api.wait_for_operation_to_finish(project_id) + try: + branch_id = project["branch"]["id"] + master_connstr = project["connection_uris"][0]["connection_uri"] + master_env = connection_parameters_to_env( + project["connection_uris"][0]["connection_parameters"] + ) + + replicas = [] + for _ in range(num_replicas): + replicas.append( + neon_api.create_endpoint( + project_id, + branch_id, + endpoint_type="read_only", + settings={"pg_settings": {"hot_standby_feedback": "on"}}, + ) + ) + neon_api.wait_for_operation_to_finish(project_id) + + replica_connstr = [ + neon_api.get_connection_uri( + project_id, + endpoint_id=replicas[i]["endpoint"]["id"], + )["uri"] + for i in range(num_replicas) + ] + replica_env = [master_env.copy() for _ in range(num_replicas)] + for i in range(num_replicas): + replica_env[i]["PGHOST"] = replicas[i]["endpoint"]["host"] + + pg_bin.run_capture(["pgbench", "-i", "-s10"], env=master_env) + + # Sync replicas + with psycopg2.connect(master_connstr) as conn_master: + with conn_master.cursor() as cur_master: + for i in range(num_replicas): + conn_replica = psycopg2.connect(replica_connstr[i]) + measure_replication_lag(cur_master, conn_replica.cursor()) + + master_pgbench = pg_bin.run_nonblocking( + [ + "pgbench", + "-c10", + pgbench_duration, + "-Mprepared", + "--log", + f"--log-prefix={test_output_dir}/{prefix}_master", + f"--aggregate-interval={configuration_test_time_sec}", + ], + env=master_env, + ) + replica_pgbench: List[Optional[subprocess.Popen[Any]]] = [None for _ in range(num_replicas)] + + # Use the bits of iconfig to tell us which configuration we are on. For example + # a iconfig of 2 is 10 in binary, indicating replica 0 is suspended and replica 1 is + # alive. + for iconfig in range((1 << num_replicas) - 1, -1, -1): + + def replica_enabled(iconfig: int = iconfig): + return bool((iconfig >> 1) & 1) + + # Change configuration + for ireplica in range(num_replicas): + if replica_enabled() and replica_pgbench[ireplica] is None: + replica_pgbench[ireplica] = pg_bin.run_nonblocking( + [ + "pgbench", + "-c10", + "-S", + pgbench_duration, + "--log", + f"--log-prefix={test_output_dir}/{prefix}_replica_{ireplica}", + f"--aggregate-interval={configuration_test_time_sec}", + ], + env=replica_env[ireplica], + ) + elif not replica_enabled() and replica_pgbench[ireplica] is not None: + pgb = replica_pgbench[ireplica] + assert pgb is not None + pgb.terminate() + pgb.wait() + replica_pgbench[ireplica] = None + + neon_api.suspend_endpoint( + project_id, + replicas[ireplica]["endpoint"]["id"], + ) + neon_api.wait_for_operation_to_finish(project_id) + + time.sleep(configuration_test_time_sec) + + with psycopg2.connect(master_connstr) as conn_master: + with conn_master.cursor() as cur_master: + for ireplica in range(num_replicas): + replica_conn = psycopg2.connect(replica_connstr[ireplica]) + lag = measure_replication_lag(cur_master, replica_conn.cursor()) + zenbenchmark.record( + f"Replica {ireplica} lag", lag, "s", MetricReport.LOWER_IS_BETTER + ) + log.info( + f"Replica {ireplica} lagging behind master by {lag} seconds after configuration {iconfig:>b}" + ) + master_pgbench.terminate() + except Exception as e: + error_occurred = True + log.error(f"Caught exception {e}") + log.error(traceback.format_exc()) + finally: + assert not error_occurred + neon_api.delete_project(project_id) + # Only report results if we didn't error out + report_pgbench_aggregate_intervals(test_output_dir, prefix, zenbenchmark) From fcdf060816b50efe840907748fe3d856277a4e80 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 8 Jul 2024 15:39:41 +0100 Subject: [PATCH 062/194] pageserver: respect has_relmap_file in collect_keyspace (#8276) ## Problem Rarely, a dbdir entry can exist with no `relmap_file_key` data. This causes compaction to fail, because it assumes that if the database exists, then so does the relmap file. Basebackup already handled this using a boolean to record whether such a key exists, but `collect_keyspace` didn't. ## Summary of changes - Respect the flag for whether a relfilemap exists in collect_keyspace - The reproducer for this issue will merge separately in https://github.com/neondatabase/neon/pull/8232 --- pageserver/src/pgdatadir_mapping.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index fefd8d88ff21..8a6cfea92b3b 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -854,13 +854,14 @@ impl Timeline { result.add_key(DBDIR_KEY); // Fetch list of database dirs and iterate them - let buf = self.get(DBDIR_KEY, lsn, ctx).await?; - let dbdir = DbDirectory::des(&buf)?; + let dbdir = self.list_dbdirs(lsn, ctx).await?; + let mut dbs: Vec<((Oid, Oid), bool)> = dbdir.into_iter().collect(); - let mut dbs: Vec<(Oid, Oid)> = dbdir.dbdirs.keys().cloned().collect(); - dbs.sort_unstable(); - for (spcnode, dbnode) in dbs { - result.add_key(relmap_file_key(spcnode, dbnode)); + dbs.sort_unstable_by(|(k_a, _), (k_b, _)| k_a.cmp(k_b)); + for ((spcnode, dbnode), has_relmap_file) in dbs { + if has_relmap_file { + result.add_key(relmap_file_key(spcnode, dbnode)); + } result.add_key(rel_dir_to_key(spcnode, dbnode)); let mut rels: Vec = self From a68edad913fa54d3d12f0cbd6816b7b3ab8d7676 Mon Sep 17 00:00:00 2001 From: Yuchen Liang <70461588+yliang412@users.noreply.github.com> Date: Mon, 8 Jul 2024 10:43:10 -0400 Subject: [PATCH 063/194] refactor: move part of sharding API from `pageserver_api` to `utils` (#8254) ## Problem LSN Leases introduced in #8084 is a new API that is made shard-aware from day 1. To support ephemeral endpoint in #7994 without linking Postgres C API against `compute_ctl`, part of the sharding needs to reside in `utils`. ## Summary of changes - Create a new `shard` module in utils crate. - Move more interface related part of tenant sharding API to utils and re-export them in pageserver_api. Signed-off-by: Yuchen Liang --- libs/pageserver_api/src/shard.rs | 516 +++---------------------------- libs/utils/src/lib.rs | 2 + libs/utils/src/shard.rs | 451 +++++++++++++++++++++++++++ 3 files changed, 490 insertions(+), 479 deletions(-) create mode 100644 libs/utils/src/shard.rs diff --git a/libs/pageserver_api/src/shard.rs b/libs/pageserver_api/src/shard.rs index 8c5a4e616869..e83cf4c855a1 100644 --- a/libs/pageserver_api/src/shard.rs +++ b/libs/pageserver_api/src/shard.rs @@ -1,59 +1,42 @@ -use std::{ops::RangeInclusive, str::FromStr}; +//! See docs/rfcs/031-sharding-static.md for an overview of sharding. +//! +//! This module contains a variety of types used to represent the concept of sharding +//! a Neon tenant across multiple physical shards. Since there are quite a few of these, +//! we provide an summary here. +//! +//! Types used to describe shards: +//! - [`ShardCount`] describes how many shards make up a tenant, plus the magic `unsharded` value +//! which identifies a tenant which is not shard-aware. This means its storage paths do not include +//! a shard suffix. +//! - [`ShardNumber`] is simply the zero-based index of a shard within a tenant. +//! - [`ShardIndex`] is the 2-tuple of `ShardCount` and `ShardNumber`, it's just like a `TenantShardId` +//! without the tenant ID. This is useful for things that are implicitly scoped to a particular +//! tenant, such as layer files. +//! - [`ShardIdentity`]` is the full description of a particular shard's parameters, in sufficient +//! detail to convert a [`Key`] to a [`ShardNumber`] when deciding where to write/read. +//! - The [`ShardSlug`] is a terse formatter for ShardCount and ShardNumber, written as +//! four hex digits. An unsharded tenant is `0000`. +//! - [`TenantShardId`] is the unique ID of a particular shard within a particular tenant +//! +//! Types used to describe the parameters for data distribution in a sharded tenant: +//! - [`ShardStripeSize`] controls how long contiguous runs of [`Key`]s (stripes) are when distributed across +//! multiple shards. Its value is given in 8kiB pages. +//! - [`ShardLayout`] describes the data distribution scheme, and at time of writing is +//! always zero: this is provided for future upgrades that might introduce different +//! data distribution schemes. +//! +//! Examples: +//! - A legacy unsharded tenant has one shard with ShardCount(0), ShardNumber(0), and its slug is 0000 +//! - A single sharded tenant has one shard with ShardCount(1), ShardNumber(0), and its slug is 0001 +//! - In a tenant with 4 shards, each shard has ShardCount(N), ShardNumber(i) where i in 0..N-1 (inclusive), +//! and their slugs are 0004, 0104, 0204, and 0304. use crate::{key::Key, models::ShardParameters}; -use hex::FromHex; use postgres_ffi::relfile_utils::INIT_FORKNUM; use serde::{Deserialize, Serialize}; -use utils::id::TenantId; -/// See docs/rfcs/031-sharding-static.md for an overview of sharding. -/// -/// This module contains a variety of types used to represent the concept of sharding -/// a Neon tenant across multiple physical shards. Since there are quite a few of these, -/// we provide an summary here. -/// -/// Types used to describe shards: -/// - [`ShardCount`] describes how many shards make up a tenant, plus the magic `unsharded` value -/// which identifies a tenant which is not shard-aware. This means its storage paths do not include -/// a shard suffix. -/// - [`ShardNumber`] is simply the zero-based index of a shard within a tenant. -/// - [`ShardIndex`] is the 2-tuple of `ShardCount` and `ShardNumber`, it's just like a `TenantShardId` -/// without the tenant ID. This is useful for things that are implicitly scoped to a particular -/// tenant, such as layer files. -/// - [`ShardIdentity`]` is the full description of a particular shard's parameters, in sufficient -/// detail to convert a [`Key`] to a [`ShardNumber`] when deciding where to write/read. -/// - The [`ShardSlug`] is a terse formatter for ShardCount and ShardNumber, written as -/// four hex digits. An unsharded tenant is `0000`. -/// - [`TenantShardId`] is the unique ID of a particular shard within a particular tenant -/// -/// Types used to describe the parameters for data distribution in a sharded tenant: -/// - [`ShardStripeSize`] controls how long contiguous runs of [`Key`]s (stripes) are when distributed across -/// multiple shards. Its value is given in 8kiB pages. -/// - [`ShardLayout`] describes the data distribution scheme, and at time of writing is -/// always zero: this is provided for future upgrades that might introduce different -/// data distribution schemes. -/// -/// Examples: -/// - A legacy unsharded tenant has one shard with ShardCount(0), ShardNumber(0), and its slug is 0000 -/// - A single sharded tenant has one shard with ShardCount(1), ShardNumber(0), and its slug is 0001 -/// - In a tenant with 4 shards, each shard has ShardCount(N), ShardNumber(i) where i in 0..N-1 (inclusive), -/// and their slugs are 0004, 0104, 0204, and 0304. - -#[derive(Ord, PartialOrd, Eq, PartialEq, Clone, Copy, Serialize, Deserialize, Debug, Hash)] -pub struct ShardNumber(pub u8); - -#[derive(Ord, PartialOrd, Eq, PartialEq, Clone, Copy, Serialize, Deserialize, Debug, Hash)] -pub struct ShardCount(u8); - -/// Combination of ShardNumber and ShardCount. For use within the context of a particular tenant, -/// when we need to know which shard we're dealing with, but do not need to know the full -/// ShardIdentity (because we won't be doing any page->shard mapping), and do not need to know -/// the fully qualified TenantShardId. -#[derive(Eq, PartialEq, PartialOrd, Ord, Clone, Copy, Hash)] -pub struct ShardIndex { - pub shard_number: ShardNumber, - pub shard_count: ShardCount, -} +#[doc(inline)] +pub use ::utils::shard::*; /// The ShardIdentity contains enough information to map a [`Key`] to a [`ShardNumber`], /// and to check whether that [`ShardNumber`] is the same as the current shard. @@ -65,362 +48,6 @@ pub struct ShardIdentity { layout: ShardLayout, } -/// Formatting helper, for generating the `shard_id` label in traces. -struct ShardSlug<'a>(&'a TenantShardId); - -/// TenantShardId globally identifies a particular shard in a particular tenant. -/// -/// These are written as `-`, for example: -/// # The second shard in a two-shard tenant -/// 072f1291a5310026820b2fe4b2968934-0102 -/// -/// If the `ShardCount` is _unsharded_, the `TenantShardId` is written without -/// a shard suffix and is equivalent to the encoding of a `TenantId`: this enables -/// an unsharded [`TenantShardId`] to be used interchangably with a [`TenantId`]. -/// -/// The human-readable encoding of an unsharded TenantShardId, such as used in API URLs, -/// is both forward and backward compatible with TenantId: a legacy TenantId can be -/// decoded as a TenantShardId, and when re-encoded it will be parseable -/// as a TenantId. -#[derive(Eq, PartialEq, PartialOrd, Ord, Clone, Copy, Hash)] -pub struct TenantShardId { - pub tenant_id: TenantId, - pub shard_number: ShardNumber, - pub shard_count: ShardCount, -} - -impl ShardCount { - pub const MAX: Self = Self(u8::MAX); - - /// The internal value of a ShardCount may be zero, which means "1 shard, but use - /// legacy format for TenantShardId that excludes the shard suffix", also known - /// as [`TenantShardId::unsharded`]. - /// - /// This method returns the actual number of shards, i.e. if our internal value is - /// zero, we return 1 (unsharded tenants have 1 shard). - pub fn count(&self) -> u8 { - if self.0 > 0 { - self.0 - } else { - 1 - } - } - - /// The literal internal value: this is **not** the number of shards in the - /// tenant, as we have a special zero value for legacy unsharded tenants. Use - /// [`Self::count`] if you want to know the cardinality of shards. - pub fn literal(&self) -> u8 { - self.0 - } - - /// Whether the `ShardCount` is for an unsharded tenant, so uses one shard but - /// uses the legacy format for `TenantShardId`. See also the documentation for - /// [`Self::count`]. - pub fn is_unsharded(&self) -> bool { - self.0 == 0 - } - - /// `v` may be zero, or the number of shards in the tenant. `v` is what - /// [`Self::literal`] would return. - pub const fn new(val: u8) -> Self { - Self(val) - } -} - -impl ShardNumber { - pub const MAX: Self = Self(u8::MAX); -} - -impl TenantShardId { - pub fn unsharded(tenant_id: TenantId) -> Self { - Self { - tenant_id, - shard_number: ShardNumber(0), - shard_count: ShardCount(0), - } - } - - /// The range of all TenantShardId that belong to a particular TenantId. This is useful when - /// you have a BTreeMap of TenantShardId, and are querying by TenantId. - pub fn tenant_range(tenant_id: TenantId) -> RangeInclusive { - RangeInclusive::new( - Self { - tenant_id, - shard_number: ShardNumber(0), - shard_count: ShardCount(0), - }, - Self { - tenant_id, - shard_number: ShardNumber::MAX, - shard_count: ShardCount::MAX, - }, - ) - } - - pub fn shard_slug(&self) -> impl std::fmt::Display + '_ { - ShardSlug(self) - } - - /// Convenience for code that has special behavior on the 0th shard. - pub fn is_shard_zero(&self) -> bool { - self.shard_number == ShardNumber(0) - } - - /// The "unsharded" value is distinct from simply having a single shard: it represents - /// a tenant which is not shard-aware at all, and whose storage paths will not include - /// a shard suffix. - pub fn is_unsharded(&self) -> bool { - self.shard_number == ShardNumber(0) && self.shard_count.is_unsharded() - } - - /// Convenience for dropping the tenant_id and just getting the ShardIndex: this - /// is useful when logging from code that is already in a span that includes tenant ID, to - /// keep messages reasonably terse. - pub fn to_index(&self) -> ShardIndex { - ShardIndex { - shard_number: self.shard_number, - shard_count: self.shard_count, - } - } - - /// Calculate the children of this TenantShardId when splitting the overall tenant into - /// the given number of shards. - pub fn split(&self, new_shard_count: ShardCount) -> Vec { - let effective_old_shard_count = std::cmp::max(self.shard_count.0, 1); - let mut child_shards = Vec::new(); - for shard_number in 0..ShardNumber(new_shard_count.0).0 { - // Key mapping is based on a round robin mapping of key hash modulo shard count, - // so our child shards are the ones which the same keys would map to. - if shard_number % effective_old_shard_count == self.shard_number.0 { - child_shards.push(TenantShardId { - tenant_id: self.tenant_id, - shard_number: ShardNumber(shard_number), - shard_count: new_shard_count, - }) - } - } - - child_shards - } -} - -impl<'a> std::fmt::Display for ShardSlug<'a> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{:02x}{:02x}", - self.0.shard_number.0, self.0.shard_count.0 - ) - } -} - -impl std::fmt::Display for TenantShardId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - if self.shard_count != ShardCount(0) { - write!(f, "{}-{}", self.tenant_id, self.shard_slug()) - } else { - // Legacy case (shard_count == 0) -- format as just the tenant id. Note that this - // is distinct from the normal single shard case (shard count == 1). - self.tenant_id.fmt(f) - } - } -} - -impl std::fmt::Debug for TenantShardId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - // Debug is the same as Display: the compact hex representation - write!(f, "{}", self) - } -} - -impl std::str::FromStr for TenantShardId { - type Err = hex::FromHexError; - - fn from_str(s: &str) -> Result { - // Expect format: 16 byte TenantId, '-', 1 byte shard number, 1 byte shard count - if s.len() == 32 { - // Legacy case: no shard specified - Ok(Self { - tenant_id: TenantId::from_str(s)?, - shard_number: ShardNumber(0), - shard_count: ShardCount(0), - }) - } else if s.len() == 37 { - let bytes = s.as_bytes(); - let tenant_id = TenantId::from_hex(&bytes[0..32])?; - let mut shard_parts: [u8; 2] = [0u8; 2]; - hex::decode_to_slice(&bytes[33..37], &mut shard_parts)?; - Ok(Self { - tenant_id, - shard_number: ShardNumber(shard_parts[0]), - shard_count: ShardCount(shard_parts[1]), - }) - } else { - Err(hex::FromHexError::InvalidStringLength) - } - } -} - -impl From<[u8; 18]> for TenantShardId { - fn from(b: [u8; 18]) -> Self { - let tenant_id_bytes: [u8; 16] = b[0..16].try_into().unwrap(); - - Self { - tenant_id: TenantId::from(tenant_id_bytes), - shard_number: ShardNumber(b[16]), - shard_count: ShardCount(b[17]), - } - } -} - -impl ShardIndex { - pub fn new(number: ShardNumber, count: ShardCount) -> Self { - Self { - shard_number: number, - shard_count: count, - } - } - pub fn unsharded() -> Self { - Self { - shard_number: ShardNumber(0), - shard_count: ShardCount(0), - } - } - - /// The "unsharded" value is distinct from simply having a single shard: it represents - /// a tenant which is not shard-aware at all, and whose storage paths will not include - /// a shard suffix. - pub fn is_unsharded(&self) -> bool { - self.shard_number == ShardNumber(0) && self.shard_count == ShardCount(0) - } - - /// For use in constructing remote storage paths: concatenate this with a TenantId - /// to get a fully qualified TenantShardId. - /// - /// Backward compat: this function returns an empty string if Self::is_unsharded, such - /// that the legacy pre-sharding remote key format is preserved. - pub fn get_suffix(&self) -> String { - if self.is_unsharded() { - "".to_string() - } else { - format!("-{:02x}{:02x}", self.shard_number.0, self.shard_count.0) - } - } -} - -impl std::fmt::Display for ShardIndex { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:02x}{:02x}", self.shard_number.0, self.shard_count.0) - } -} - -impl std::fmt::Debug for ShardIndex { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - // Debug is the same as Display: the compact hex representation - write!(f, "{}", self) - } -} - -impl std::str::FromStr for ShardIndex { - type Err = hex::FromHexError; - - fn from_str(s: &str) -> Result { - // Expect format: 1 byte shard number, 1 byte shard count - if s.len() == 4 { - let bytes = s.as_bytes(); - let mut shard_parts: [u8; 2] = [0u8; 2]; - hex::decode_to_slice(bytes, &mut shard_parts)?; - Ok(Self { - shard_number: ShardNumber(shard_parts[0]), - shard_count: ShardCount(shard_parts[1]), - }) - } else { - Err(hex::FromHexError::InvalidStringLength) - } - } -} - -impl From<[u8; 2]> for ShardIndex { - fn from(b: [u8; 2]) -> Self { - Self { - shard_number: ShardNumber(b[0]), - shard_count: ShardCount(b[1]), - } - } -} - -impl Serialize for TenantShardId { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - if serializer.is_human_readable() { - serializer.collect_str(self) - } else { - // Note: while human encoding of [`TenantShardId`] is backward and forward - // compatible, this binary encoding is not. - let mut packed: [u8; 18] = [0; 18]; - packed[0..16].clone_from_slice(&self.tenant_id.as_arr()); - packed[16] = self.shard_number.0; - packed[17] = self.shard_count.0; - - packed.serialize(serializer) - } - } -} - -impl<'de> Deserialize<'de> for TenantShardId { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - struct IdVisitor { - is_human_readable_deserializer: bool, - } - - impl<'de> serde::de::Visitor<'de> for IdVisitor { - type Value = TenantShardId; - - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - if self.is_human_readable_deserializer { - formatter.write_str("value in form of hex string") - } else { - formatter.write_str("value in form of integer array([u8; 18])") - } - } - - fn visit_seq(self, seq: A) -> Result - where - A: serde::de::SeqAccess<'de>, - { - let s = serde::de::value::SeqAccessDeserializer::new(seq); - let id: [u8; 18] = Deserialize::deserialize(s)?; - Ok(TenantShardId::from(id)) - } - - fn visit_str(self, v: &str) -> Result - where - E: serde::de::Error, - { - TenantShardId::from_str(v).map_err(E::custom) - } - } - - if deserializer.is_human_readable() { - deserializer.deserialize_str(IdVisitor { - is_human_readable_deserializer: true, - }) - } else { - deserializer.deserialize_tuple( - 18, - IdVisitor { - is_human_readable_deserializer: false, - }, - ) - } - } -} - /// Stripe size in number of pages #[derive(Clone, Copy, Serialize, Deserialize, Eq, PartialEq, Debug)] pub struct ShardStripeSize(pub u32); @@ -585,77 +212,6 @@ impl ShardIdentity { } } -impl Serialize for ShardIndex { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - if serializer.is_human_readable() { - serializer.collect_str(self) - } else { - // Binary encoding is not used in index_part.json, but is included in anticipation of - // switching various structures (e.g. inter-process communication, remote metadata) to more - // compact binary encodings in future. - let mut packed: [u8; 2] = [0; 2]; - packed[0] = self.shard_number.0; - packed[1] = self.shard_count.0; - packed.serialize(serializer) - } - } -} - -impl<'de> Deserialize<'de> for ShardIndex { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - struct IdVisitor { - is_human_readable_deserializer: bool, - } - - impl<'de> serde::de::Visitor<'de> for IdVisitor { - type Value = ShardIndex; - - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - if self.is_human_readable_deserializer { - formatter.write_str("value in form of hex string") - } else { - formatter.write_str("value in form of integer array([u8; 2])") - } - } - - fn visit_seq(self, seq: A) -> Result - where - A: serde::de::SeqAccess<'de>, - { - let s = serde::de::value::SeqAccessDeserializer::new(seq); - let id: [u8; 2] = Deserialize::deserialize(s)?; - Ok(ShardIndex::from(id)) - } - - fn visit_str(self, v: &str) -> Result - where - E: serde::de::Error, - { - ShardIndex::from_str(v).map_err(E::custom) - } - } - - if deserializer.is_human_readable() { - deserializer.deserialize_str(IdVisitor { - is_human_readable_deserializer: true, - }) - } else { - deserializer.deserialize_tuple( - 2, - IdVisitor { - is_human_readable_deserializer: false, - }, - ) - } - } -} - /// Whether this key is always held on shard 0 (e.g. shard 0 holds all SLRU keys /// in order to be able to serve basebackup requests without peer communication). fn key_is_shard0(key: &Key) -> bool { @@ -737,7 +293,9 @@ pub fn describe( #[cfg(test)] mod tests { - use utils::Hex; + use std::str::FromStr; + + use utils::{id::TenantId, Hex}; use super::*; diff --git a/libs/utils/src/lib.rs b/libs/utils/src/lib.rs index 2a397d97d2b9..711e617801ea 100644 --- a/libs/utils/src/lib.rs +++ b/libs/utils/src/lib.rs @@ -26,6 +26,8 @@ pub mod auth; // utility functions and helper traits for unified unique id generation/serialization etc. pub mod id; +pub mod shard; + mod hex; pub use hex::Hex; diff --git a/libs/utils/src/shard.rs b/libs/utils/src/shard.rs new file mode 100644 index 000000000000..4f9ac6bdb49a --- /dev/null +++ b/libs/utils/src/shard.rs @@ -0,0 +1,451 @@ +//! See `pageserver_api::shard` for description on sharding. + +use std::{ops::RangeInclusive, str::FromStr}; + +use hex::FromHex; +use serde::{Deserialize, Serialize}; + +use crate::id::TenantId; + +#[derive(Ord, PartialOrd, Eq, PartialEq, Clone, Copy, Serialize, Deserialize, Debug, Hash)] +pub struct ShardNumber(pub u8); + +#[derive(Ord, PartialOrd, Eq, PartialEq, Clone, Copy, Serialize, Deserialize, Debug, Hash)] +pub struct ShardCount(pub u8); + +/// Combination of ShardNumber and ShardCount. For use within the context of a particular tenant, +/// when we need to know which shard we're dealing with, but do not need to know the full +/// ShardIdentity (because we won't be doing any page->shard mapping), and do not need to know +/// the fully qualified TenantShardId. +#[derive(Eq, PartialEq, PartialOrd, Ord, Clone, Copy, Hash)] +pub struct ShardIndex { + pub shard_number: ShardNumber, + pub shard_count: ShardCount, +} + +/// Formatting helper, for generating the `shard_id` label in traces. +pub struct ShardSlug<'a>(&'a TenantShardId); + +/// TenantShardId globally identifies a particular shard in a particular tenant. +/// +/// These are written as `-`, for example: +/// # The second shard in a two-shard tenant +/// 072f1291a5310026820b2fe4b2968934-0102 +/// +/// If the `ShardCount` is _unsharded_, the `TenantShardId` is written without +/// a shard suffix and is equivalent to the encoding of a `TenantId`: this enables +/// an unsharded [`TenantShardId`] to be used interchangably with a [`TenantId`]. +/// +/// The human-readable encoding of an unsharded TenantShardId, such as used in API URLs, +/// is both forward and backward compatible with TenantId: a legacy TenantId can be +/// decoded as a TenantShardId, and when re-encoded it will be parseable +/// as a TenantId. +#[derive(Eq, PartialEq, PartialOrd, Ord, Clone, Copy, Hash)] +pub struct TenantShardId { + pub tenant_id: TenantId, + pub shard_number: ShardNumber, + pub shard_count: ShardCount, +} + +impl ShardCount { + pub const MAX: Self = Self(u8::MAX); + + /// The internal value of a ShardCount may be zero, which means "1 shard, but use + /// legacy format for TenantShardId that excludes the shard suffix", also known + /// as [`TenantShardId::unsharded`]. + /// + /// This method returns the actual number of shards, i.e. if our internal value is + /// zero, we return 1 (unsharded tenants have 1 shard). + pub fn count(&self) -> u8 { + if self.0 > 0 { + self.0 + } else { + 1 + } + } + + /// The literal internal value: this is **not** the number of shards in the + /// tenant, as we have a special zero value for legacy unsharded tenants. Use + /// [`Self::count`] if you want to know the cardinality of shards. + pub fn literal(&self) -> u8 { + self.0 + } + + /// Whether the `ShardCount` is for an unsharded tenant, so uses one shard but + /// uses the legacy format for `TenantShardId`. See also the documentation for + /// [`Self::count`]. + pub fn is_unsharded(&self) -> bool { + self.0 == 0 + } + + /// `v` may be zero, or the number of shards in the tenant. `v` is what + /// [`Self::literal`] would return. + pub const fn new(val: u8) -> Self { + Self(val) + } +} + +impl ShardNumber { + pub const MAX: Self = Self(u8::MAX); +} + +impl TenantShardId { + pub fn unsharded(tenant_id: TenantId) -> Self { + Self { + tenant_id, + shard_number: ShardNumber(0), + shard_count: ShardCount(0), + } + } + + /// The range of all TenantShardId that belong to a particular TenantId. This is useful when + /// you have a BTreeMap of TenantShardId, and are querying by TenantId. + pub fn tenant_range(tenant_id: TenantId) -> RangeInclusive { + RangeInclusive::new( + Self { + tenant_id, + shard_number: ShardNumber(0), + shard_count: ShardCount(0), + }, + Self { + tenant_id, + shard_number: ShardNumber::MAX, + shard_count: ShardCount::MAX, + }, + ) + } + + pub fn shard_slug(&self) -> impl std::fmt::Display + '_ { + ShardSlug(self) + } + + /// Convenience for code that has special behavior on the 0th shard. + pub fn is_shard_zero(&self) -> bool { + self.shard_number == ShardNumber(0) + } + + /// The "unsharded" value is distinct from simply having a single shard: it represents + /// a tenant which is not shard-aware at all, and whose storage paths will not include + /// a shard suffix. + pub fn is_unsharded(&self) -> bool { + self.shard_number == ShardNumber(0) && self.shard_count.is_unsharded() + } + + /// Convenience for dropping the tenant_id and just getting the ShardIndex: this + /// is useful when logging from code that is already in a span that includes tenant ID, to + /// keep messages reasonably terse. + pub fn to_index(&self) -> ShardIndex { + ShardIndex { + shard_number: self.shard_number, + shard_count: self.shard_count, + } + } + + /// Calculate the children of this TenantShardId when splitting the overall tenant into + /// the given number of shards. + pub fn split(&self, new_shard_count: ShardCount) -> Vec { + let effective_old_shard_count = std::cmp::max(self.shard_count.0, 1); + let mut child_shards = Vec::new(); + for shard_number in 0..ShardNumber(new_shard_count.0).0 { + // Key mapping is based on a round robin mapping of key hash modulo shard count, + // so our child shards are the ones which the same keys would map to. + if shard_number % effective_old_shard_count == self.shard_number.0 { + child_shards.push(TenantShardId { + tenant_id: self.tenant_id, + shard_number: ShardNumber(shard_number), + shard_count: new_shard_count, + }) + } + } + + child_shards + } +} + +impl<'a> std::fmt::Display for ShardSlug<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{:02x}{:02x}", + self.0.shard_number.0, self.0.shard_count.0 + ) + } +} + +impl std::fmt::Display for TenantShardId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.shard_count != ShardCount(0) { + write!(f, "{}-{}", self.tenant_id, self.shard_slug()) + } else { + // Legacy case (shard_count == 0) -- format as just the tenant id. Note that this + // is distinct from the normal single shard case (shard count == 1). + self.tenant_id.fmt(f) + } + } +} + +impl std::fmt::Debug for TenantShardId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Debug is the same as Display: the compact hex representation + write!(f, "{}", self) + } +} + +impl std::str::FromStr for TenantShardId { + type Err = hex::FromHexError; + + fn from_str(s: &str) -> Result { + // Expect format: 16 byte TenantId, '-', 1 byte shard number, 1 byte shard count + if s.len() == 32 { + // Legacy case: no shard specified + Ok(Self { + tenant_id: TenantId::from_str(s)?, + shard_number: ShardNumber(0), + shard_count: ShardCount(0), + }) + } else if s.len() == 37 { + let bytes = s.as_bytes(); + let tenant_id = TenantId::from_hex(&bytes[0..32])?; + let mut shard_parts: [u8; 2] = [0u8; 2]; + hex::decode_to_slice(&bytes[33..37], &mut shard_parts)?; + Ok(Self { + tenant_id, + shard_number: ShardNumber(shard_parts[0]), + shard_count: ShardCount(shard_parts[1]), + }) + } else { + Err(hex::FromHexError::InvalidStringLength) + } + } +} + +impl From<[u8; 18]> for TenantShardId { + fn from(b: [u8; 18]) -> Self { + let tenant_id_bytes: [u8; 16] = b[0..16].try_into().unwrap(); + + Self { + tenant_id: TenantId::from(tenant_id_bytes), + shard_number: ShardNumber(b[16]), + shard_count: ShardCount(b[17]), + } + } +} + +impl ShardIndex { + pub fn new(number: ShardNumber, count: ShardCount) -> Self { + Self { + shard_number: number, + shard_count: count, + } + } + pub fn unsharded() -> Self { + Self { + shard_number: ShardNumber(0), + shard_count: ShardCount(0), + } + } + + /// The "unsharded" value is distinct from simply having a single shard: it represents + /// a tenant which is not shard-aware at all, and whose storage paths will not include + /// a shard suffix. + pub fn is_unsharded(&self) -> bool { + self.shard_number == ShardNumber(0) && self.shard_count == ShardCount(0) + } + + /// For use in constructing remote storage paths: concatenate this with a TenantId + /// to get a fully qualified TenantShardId. + /// + /// Backward compat: this function returns an empty string if Self::is_unsharded, such + /// that the legacy pre-sharding remote key format is preserved. + pub fn get_suffix(&self) -> String { + if self.is_unsharded() { + "".to_string() + } else { + format!("-{:02x}{:02x}", self.shard_number.0, self.shard_count.0) + } + } +} + +impl std::fmt::Display for ShardIndex { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:02x}{:02x}", self.shard_number.0, self.shard_count.0) + } +} + +impl std::fmt::Debug for ShardIndex { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Debug is the same as Display: the compact hex representation + write!(f, "{}", self) + } +} + +impl std::str::FromStr for ShardIndex { + type Err = hex::FromHexError; + + fn from_str(s: &str) -> Result { + // Expect format: 1 byte shard number, 1 byte shard count + if s.len() == 4 { + let bytes = s.as_bytes(); + let mut shard_parts: [u8; 2] = [0u8; 2]; + hex::decode_to_slice(bytes, &mut shard_parts)?; + Ok(Self { + shard_number: ShardNumber(shard_parts[0]), + shard_count: ShardCount(shard_parts[1]), + }) + } else { + Err(hex::FromHexError::InvalidStringLength) + } + } +} + +impl From<[u8; 2]> for ShardIndex { + fn from(b: [u8; 2]) -> Self { + Self { + shard_number: ShardNumber(b[0]), + shard_count: ShardCount(b[1]), + } + } +} + +impl Serialize for TenantShardId { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + if serializer.is_human_readable() { + serializer.collect_str(self) + } else { + // Note: while human encoding of [`TenantShardId`] is backward and forward + // compatible, this binary encoding is not. + let mut packed: [u8; 18] = [0; 18]; + packed[0..16].clone_from_slice(&self.tenant_id.as_arr()); + packed[16] = self.shard_number.0; + packed[17] = self.shard_count.0; + + packed.serialize(serializer) + } + } +} + +impl<'de> Deserialize<'de> for TenantShardId { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct IdVisitor { + is_human_readable_deserializer: bool, + } + + impl<'de> serde::de::Visitor<'de> for IdVisitor { + type Value = TenantShardId; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + if self.is_human_readable_deserializer { + formatter.write_str("value in form of hex string") + } else { + formatter.write_str("value in form of integer array([u8; 18])") + } + } + + fn visit_seq(self, seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let s = serde::de::value::SeqAccessDeserializer::new(seq); + let id: [u8; 18] = Deserialize::deserialize(s)?; + Ok(TenantShardId::from(id)) + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + TenantShardId::from_str(v).map_err(E::custom) + } + } + + if deserializer.is_human_readable() { + deserializer.deserialize_str(IdVisitor { + is_human_readable_deserializer: true, + }) + } else { + deserializer.deserialize_tuple( + 18, + IdVisitor { + is_human_readable_deserializer: false, + }, + ) + } + } +} + +impl Serialize for ShardIndex { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + if serializer.is_human_readable() { + serializer.collect_str(self) + } else { + // Binary encoding is not used in index_part.json, but is included in anticipation of + // switching various structures (e.g. inter-process communication, remote metadata) to more + // compact binary encodings in future. + let mut packed: [u8; 2] = [0; 2]; + packed[0] = self.shard_number.0; + packed[1] = self.shard_count.0; + packed.serialize(serializer) + } + } +} + +impl<'de> Deserialize<'de> for ShardIndex { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct IdVisitor { + is_human_readable_deserializer: bool, + } + + impl<'de> serde::de::Visitor<'de> for IdVisitor { + type Value = ShardIndex; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + if self.is_human_readable_deserializer { + formatter.write_str("value in form of hex string") + } else { + formatter.write_str("value in form of integer array([u8; 2])") + } + } + + fn visit_seq(self, seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let s = serde::de::value::SeqAccessDeserializer::new(seq); + let id: [u8; 2] = Deserialize::deserialize(s)?; + Ok(ShardIndex::from(id)) + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + ShardIndex::from_str(v).map_err(E::custom) + } + } + + if deserializer.is_human_readable() { + deserializer.deserialize_str(IdVisitor { + is_human_readable_deserializer: true, + }) + } else { + deserializer.deserialize_tuple( + 2, + IdVisitor { + is_human_readable_deserializer: false, + }, + ) + } + } +} From 84b039e615e9e7391e22e97fc5ee306cac29385b Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Mon, 8 Jul 2024 19:54:02 +0200 Subject: [PATCH 064/194] compute_ctl: Use 'fast' shutdown for Postgres termination (#8289) ## Problem We currently use 'immediate' mode in the most commonly used shutdown path, when the control plane calls a `compute_ctl` API to terminate Postgres inside compute without waiting for the actual pod / VM termination. Yet, 'immediate' shutdown doesn't create a shutdown checkpoint and ROs have bad times figuring out the list of running xacts during next start. ## Summary of changes Use 'fast' mode, which creates a shutdown checkpoint that is important for ROs to get a list of running xacts faster instead of going through the CLOG. On the control plane side, we poll this `compute_ctl` termination API for 10s, it should be enough as we don't really write any data at checkpoint time. If it times out, we anyway switch to the slow k8s-based termination. See https://www.postgresql.org/docs/current/server-shutdown.html for the list of modes and signals. The default VM shutdown hook already uses `fast` mode, see [1] [1] https://github.com/neondatabase/neon/blob/c9fd8d76937c2031fd4fea1cdf661d6cf4f00dc3/vm-image-spec.yaml#L30-L31 Related to #6211 --- compute_tools/src/compute.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index 41a52ef5b641..1fa2b9f71d64 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -1386,7 +1386,9 @@ pub fn forward_termination_signal() { let pg_pid = PG_PID.load(Ordering::SeqCst); if pg_pid != 0 { let pg_pid = nix::unistd::Pid::from_raw(pg_pid as i32); - // use 'immediate' shutdown (SIGQUIT): https://www.postgresql.org/docs/current/server-shutdown.html - kill(pg_pid, Signal::SIGQUIT).ok(); + // Use 'fast' shutdown (SIGINT) because it also creates a shutdown checkpoint, which is important for + // ROs to get a list of running xacts faster instead of going through the CLOG. + // See https://www.postgresql.org/docs/current/server-shutdown.html for the list of modes and signals. + kill(pg_pid, Signal::SIGINT).ok(); } } From daea26a22f98ca2399f55c0db7eb8932865d7ede Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 8 Jul 2024 20:05:35 +0100 Subject: [PATCH 065/194] tests: use smaller layers in test_pg_regress (#8232) ## Problem Debug-mode runs of test_pg_regress are rather slow since https://github.com/neondatabase/neon/pull/8105, and occasionally exceed their 600s timeout. ## Summary of changes - Use 8MiB layer files, avoiding large ephemeral layers On a hetzner AX102, this takes the runtime from 230s to 190s. Which hopefully will be enough to get the runtime on github runners more reliably below its 600s timeout. This has the side benefit of exercising more of the pageserver stack (including compaction) under a workload that exercises a more diverse set of postgres functionality than most of our tests. --- pageserver/src/tenant/timeline.rs | 3 + test_runner/regress/test_pg_regress.py | 182 ++++++++++++++++--------- 2 files changed, 118 insertions(+), 67 deletions(-) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 92baf1073aae..541704e8d668 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -728,6 +728,9 @@ impl From for CompactionError { fn from(e: CreateImageLayersError) -> Self { match e { CreateImageLayersError::Cancelled => CompactionError::ShuttingDown, + CreateImageLayersError::Other(e) => { + CompactionError::Other(e.context("create image layers")) + } _ => CompactionError::Other(e.into()), } } diff --git a/test_runner/regress/test_pg_regress.py b/test_runner/regress/test_pg_regress.py index 756a2c17c909..54b493ec705d 100644 --- a/test_runner/regress/test_pg_regress.py +++ b/test_runner/regress/test_pg_regress.py @@ -8,8 +8,11 @@ import pytest from fixtures.neon_fixtures import ( + Endpoint, + NeonEnv, NeonEnvBuilder, check_restored_datadir_content, + tenant_get_shards, ) from fixtures.pg_version import PgVersion from fixtures.remote_storage import s3_storage @@ -21,6 +24,97 @@ from pytest import CaptureFixture +TENANT_CONF = { + # Scaled down thresholds so that we are exercising the pageserver beyond just writing + # ephemeral/L0 layers, and because debug-mode code is slow to read from full sized ephemeral layer files. + "pitr_interval": "60s", + "checkpoint_distance": f"{8 * 1024 * 1024}", + "compaction_target_size": f"{8 * 1024 * 1024}", +} + +# # Ensure that compaction works, on a timeline containing all the diversity that postgres regression tests create. +# # There should have been compactions mid-test as well, this final check is in addition those. +# for (shard, pageserver) in tenant_get_shards(env, env.initial_tenant): +# pageserver.http_client().timeline_checkpoint(env.initial_tenant, env.initial_timeline, force_repartition=True, force_image_layer_creation=True) + + +def post_checks(env: NeonEnv, test_output_dir: Path, db_name: str, endpoint: Endpoint): + """ + After running some opaque tests that create interesting content in a timeline, run + some generic integrity checks that the storage stack is able to reproduce the written + data properly. + """ + + ignored_files: Optional[list[str]] = None + + # Neon handles unlogged relations in a special manner. During a + # basebackup, we ship the init fork as the main fork. This presents a + # problem in that the endpoint's data directory and the basebackup will + # have differences and will fail the eventual file comparison. + # + # Unlogged tables were introduced in version 9.1. ALTER TABLE grew + # support for setting the persistence of a table in 9.5. The reason that + # this doesn't affect versions < 15 (but probably would between 9.1 and + # 9.5) is that all the regression tests that deal with unlogged tables + # up until that point dropped the unlogged tables or set them to logged + # at some point during the test. + # + # In version 15, Postgres grew support for unlogged sequences, and with + # that came a few more regression tests. These tests did not all drop + # the unlogged tables/sequences prior to finishing. + # + # But unlogged sequences came with a bug in that, sequences didn't + # inherit the persistence of their "parent" tables if they had one. This + # was fixed and backported to 15, thus exacerbating our problem a bit. + # + # So what we can do is just ignore file differences between the data + # directory and basebackup for unlogged relations. + results = cast( + "list[tuple[str, str]]", + endpoint.safe_psql( + """ + SELECT + relkind, + pg_relation_filepath( + pg_filenode_relation(reltablespace, relfilenode) + ) AS unlogged_relation_paths + FROM pg_class + WHERE relpersistence = 'u' + """, + dbname=db_name, + ), + ) + + unlogged_relation_files: list[str] = [] + for r in results: + unlogged_relation_files.append(r[1]) + # This is related to the following Postgres commit: + # + # commit ccadf73163ca88bdaa74b8223d4dde05d17f550b + # Author: Heikki Linnakangas + # Date: 2023-08-23 09:21:31 -0500 + # + # Use the buffer cache when initializing an unlogged index. + # + # This patch was backpatched to 16. Without it, the LSN in the + # page header would be 0/0 in the data directory, which wouldn't + # match the LSN generated during the basebackup, thus creating + # a difference. + if env.pg_version <= PgVersion.V15 and r[0] == "i": + unlogged_relation_files.append(f"{r[1]}_init") + + ignored_files = unlogged_relation_files + + check_restored_datadir_content(test_output_dir, env, endpoint, ignored_files=ignored_files) + + # Ensure that compaction works, on a timeline containing all the diversity that postgres regression tests create. + # There should have been compactions mid-test as well, this final check is in addition those. + for shard, pageserver in tenant_get_shards(env, env.initial_tenant): + pageserver.http_client().timeline_checkpoint( + shard, env.initial_timeline, force_repartition=True, force_image_layer_creation=True + ) + + # Run the main PostgreSQL regression tests, in src/test/regress. # @pytest.mark.timeout(600) @@ -45,7 +139,10 @@ def test_pg_regress( neon_env_builder.enable_pageserver_remote_storage(s3_storage()) neon_env_builder.enable_scrub_on_exit() - env = neon_env_builder.init_start(initial_tenant_shard_count=shard_count) + env = neon_env_builder.init_start( + initial_tenant_conf=TENANT_CONF, + initial_tenant_shard_count=shard_count, + ) # Connect to postgres and create a database called "regression". endpoint = env.endpoints.create_start("main") @@ -84,67 +181,7 @@ def test_pg_regress( with capsys.disabled(): pg_bin.run(pg_regress_command, env=env_vars, cwd=runpath) - ignored_files: Optional[list[str]] = None - - # Neon handles unlogged relations in a special manner. During a - # basebackup, we ship the init fork as the main fork. This presents a - # problem in that the endpoint's data directory and the basebackup will - # have differences and will fail the eventual file comparison. - # - # Unlogged tables were introduced in version 9.1. ALTER TABLE grew - # support for setting the persistence of a table in 9.5. The reason that - # this doesn't affect versions < 15 (but probably would between 9.1 and - # 9.5) is that all the regression tests that deal with unlogged tables - # up until that point dropped the unlogged tables or set them to logged - # at some point during the test. - # - # In version 15, Postgres grew support for unlogged sequences, and with - # that came a few more regression tests. These tests did not all drop - # the unlogged tables/sequences prior to finishing. - # - # But unlogged sequences came with a bug in that, sequences didn't - # inherit the persistence of their "parent" tables if they had one. This - # was fixed and backported to 15, thus exacerbating our problem a bit. - # - # So what we can do is just ignore file differences between the data - # directory and basebackup for unlogged relations. - results = cast( - "list[tuple[str, str]]", - endpoint.safe_psql( - """ - SELECT - relkind, - pg_relation_filepath( - pg_filenode_relation(reltablespace, relfilenode) - ) AS unlogged_relation_paths - FROM pg_class - WHERE relpersistence = 'u' - """, - dbname=DBNAME, - ), - ) - - unlogged_relation_files: list[str] = [] - for r in results: - unlogged_relation_files.append(r[1]) - # This is related to the following Postgres commit: - # - # commit ccadf73163ca88bdaa74b8223d4dde05d17f550b - # Author: Heikki Linnakangas - # Date: 2023-08-23 09:21:31 -0500 - # - # Use the buffer cache when initializing an unlogged index. - # - # This patch was backpatched to 16. Without it, the LSN in the - # page header would be 0/0 in the data directory, which wouldn't - # match the LSN generated during the basebackup, thus creating - # a difference. - if env.pg_version <= PgVersion.V15 and r[0] == "i": - unlogged_relation_files.append(f"{r[1]}_init") - - ignored_files = unlogged_relation_files - - check_restored_datadir_content(test_output_dir, env, endpoint, ignored_files=ignored_files) + post_checks(env, test_output_dir, DBNAME, endpoint) # Run the PostgreSQL "isolation" tests, in src/test/isolation. @@ -159,16 +196,20 @@ def test_isolation( pg_distrib_dir: Path, shard_count: Optional[int], ): + DBNAME = "isolation_regression" + if shard_count is not None: neon_env_builder.num_pageservers = shard_count neon_env_builder.enable_pageserver_remote_storage(s3_storage()) neon_env_builder.enable_scrub_on_exit() - env = neon_env_builder.init_start(initial_tenant_shard_count=shard_count) + env = neon_env_builder.init_start( + initial_tenant_conf=TENANT_CONF, initial_tenant_shard_count=shard_count + ) # Connect to postgres and create a database called "regression". # isolation tests use prepared transactions, so enable them endpoint = env.endpoints.create_start("main", config_lines=["max_prepared_transactions=100"]) - endpoint.safe_psql("CREATE DATABASE isolation_regression") + endpoint.safe_psql(f"CREATE DATABASE {DBNAME}") # Create some local directories for pg_isolation_regress to run in. runpath = test_output_dir / "regress" @@ -202,6 +243,9 @@ def test_isolation( with capsys.disabled(): pg_bin.run(pg_isolation_regress_command, env=env_vars, cwd=runpath) + # This fails with a mismatch on `pg_multixact/offsets/0000` + # post_checks(env, test_output_dir, DBNAME, endpoint) + # Run extra Neon-specific pg_regress-based tests. The tests and their # schedule file are in the sql_regress/ directory. @@ -215,15 +259,19 @@ def test_sql_regress( pg_distrib_dir: Path, shard_count: Optional[int], ): + DBNAME = "regression" + if shard_count is not None: neon_env_builder.num_pageservers = shard_count neon_env_builder.enable_pageserver_remote_storage(s3_storage()) neon_env_builder.enable_scrub_on_exit() - env = neon_env_builder.init_start(initial_tenant_shard_count=shard_count) + env = neon_env_builder.init_start( + initial_tenant_conf=TENANT_CONF, initial_tenant_shard_count=shard_count + ) # Connect to postgres and create a database called "regression". endpoint = env.endpoints.create_start("main") - endpoint.safe_psql("CREATE DATABASE regression") + endpoint.safe_psql(f"CREATE DATABASE {DBNAME}") # Create some local directories for pg_regress to run in. runpath = test_output_dir / "regress" @@ -258,4 +306,4 @@ def test_sql_regress( with capsys.disabled(): pg_bin.run(pg_regress_command, env=env_vars, cwd=runpath) - check_restored_datadir_content(test_output_dir, env, endpoint) + post_checks(env, test_output_dir, DBNAME, endpoint) From df3dc6e4c1f13a36567813f7f445734dd3a8b902 Mon Sep 17 00:00:00 2001 From: Alex Chi Z Date: Mon, 8 Jul 2024 15:05:59 -0400 Subject: [PATCH 066/194] fix(pageserver): write to both v1+v2 for aux tenant import (#8316) close https://github.com/neondatabase/neon/issues/8202 ref https://github.com/neondatabase/neon/pull/6560 For tenant imports, we now write the aux files into both v1+v2 storage, so that the test case can pick either one for testing. Given the API is only used for testing, this looks like a safe change. Signed-off-by: Alex Chi Z --- storage_controller/src/service.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 3965d7453d49..78f0848c241e 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -4062,7 +4062,14 @@ impl Service { placement_policy: Some(PlacementPolicy::Attached(0)), // No secondaries, for convenient debug/hacking // There is no way to know what the tenant's config was: revert to defaults - config: TenantConfig::default(), + // + // TODO: remove `switch_aux_file_policy` once we finish auxv2 migration + // + // we write to both v1+v2 storage, so that the test case can use either storage format for testing + config: TenantConfig { + switch_aux_file_policy: Some(models::AuxFilePolicy::CrossValidation), + ..TenantConfig::default() + }, }) .await?; From 811eb88b89207be4342e2e8d4a7d6fc2328e6141 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 8 Jul 2024 21:06:34 +0100 Subject: [PATCH 067/194] tests: stabilize test_timeline_size_quota_on_startup (#8255) ## Problem `test_timeline_size_quota_on_startup` assumed that writing data beyond the size limit would always be blocked. This is not so: the limit is only enforced if feedback makes it back from the pageserver to the safekeeper + compute. Closes: https://github.com/neondatabase/neon/issues/6562 ## Summary of changes - Modify the test to wait for the pageserver to catch up. The size limit was never actually being enforced robustly, the original version of this test was just writing much more than 30MB and about 98% of the time getting lucky such that the feedback happened to arrive before the tests for loop was done. - If the test fails, log the logical size as seen by the pageserver. --- test_runner/regress/test_timeline_size.py | 46 +++++++++++++++++------ 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/test_runner/regress/test_timeline_size.py b/test_runner/regress/test_timeline_size.py index f47356839c26..5e9a42f6b41e 100644 --- a/test_runner/regress/test_timeline_size.py +++ b/test_runner/regress/test_timeline_size.py @@ -152,10 +152,12 @@ def test_timeline_size_quota_on_startup(neon_env_builder: NeonEnvBuilder): client.timeline_wait_logical_size(env.initial_tenant, new_timeline_id) + size_limit_mb = 30 + endpoint_main = env.endpoints.create( "test_timeline_size_quota_on_startup", # Set small limit for the test - config_lines=["neon.max_cluster_size=30MB"], + config_lines=[f"neon.max_cluster_size={size_limit_mb}MB"], ) endpoint_main.start() @@ -165,17 +167,39 @@ def test_timeline_size_quota_on_startup(neon_env_builder: NeonEnvBuilder): # Insert many rows. This query must fail because of space limit try: - for _i in range(5000): - cur.execute( - """ - INSERT INTO foo - SELECT 'long string to consume some space' || g - FROM generate_series(1, 100) g - """ - ) - # If we get here, the timeline size limit failed - log.error("Query unexpectedly succeeded") + def write_rows(count): + for _i in range(count): + cur.execute( + """ + INSERT INTO foo + SELECT 'long string to consume some space' || g + FROM generate_series(1, 100) g + """ + ) + + # Write some data that exceeds limit, then let the pageserver ingest it to guarantee that some feedback has made it to + # the safekeeper, then try to write some more. We expect either the initial writes or the ones after + # the wait_for_last_flush_lsn to generate an exception. + # + # Without the wait_for_last_flush_lsn, the size limit sometimes isn't enforced (see https://github.com/neondatabase/neon/issues/6562) + write_rows(2500) + wait_for_last_flush_lsn(env, endpoint_main, env.initial_tenant, new_timeline_id) + logical_size = env.pageserver.http_client().timeline_detail( + env.initial_tenant, new_timeline_id + )["current_logical_size"] + assert logical_size > size_limit_mb * 1024 * 1024 + write_rows(2500) + + # If we get here, the timeline size limit failed. Find out from the pageserver how large it + # thinks the timeline is. + wait_for_last_flush_lsn(env, endpoint_main, env.initial_tenant, new_timeline_id) + logical_size = env.pageserver.http_client().timeline_detail( + env.initial_tenant, new_timeline_id + )["current_logical_size"] + log.error( + f"Query unexpectedly succeeded, pageserver logical size is {logical_size}" + ) raise AssertionError() except psycopg2.errors.DiskFull as err: From d9c1068cf465c508205d58f5f0c962d6757babda Mon Sep 17 00:00:00 2001 From: Yuchen Liang <70461588+yliang412@users.noreply.github.com> Date: Mon, 8 Jul 2024 16:50:13 -0400 Subject: [PATCH 068/194] tests: increase approx size equal threshold to avoid `test_lsn_lease_size` flakiness (#8282) ## Summary of changes Increase the `assert_size_approx_equal` threshold to avoid flakiness of `test_lsn_lease_size`. Still needs more investigation to fully resolve #8293. - Also set `autovacuum=off` for the endpoint we are running in the test. Signed-off-by: Yuchen Liang --- test_runner/regress/test_tenant_size.py | 29 +++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/test_runner/regress/test_tenant_size.py b/test_runner/regress/test_tenant_size.py index 70e8fe67d595..b1ade77a1474 100644 --- a/test_runner/regress/test_tenant_size.py +++ b/test_runner/regress/test_tenant_size.py @@ -720,9 +720,30 @@ def test_lsn_lease_size(neon_env_builder: NeonEnvBuilder, test_output_dir: Path, They should have the same effect. """ + def assert_size_approx_equal_for_lease_test(size_lease, size_branch): + """ + Tests that evaluate sizes are checking the pageserver space consumption + that sits many layers below the user input. The exact space needed + varies slightly depending on postgres behavior. + + Rather than expecting postgres to be determinstic and occasionally + failing the test, we permit sizes for the same data to vary by a few pages. + """ + + # FIXME(yuchen): The delta is too large, used as temp solution to pass the test reliably. + # Investigate and reduce the threshold. + threshold = 22 * 8272 + + log.info( + f"delta: size_branch({size_branch}) - size_lease({size_lease}) = {size_branch - size_lease}" + ) + + assert size_lease == pytest.approx(size_branch, abs=threshold) + conf = { "pitr_interval": "0s" if zero_gc else "3600s", "gc_period": "0s", + "compaction_period": "0s", } env = neon_env_builder.init_start(initial_tenant_conf=conf) @@ -734,7 +755,7 @@ def test_lsn_lease_size(neon_env_builder: NeonEnvBuilder, test_output_dir: Path, tenant, timeline = env.neon_cli.create_tenant(conf=conf) lease_res = insert_with_action(env, tenant, timeline, test_output_dir, action="lease") - assert_size_approx_equal(lease_res, ro_branch_res) + assert_size_approx_equal_for_lease_test(lease_res, ro_branch_res) def insert_with_action( @@ -754,7 +775,11 @@ def insert_with_action( """ client = env.pageserver.http_client() - with env.endpoints.create_start("main", tenant_id=tenant) as ep: + with env.endpoints.create_start( + "main", + tenant_id=tenant, + config_lines=["autovacuum=off"], + ) as ep: initial_size = client.tenant_size(tenant) log.info(f"initial size: {initial_size}") From 8b15864f5927a3881e94c46a7b88f058a0659c2b Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Tue, 9 Jul 2024 09:39:10 +0100 Subject: [PATCH 069/194] CI(promote-compatibility-data): take into account commit sha (#8283) ## Problem In https://github.com/neondatabase/neon/pull/8161, we changed the path to Neon artefacts by adding commit sha to it, but we missed adding these changes to `promote-compatibility-data` job that we use for backward/forward- compatibility testing. ## Summary of changes - Add commit sha to `promote-compatibility-data` --- .github/workflows/build_and_test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index a3246987e2b8..cb7655e03908 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1336,6 +1336,7 @@ jobs: env: BUCKET: neon-github-public-dev PREFIX: artifacts/latest + COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} run: | # Update compatibility snapshot for the release for pg_version in v14 v15 v16; do @@ -1349,7 +1350,7 @@ jobs: # Update Neon artifact for the release (reuse already uploaded artifact) for build_type in debug release; do - OLD_PREFIX=artifacts/${GITHUB_RUN_ID} + OLD_PREFIX=artifacts/${COMMIT_SHA}/${GITHUB_RUN_ID} FILENAME=neon-${{ runner.os }}-${{ runner.arch }}-${build_type}-artifact.tar.zst S3_KEY=$(aws s3api list-objects-v2 --bucket ${BUCKET} --prefix ${OLD_PREFIX} | jq -r '.Contents[]?.Key' | grep ${FILENAME} | sort --version-sort | tail -1 || true) From c196cf6ac15170910c8deff40e5830379c31edbe Mon Sep 17 00:00:00 2001 From: Luca BRUNO Date: Tue, 9 Jul 2024 10:43:42 +0200 Subject: [PATCH 070/194] proxy/http: avoid spurious vector reallocations This tweaks the rows-to-JSON rendering logic in order to avoid allocating 0-sized temporary vectors and later growing them to insert elements. As the exact size is known in advance, both vectors can be built with an exact capacity upfront. This will avoid further vector growing/reallocation in the rendering hotpath. Signed-off-by: Luca BRUNO --- proxy/src/serverless/sql_over_http.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/proxy/src/serverless/sql_over_http.rs b/proxy/src/serverless/sql_over_http.rs index 583ff75f7ca7..8118ae5ea89d 100644 --- a/proxy/src/serverless/sql_over_http.rs +++ b/proxy/src/serverless/sql_over_http.rs @@ -838,8 +838,9 @@ async fn query_to_json( "finished reading rows" ); - let mut fields = vec![]; - let mut columns = vec![]; + let columns_len = row_stream.columns().len(); + let mut fields = Vec::with_capacity(columns_len); + let mut columns = Vec::with_capacity(columns_len); for c in row_stream.columns() { fields.push(json!({ From 73fa3c014bf4717615a453ccf0e50bca98ba64cf Mon Sep 17 00:00:00 2001 From: Alex Chi Z Date: Tue, 9 Jul 2024 12:11:37 -0400 Subject: [PATCH 071/194] chore(storage-scrubber): allow disable file logging (#8297) part of https://github.com/neondatabase/cloud/issues/14024, k8s does not always have a volume available for logging, and I'm running into weird permission errors... While I could spend time figuring out how to create temp directories for logging, I think it would be better to just disable file logging as k8s containers are ephemeral and we cannot retrieve anything on the fs after the container gets removed. ## Summary of changes `PAGESERVER_DISABLE_FILE_LOGGING=1` -> file logging disabled Signed-off-by: Alex Chi Z --- storage_scrubber/src/lib.rs | 40 ++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/storage_scrubber/src/lib.rs b/storage_scrubber/src/lib.rs index 6adaa5d38f6b..8f567b22e022 100644 --- a/storage_scrubber/src/lib.rs +++ b/storage_scrubber/src/lib.rs @@ -242,24 +242,36 @@ impl ConsoleConfig { } } -pub fn init_logging(file_name: &str) -> WorkerGuard { - let (file_writer, guard) = - tracing_appender::non_blocking(tracing_appender::rolling::never("./logs/", file_name)); - - let file_logs = fmt::Layer::new() - .with_target(false) - .with_ansi(false) - .with_writer(file_writer); +pub fn init_logging(file_name: &str) -> Option { let stderr_logs = fmt::Layer::new() .with_target(false) .with_writer(std::io::stderr); - tracing_subscriber::registry() - .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))) - .with(file_logs) - .with(stderr_logs) - .init(); - guard + let disable_file_logging = match std::env::var("PAGESERVER_DISABLE_FILE_LOGGING") { + Ok(s) => s == "1" || s.to_lowercase() == "true", + Err(_) => false, + }; + + if disable_file_logging { + tracing_subscriber::registry() + .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))) + .with(stderr_logs) + .init(); + None + } else { + let (file_writer, guard) = + tracing_appender::non_blocking(tracing_appender::rolling::never("./logs/", file_name)); + let file_logs = fmt::Layer::new() + .with_target(false) + .with_ansi(false) + .with_writer(file_writer); + tracing_subscriber::registry() + .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))) + .with(stderr_logs) + .with(file_logs) + .init(); + Some(guard) + } } pub fn init_s3_client(bucket_region: Region) -> Client { From 4a5b55c8346fc10ebbf7de3040d605c42dce31d3 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Tue, 9 Jul 2024 18:25:49 +0100 Subject: [PATCH 072/194] chore: fix nightly build (#8142) ## Problem `cargo +nightly check` fails ## Summary of changes Updates `measured`, `time`, and `crc32c`. * `measured`: updated to fix https://github.com/rust-lang/rust/issues/125763. * `time`: updated to fix https://github.com/rust-lang/rust/issues/125319 * `crc32c`: updated to remove some nightly feature detection with a removed nightly feature --- Cargo.lock | 65 ++++++++++++++++++++++++++++----------- Cargo.toml | 4 +-- libs/metrics/src/hll.rs | 14 ++++----- libs/metrics/src/lib.rs | 27 ++++------------ proxy/src/jemalloc.rs | 6 ++-- proxy/src/metrics.rs | 28 ++++++++++++++++- workspace_hack/Cargo.toml | 3 ++ 7 files changed, 94 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 716b6690d9f1..63628160d18f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1397,9 +1397,9 @@ dependencies = [ [[package]] name = "crc32c" -version = "0.6.5" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89254598aa9b9fa608de44b3ae54c810f0f06d755e24c50177f1f8f31ff50ce2" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" dependencies = [ "rustc_version", ] @@ -1651,6 +1651,16 @@ dependencies = [ "rusticata-macros", ] +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", + "serde", +] + [[package]] name = "desim" version = "0.1.0" @@ -3008,9 +3018,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "measured" -version = "0.0.21" +version = "0.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "652bc741286361c06de8cb4d89b21a6437f120c508c51713663589eeb9928ac5" +checksum = "3051f3a030d55d680cdef6ca50e80abd1182f8da29f2344a7c9cb575721138f0" dependencies = [ "bytes", "crossbeam-utils", @@ -3026,9 +3036,9 @@ dependencies = [ [[package]] name = "measured-derive" -version = "0.0.21" +version = "0.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea497f33e1e856a376c32ad916f69a0bd3c597db1f912a399f842b01a4a685d" +checksum = "b9e6777fc80a575f9503d908c8b498782a6c3ee88a06cb416dc3941401e43b94" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -3038,9 +3048,9 @@ dependencies = [ [[package]] name = "measured-process" -version = "0.0.21" +version = "0.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b364ccb66937a814b6b2ad751d1a2f7a9d5a78c761144036825fb36bb0771000" +checksum = "7c4b80445aeb08e832d87bf1830049a924cdc1d6b7ef40b6b9b365bff17bf8ec" dependencies = [ "libc", "measured", @@ -3275,6 +3285,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" version = "0.1.45" @@ -4118,6 +4134,12 @@ dependencies = [ "workspace_hack", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -5397,9 +5419,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.183" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" dependencies = [ "serde_derive", ] @@ -5416,9 +5438,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.183" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", @@ -6108,12 +6130,15 @@ dependencies = [ [[package]] name = "time" -version = "0.3.21" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3403384eaacbca9923fa06940178ac13e4edb725486d70e8e15881d0c836cc" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ + "deranged", "itoa", "js-sys", + "num-conv", + "powerfmt", "serde", "time-core", "time-macros", @@ -6121,16 +6146,17 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.9" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ + "num-conv", "time-core", ] @@ -7428,6 +7454,7 @@ dependencies = [ "clap", "clap_builder", "crossbeam-utils", + "deranged", "either", "fail", "futures-channel", @@ -7452,7 +7479,9 @@ dependencies = [ "num-traits", "once_cell", "parquet", + "proc-macro2", "prost", + "quote", "rand 0.8.5", "regex", "regex-automata 0.4.3", diff --git a/Cargo.toml b/Cargo.toml index 8fddaaef12dd..fc3dd5180922 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -111,8 +111,8 @@ lasso = "0.7" leaky-bucket = "1.0.1" libc = "0.2" md5 = "0.7.0" -measured = { version = "0.0.21", features=["lasso"] } -measured-process = { version = "0.0.21" } +measured = { version = "0.0.22", features=["lasso"] } +measured-process = { version = "0.0.22" } memoffset = "0.8" nix = { version = "0.27", features = ["fs", "process", "socket", "signal", "poll"] } notify = "6.0.0" diff --git a/libs/metrics/src/hll.rs b/libs/metrics/src/hll.rs index f53511ab5cc3..723916a7421a 100644 --- a/libs/metrics/src/hll.rs +++ b/libs/metrics/src/hll.rs @@ -13,11 +13,7 @@ use std::{ use measured::{ label::{LabelGroupVisitor, LabelName, LabelValue, LabelVisitor}, - metric::{ - group::{Encoding, MetricValue}, - name::MetricNameEncoder, - Metric, MetricType, MetricVec, - }, + metric::{counter::CounterState, name::MetricNameEncoder, Metric, MetricType, MetricVec}, text::TextEncoder, LabelGroup, }; @@ -144,6 +140,7 @@ impl HyperLogLogState { }) } } + impl measured::metric::MetricEncoding> for HyperLogLogState { @@ -182,12 +179,13 @@ impl measured::metric::MetricEncoding( labels: impl LabelGroup, name: impl MetricNameEncoder, enc: &mut Enc, -) -> Result<(), Enc::Err> { - enc.write_metric_value(name, labels, MetricValue::Int(x)) +) -> Result<(), Enc::Err> +where + GaugeState: MetricEncoding, +{ + GaugeState::new(x).collect_into(&(), labels, name, enc) } #[derive(Default)] @@ -544,15 +547,6 @@ impl Encoding for Inc { fn write_help(&mut self, name: impl MetricNameEncoder, help: &str) -> Result<(), Self::Err> { self.0.write_help(name, help) } - - fn write_metric_value( - &mut self, - name: impl MetricNameEncoder, - labels: impl LabelGroup, - value: MetricValue, - ) -> Result<(), Self::Err> { - self.0.write_metric_value(name, labels, value) - } } impl MetricEncoding> for MeasuredCounterPairState @@ -579,15 +573,6 @@ impl Encoding for Dec { fn write_help(&mut self, name: impl MetricNameEncoder, help: &str) -> Result<(), Self::Err> { self.0.write_help(name, help) } - - fn write_metric_value( - &mut self, - name: impl MetricNameEncoder, - labels: impl LabelGroup, - value: MetricValue, - ) -> Result<(), Self::Err> { - self.0.write_metric_value(name, labels, value) - } } /// Write the dec counter to the encoder diff --git a/proxy/src/jemalloc.rs b/proxy/src/jemalloc.rs index 3243e6a14010..d307d80f4af9 100644 --- a/proxy/src/jemalloc.rs +++ b/proxy/src/jemalloc.rs @@ -3,8 +3,8 @@ use std::marker::PhantomData; use measured::{ label::NoLabels, metric::{ - gauge::GaugeState, group::Encoding, group::MetricValue, name::MetricNameEncoder, - MetricEncoding, MetricFamilyEncoding, MetricType, + gauge::GaugeState, group::Encoding, name::MetricNameEncoder, MetricEncoding, + MetricFamilyEncoding, MetricType, }, text::TextEncoder, LabelGroup, MetricGroup, @@ -100,7 +100,7 @@ macro_rules! jemalloc_gauge { enc: &mut TextEncoder, ) -> Result<(), std::io::Error> { if let Ok(v) = mib.read() { - enc.write_metric_value(name, labels, MetricValue::Int(v as i64))?; + GaugeState::new(v as i64).collect_into(&(), labels, name, enc)?; } Ok(()) } diff --git a/proxy/src/metrics.rs b/proxy/src/metrics.rs index e2a75a872009..db25ac031115 100644 --- a/proxy/src/metrics.rs +++ b/proxy/src/metrics.rs @@ -2,7 +2,7 @@ use std::sync::{Arc, OnceLock}; use lasso::ThreadedRodeo; use measured::{ - label::{FixedCardinalitySet, LabelName, LabelSet, LabelValue, StaticLabelSet}, + label::{FixedCardinalitySet, LabelGroupSet, LabelName, LabelSet, LabelValue, StaticLabelSet}, metric::{histogram::Thresholds, name::MetricName}, Counter, CounterVec, FixedCardinalityLabel, Gauge, GaugeVec, Histogram, HistogramVec, LabelGroup, MetricGroup, @@ -577,6 +577,32 @@ impl LabelGroup for ThreadPoolWorkerId { } } +impl LabelGroupSet for ThreadPoolWorkers { + type Group<'a> = ThreadPoolWorkerId; + + fn cardinality(&self) -> Option { + Some(self.0) + } + + fn encode_dense(&self, value: Self::Unique) -> Option { + Some(value) + } + + fn decode_dense(&self, value: usize) -> Self::Group<'_> { + ThreadPoolWorkerId(value) + } + + type Unique = usize; + + fn encode(&self, value: Self::Group<'_>) -> Option { + Some(value.0) + } + + fn decode(&self, value: &Self::Unique) -> Self::Group<'_> { + ThreadPoolWorkerId(*value) + } +} + impl LabelSet for ThreadPoolWorkers { type Value<'a> = ThreadPoolWorkerId; diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index e1b1806bc877..7f5758599430 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -30,6 +30,7 @@ chrono = { version = "0.4", default-features = false, features = ["clock", "serd clap = { version = "4", features = ["derive", "string"] } clap_builder = { version = "4", default-features = false, features = ["color", "help", "std", "string", "suggestions", "usage"] } crossbeam-utils = { version = "0.8" } +deranged = { version = "0.3", default-features = false, features = ["powerfmt", "serde", "std"] } either = { version = "1" } fail = { version = "0.5", default-features = false, features = ["failpoints"] } futures-channel = { version = "0.3", features = ["sink"] } @@ -107,7 +108,9 @@ num-integer = { version = "0.1", features = ["i128"] } num-traits = { version = "0.2", features = ["i128", "libm"] } once_cell = { version = "1" } parquet = { git = "https://github.com/apache/arrow-rs", branch = "master", default-features = false, features = ["zstd"] } +proc-macro2 = { version = "1" } prost = { version = "0.11" } +quote = { version = "1" } regex = { version = "1" } regex-automata = { version = "0.4", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] } regex-syntax = { version = "0.8" } From b1fe8259b44ba0d0f0ce4d777edbc0e7e76ebd62 Mon Sep 17 00:00:00 2001 From: Alex Chi Z Date: Tue, 9 Jul 2024 13:41:37 -0400 Subject: [PATCH 073/194] fix(storage-scrubber): use default AWS authentication (#8299) part of https://github.com/neondatabase/cloud/issues/14024 close https://github.com/neondatabase/neon/issues/7665 Things running in k8s container use this authentication: https://docs.aws.amazon.com/sdkref/latest/guide/feature-container-credentials.html while we did not configure the client to use it. This pull request simply uses the default s3 client credential chain for storage scrubber. It might break compatibility with minio. ## Summary of changes * Use default AWS credential provider chain. * Improvements for s3 errors, we now have detailed errors and correct backtrace on last trial of the operation. --------- Signed-off-by: Alex Chi Z Co-authored-by: Joonas Koivunen --- storage_scrubber/src/find_large_objects.rs | 2 +- storage_scrubber/src/garbage.rs | 4 +- storage_scrubber/src/lib.rs | 89 +++++-------------- storage_scrubber/src/main.rs | 2 +- .../src/pageserver_physical_gc.rs | 2 +- .../src/scan_pageserver_metadata.rs | 2 +- .../src/scan_safekeeper_metadata.rs | 2 +- storage_scrubber/src/tenant_snapshot.rs | 7 +- 8 files changed, 33 insertions(+), 77 deletions(-) diff --git a/storage_scrubber/src/find_large_objects.rs b/storage_scrubber/src/find_large_objects.rs index 1422545f2fce..2ef802229d1d 100644 --- a/storage_scrubber/src/find_large_objects.rs +++ b/storage_scrubber/src/find_large_objects.rs @@ -47,7 +47,7 @@ pub async fn find_large_objects( ignore_deltas: bool, concurrency: usize, ) -> anyhow::Result { - let (s3_client, target) = init_remote(bucket_config.clone(), NodeKind::Pageserver)?; + let (s3_client, target) = init_remote(bucket_config.clone(), NodeKind::Pageserver).await?; let tenants = std::pin::pin!(stream_tenants(&s3_client, &target)); let objects_stream = tenants.map_ok(|tenant_shard_id| { diff --git a/storage_scrubber/src/garbage.rs b/storage_scrubber/src/garbage.rs index ce0ff10ec6e1..04508519881e 100644 --- a/storage_scrubber/src/garbage.rs +++ b/storage_scrubber/src/garbage.rs @@ -140,7 +140,7 @@ async fn find_garbage_inner( node_kind: NodeKind, ) -> anyhow::Result { // Construct clients for S3 and for Console API - let (s3_client, target) = init_remote(bucket_config.clone(), node_kind)?; + let (s3_client, target) = init_remote(bucket_config.clone(), node_kind).await?; let cloud_admin_api_client = Arc::new(CloudAdminApiClient::new(console_config)); // Build a set of console-known tenants, for quickly eliminating known-active tenants without having @@ -432,7 +432,7 @@ pub async fn purge_garbage( ); let (s3_client, target) = - init_remote(garbage_list.bucket_config.clone(), garbage_list.node_kind)?; + init_remote(garbage_list.bucket_config.clone(), garbage_list.node_kind).await?; // Sanity checks on the incoming list if garbage_list.active_tenant_count == 0 { diff --git a/storage_scrubber/src/lib.rs b/storage_scrubber/src/lib.rs index 8f567b22e022..9102ad9906f2 100644 --- a/storage_scrubber/src/lib.rs +++ b/storage_scrubber/src/lib.rs @@ -15,17 +15,10 @@ use std::fmt::Display; use std::sync::Arc; use std::time::Duration; -use anyhow::Context; -use aws_config::environment::EnvironmentVariableCredentialsProvider; -use aws_config::imds::credentials::ImdsCredentialsProvider; -use aws_config::meta::credentials::CredentialsProviderChain; -use aws_config::profile::ProfileFileCredentialsProvider; -use aws_config::retry::RetryConfig; -use aws_config::sso::SsoCredentialsProvider; -use aws_config::BehaviorVersion; -use aws_sdk_s3::config::{AsyncSleep, Region, SharedAsyncSleep}; -use aws_sdk_s3::{Client, Config}; -use aws_smithy_async::rt::sleep::TokioSleep; +use anyhow::{anyhow, Context}; +use aws_sdk_s3::config::Region; +use aws_sdk_s3::error::DisplayErrorContext; +use aws_sdk_s3::Client; use camino::{Utf8Path, Utf8PathBuf}; use clap::ValueEnum; @@ -274,65 +267,21 @@ pub fn init_logging(file_name: &str) -> Option { } } -pub fn init_s3_client(bucket_region: Region) -> Client { - let credentials_provider = { - // uses "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY" - let chain = CredentialsProviderChain::first_try( - "env", - EnvironmentVariableCredentialsProvider::new(), - ) - // uses "AWS_PROFILE" / `aws sso login --profile ` - .or_else( - "profile-sso", - ProfileFileCredentialsProvider::builder().build(), - ); - - // Use SSO if we were given an account ID - match std::env::var("SSO_ACCOUNT_ID").ok() { - Some(sso_account) => chain.or_else( - "sso", - SsoCredentialsProvider::builder() - .account_id(sso_account) - .role_name("PowerUserAccess") - .start_url("https://neondb.awsapps.com/start") - .region(bucket_region.clone()) - .build(), - ), - None => chain, - } - .or_else( - // Finally try IMDS - "imds", - ImdsCredentialsProvider::builder().build(), - ) - }; - - let sleep_impl: Arc = Arc::new(TokioSleep::new()); - - let mut builder = Config::builder() - .behavior_version( - #[allow(deprecated)] /* TODO: https://github.com/neondatabase/neon/issues/7665 */ - BehaviorVersion::v2023_11_09(), - ) +pub async fn init_s3_client(bucket_region: Region) -> Client { + let config = aws_config::defaults(aws_config::BehaviorVersion::v2024_03_28()) .region(bucket_region) - .retry_config(RetryConfig::adaptive().with_max_attempts(3)) - .sleep_impl(SharedAsyncSleep::from(sleep_impl)) - .credentials_provider(credentials_provider); - - if let Ok(endpoint) = env::var("AWS_ENDPOINT_URL") { - builder = builder.endpoint_url(endpoint) - } - - Client::from_conf(builder.build()) + .load() + .await; + Client::new(&config) } -fn init_remote( +async fn init_remote( bucket_config: BucketConfig, node_kind: NodeKind, ) -> anyhow::Result<(Arc, RootTarget)> { let bucket_region = Region::new(bucket_config.region); let delimiter = "/".to_string(); - let s3_client = Arc::new(init_s3_client(bucket_region)); + let s3_client = Arc::new(init_s3_client(bucket_region).await); let s3_root = match node_kind { NodeKind::Pageserver => RootTarget::Pageserver(S3Target { @@ -357,7 +306,7 @@ async fn list_objects_with_retries( s3_target: &S3Target, continuation_token: Option, ) -> anyhow::Result { - for _ in 0..MAX_RETRIES { + for trial in 0..MAX_RETRIES { match s3_client .list_objects_v2() .bucket(&s3_target.bucket_name) @@ -369,16 +318,22 @@ async fn list_objects_with_retries( { Ok(response) => return Ok(response), Err(e) => { + if trial == MAX_RETRIES - 1 { + return Err(e) + .with_context(|| format!("Failed to list objects {MAX_RETRIES} times")); + } error!( - "list_objects_v2 query failed: {e}, bucket_name={}, prefix={}, delimiter={}", - s3_target.bucket_name, s3_target.prefix_in_bucket, s3_target.delimiter + "list_objects_v2 query failed: bucket_name={}, prefix={}, delimiter={}, error={}", + s3_target.bucket_name, + s3_target.prefix_in_bucket, + s3_target.delimiter, + DisplayErrorContext(e), ); tokio::time::sleep(Duration::from_secs(1)).await; } } } - - anyhow::bail!("Failed to list objects {MAX_RETRIES} times") + Err(anyhow!("unreachable unless MAX_RETRIES==0")) } async fn download_object_with_retries( diff --git a/storage_scrubber/src/main.rs b/storage_scrubber/src/main.rs index 16a26613d25b..d81612119263 100644 --- a/storage_scrubber/src/main.rs +++ b/storage_scrubber/src/main.rs @@ -196,7 +196,7 @@ async fn main() -> anyhow::Result<()> { concurrency, } => { let downloader = - SnapshotDownloader::new(bucket_config, tenant_id, output_path, concurrency)?; + SnapshotDownloader::new(bucket_config, tenant_id, output_path, concurrency).await?; downloader.download().await } Command::PageserverPhysicalGc { diff --git a/storage_scrubber/src/pageserver_physical_gc.rs b/storage_scrubber/src/pageserver_physical_gc.rs index 014643312807..fb8fbc1635ae 100644 --- a/storage_scrubber/src/pageserver_physical_gc.rs +++ b/storage_scrubber/src/pageserver_physical_gc.rs @@ -160,7 +160,7 @@ pub async fn pageserver_physical_gc( min_age: Duration, mode: GcMode, ) -> anyhow::Result { - let (s3_client, target) = init_remote(bucket_config.clone(), NodeKind::Pageserver)?; + let (s3_client, target) = init_remote(bucket_config.clone(), NodeKind::Pageserver).await?; let tenants = if tenant_ids.is_empty() { futures::future::Either::Left(stream_tenants(&s3_client, &target)) diff --git a/storage_scrubber/src/scan_pageserver_metadata.rs b/storage_scrubber/src/scan_pageserver_metadata.rs index af74ffa4cdbd..df4f29acf72b 100644 --- a/storage_scrubber/src/scan_pageserver_metadata.rs +++ b/storage_scrubber/src/scan_pageserver_metadata.rs @@ -199,7 +199,7 @@ pub async fn scan_metadata( bucket_config: BucketConfig, tenant_ids: Vec, ) -> anyhow::Result { - let (s3_client, target) = init_remote(bucket_config, NodeKind::Pageserver)?; + let (s3_client, target) = init_remote(bucket_config, NodeKind::Pageserver).await?; let tenants = if tenant_ids.is_empty() { futures::future::Either::Left(stream_tenants(&s3_client, &target)) diff --git a/storage_scrubber/src/scan_safekeeper_metadata.rs b/storage_scrubber/src/scan_safekeeper_metadata.rs index 24051b03de08..553adf8f468e 100644 --- a/storage_scrubber/src/scan_safekeeper_metadata.rs +++ b/storage_scrubber/src/scan_safekeeper_metadata.rs @@ -106,7 +106,7 @@ pub async fn scan_safekeeper_metadata( let timelines = client.query(&query, &[]).await?; info!("loaded {} timelines", timelines.len()); - let (s3_client, target) = init_remote(bucket_config, NodeKind::Safekeeper)?; + let (s3_client, target) = init_remote(bucket_config, NodeKind::Safekeeper).await?; let console_config = ConsoleConfig::from_env()?; let cloud_admin_api_client = CloudAdminApiClient::new(console_config); diff --git a/storage_scrubber/src/tenant_snapshot.rs b/storage_scrubber/src/tenant_snapshot.rs index 450b337235f0..5a75f8d40ecf 100644 --- a/storage_scrubber/src/tenant_snapshot.rs +++ b/storage_scrubber/src/tenant_snapshot.rs @@ -28,13 +28,13 @@ pub struct SnapshotDownloader { } impl SnapshotDownloader { - pub fn new( + pub async fn new( bucket_config: BucketConfig, tenant_id: TenantId, output_path: Utf8PathBuf, concurrency: usize, ) -> anyhow::Result { - let (s3_client, s3_root) = init_remote(bucket_config.clone(), NodeKind::Pageserver)?; + let (s3_client, s3_root) = init_remote(bucket_config.clone(), NodeKind::Pageserver).await?; Ok(Self { s3_client, s3_root, @@ -215,7 +215,8 @@ impl SnapshotDownloader { } pub async fn download(&self) -> anyhow::Result<()> { - let (s3_client, target) = init_remote(self.bucket_config.clone(), NodeKind::Pageserver)?; + let (s3_client, target) = + init_remote(self.bucket_config.clone(), NodeKind::Pageserver).await?; // Generate a stream of TenantShardId let shards = stream_tenant_shards(&s3_client, &target, self.tenant_id).await?; From 6d3cb222ee340f11666031081d08965b19ccb317 Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Mon, 1 Jul 2024 13:45:42 -0500 Subject: [PATCH 074/194] Refactor how migrations are ran Just a small improvement I noticed while looking at fixing CVE-2024-4317 in Neon. --- compute_tools/src/lib.rs | 1 + compute_tools/src/migration.rs | 100 +++++++++++++++++++++++++++++++++ compute_tools/src/spec.rs | 65 +-------------------- 3 files changed, 103 insertions(+), 63 deletions(-) create mode 100644 compute_tools/src/migration.rs diff --git a/compute_tools/src/lib.rs b/compute_tools/src/lib.rs index 18c228ba5427..543d4462ed1c 100644 --- a/compute_tools/src/lib.rs +++ b/compute_tools/src/lib.rs @@ -11,6 +11,7 @@ pub mod logger; pub mod catalog; pub mod compute; pub mod extension_server; +mod migration; pub mod monitor; pub mod params; pub mod pg_helpers; diff --git a/compute_tools/src/migration.rs b/compute_tools/src/migration.rs new file mode 100644 index 000000000000..61dcf01c8448 --- /dev/null +++ b/compute_tools/src/migration.rs @@ -0,0 +1,100 @@ +use anyhow::{Context, Result}; +use postgres::Client; +use tracing::info; + +pub(crate) struct MigrationRunner<'m> { + client: &'m mut Client, + migrations: &'m [&'m str], +} + +impl<'m> MigrationRunner<'m> { + pub fn new(client: &'m mut Client, migrations: &'m [&'m str]) -> Self { + Self { client, migrations } + } + + fn get_migration_id(&mut self) -> Result { + let query = "SELECT id FROM neon_migration.migration_id"; + let row = self + .client + .query_one(query, &[]) + .context("run_migrations get migration_id")?; + + Ok(row.get::<&str, i64>("id")) + } + + fn update_migration_id(&mut self) -> Result<()> { + let setval = format!( + "UPDATE neon_migration.migration_id SET id={}", + self.migrations.len() + ); + + self.client + .simple_query(&setval) + .context("run_migrations update id")?; + + Ok(()) + } + + fn prepare_migrations(&mut self) -> Result<()> { + let query = "CREATE SCHEMA IF NOT EXISTS neon_migration"; + self.client.simple_query(query)?; + + let query = "CREATE TABLE IF NOT EXISTS neon_migration.migration_id (key INT NOT NULL PRIMARY KEY, id bigint NOT NULL DEFAULT 0)"; + self.client.simple_query(query)?; + + let query = "INSERT INTO neon_migration.migration_id VALUES (0, 0) ON CONFLICT DO NOTHING"; + self.client.simple_query(query)?; + + let query = "ALTER SCHEMA neon_migration OWNER TO cloud_admin"; + self.client.simple_query(query)?; + + let query = "REVOKE ALL ON SCHEMA neon_migration FROM PUBLIC"; + self.client.simple_query(query)?; + + Ok(()) + } + + pub fn run_migrations(mut self) -> Result<()> { + self.prepare_migrations()?; + + let mut current_migration: usize = self.get_migration_id()? as usize; + let starting_migration_id = current_migration; + + let query = "BEGIN"; + self.client + .simple_query(query) + .context("run_migrations begin")?; + + while current_migration < self.migrations.len() { + let migration = self.migrations[current_migration]; + + if migration.starts_with("-- SKIP") { + info!("Skipping migration id={}", current_migration); + } else { + info!( + "Running migration id={}:\n{}\n", + current_migration, migration + ); + self.client.simple_query(migration).with_context(|| { + format!("run_migration current_migration={}", current_migration) + })?; + } + + current_migration += 1; + } + + self.update_migration_id()?; + + let query = "COMMIT"; + self.client + .simple_query(query) + .context("run_migrations commit")?; + + info!( + "Ran {} migrations", + (self.migrations.len() - starting_migration_id) + ); + + Ok(()) + } +} diff --git a/compute_tools/src/spec.rs b/compute_tools/src/spec.rs index 143f6c1e5f6f..37090b08fd37 100644 --- a/compute_tools/src/spec.rs +++ b/compute_tools/src/spec.rs @@ -10,6 +10,7 @@ use tracing::{error, info, info_span, instrument, span_enabled, warn, Level}; use crate::config; use crate::logger::inlinify; +use crate::migration::MigrationRunner; use crate::params::PG_HBA_ALL_MD5; use crate::pg_helpers::*; @@ -791,69 +792,7 @@ pub fn handle_migrations(client: &mut Client) -> Result<()> { include_str!("./migrations/0008-revoke_replication_for_previously_allowed_roles.sql"), ]; - let mut func = || { - let query = "CREATE SCHEMA IF NOT EXISTS neon_migration"; - client.simple_query(query)?; - - let query = "CREATE TABLE IF NOT EXISTS neon_migration.migration_id (key INT NOT NULL PRIMARY KEY, id bigint NOT NULL DEFAULT 0)"; - client.simple_query(query)?; - - let query = "INSERT INTO neon_migration.migration_id VALUES (0, 0) ON CONFLICT DO NOTHING"; - client.simple_query(query)?; - - let query = "ALTER SCHEMA neon_migration OWNER TO cloud_admin"; - client.simple_query(query)?; - - let query = "REVOKE ALL ON SCHEMA neon_migration FROM PUBLIC"; - client.simple_query(query)?; - Ok::<_, anyhow::Error>(()) - }; - func().context("handle_migrations prepare")?; - - let query = "SELECT id FROM neon_migration.migration_id"; - let row = client - .query_one(query, &[]) - .context("handle_migrations get migration_id")?; - let mut current_migration: usize = row.get::<&str, i64>("id") as usize; - let starting_migration_id = current_migration; - - let query = "BEGIN"; - client - .simple_query(query) - .context("handle_migrations begin")?; - - while current_migration < migrations.len() { - let migration = &migrations[current_migration]; - if migration.starts_with("-- SKIP") { - info!("Skipping migration id={}", current_migration); - } else { - info!( - "Running migration id={}:\n{}\n", - current_migration, migration - ); - client.simple_query(migration).with_context(|| { - format!("handle_migrations current_migration={}", current_migration) - })?; - } - current_migration += 1; - } - let setval = format!( - "UPDATE neon_migration.migration_id SET id={}", - migrations.len() - ); - client - .simple_query(&setval) - .context("handle_migrations update id")?; - - let query = "COMMIT"; - client - .simple_query(query) - .context("handle_migrations commit")?; - - info!( - "Ran {} migrations", - (migrations.len() - starting_migration_id) - ); + MigrationRunner::new(client, &migrations).run_migrations()?; Ok(()) } From abc330e095687909c7daea515d27340b15be3810 Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Tue, 9 Jul 2024 10:21:23 -0500 Subject: [PATCH 075/194] Add an application_name to more Neon connections Helps identify connections in the logs. --- compute_tools/src/compute.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index 1fa2b9f71d64..eced6fc0b2e7 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -798,7 +798,11 @@ impl ComputeNode { // In this case we need to connect with old `zenith_admin` name // and create new user. We cannot simply rename connected user, // but we can create a new one and grant it all privileges. - let connstr = self.connstr.clone(); + let mut connstr = self.connstr.clone(); + connstr + .query_pairs_mut() + .append_pair("application_name", "apply_config"); + let mut client = match Client::connect(connstr.as_str(), NoTls) { Err(e) => match e.code() { Some(&SqlState::INVALID_PASSWORD) @@ -867,6 +871,11 @@ impl ComputeNode { // Run migrations separately to not hold up cold starts thread::spawn(move || { + let mut connstr = connstr.clone(); + connstr + .query_pairs_mut() + .append_pair("application_name", "migrations"); + let mut client = Client::connect(connstr.as_str(), NoTls)?; handle_migrations(&mut client).context("apply_config handle_migrations") }); From 3f7aebb01cd59f8c7ea9e7801832c7fb190a550c Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 9 Jul 2024 20:11:11 +0200 Subject: [PATCH 076/194] refactor: postgres_backend: replace abstract shutdown_watcher with CancellationToken (#8295) Preliminary refactoring while working on https://github.com/neondatabase/neon/issues/7427 and specifically https://github.com/neondatabase/neon/pull/8286 --- Cargo.lock | 3 +- libs/postgres_backend/Cargo.toml | 3 +- libs/postgres_backend/src/lib.rs | 33 +++++++------------- libs/postgres_backend/tests/simple_select.rs | 7 +++-- pageserver/src/page_service.rs | 2 +- proxy/src/console/mgmt.rs | 7 +++-- safekeeper/src/wal_service.rs | 5 +-- workspace_hack/Cargo.toml | 2 -- 8 files changed, 28 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 63628160d18f..776d95c3c745 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4094,6 +4094,7 @@ dependencies = [ "tokio-postgres", "tokio-postgres-rustls", "tokio-rustls 0.25.0", + "tokio-util", "tracing", "workspace_hack", ] @@ -7458,10 +7459,8 @@ dependencies = [ "either", "fail", "futures-channel", - "futures-core", "futures-executor", "futures-io", - "futures-sink", "futures-util", "getrandom 0.2.11", "hashbrown 0.14.5", diff --git a/libs/postgres_backend/Cargo.toml b/libs/postgres_backend/Cargo.toml index 8e249c09f7e0..c7611b9f213d 100644 --- a/libs/postgres_backend/Cargo.toml +++ b/libs/postgres_backend/Cargo.toml @@ -13,6 +13,7 @@ rustls.workspace = true serde.workspace = true thiserror.workspace = true tokio.workspace = true +tokio-util.workspace = true tokio-rustls.workspace = true tracing.workspace = true @@ -23,4 +24,4 @@ workspace_hack.workspace = true once_cell.workspace = true rustls-pemfile.workspace = true tokio-postgres.workspace = true -tokio-postgres-rustls.workspace = true \ No newline at end of file +tokio-postgres-rustls.workspace = true diff --git a/libs/postgres_backend/src/lib.rs b/libs/postgres_backend/src/lib.rs index 6c41b7f347a9..c79ee4e0533a 100644 --- a/libs/postgres_backend/src/lib.rs +++ b/libs/postgres_backend/src/lib.rs @@ -16,6 +16,7 @@ use std::{fmt, io}; use std::{future::Future, str::FromStr}; use tokio::io::{AsyncRead, AsyncWrite}; use tokio_rustls::TlsAcceptor; +use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, trace, warn}; use pq_proto::framed::{ConnectionError, Framed, FramedReader, FramedWriter}; @@ -400,21 +401,15 @@ impl PostgresBackend { } /// Wrapper for run_message_loop() that shuts down socket when we are done - pub async fn run( + pub async fn run( mut self, handler: &mut impl Handler, - shutdown_watcher: F, - ) -> Result<(), QueryError> - where - F: Fn() -> S + Clone, - S: Future, - { - let ret = self - .run_message_loop(handler, shutdown_watcher.clone()) - .await; + cancel: &CancellationToken, + ) -> Result<(), QueryError> { + let ret = self.run_message_loop(handler, cancel).await; tokio::select! { - _ = shutdown_watcher() => { + _ = cancel.cancelled() => { // do nothing; we most likely got already stopped by shutdown and will log it next. } _ = self.framed.shutdown() => { @@ -444,21 +439,17 @@ impl PostgresBackend { } } - async fn run_message_loop( + async fn run_message_loop( &mut self, handler: &mut impl Handler, - shutdown_watcher: F, - ) -> Result<(), QueryError> - where - F: Fn() -> S, - S: Future, - { + cancel: &CancellationToken, + ) -> Result<(), QueryError> { trace!("postgres backend to {:?} started", self.peer_addr); tokio::select!( biased; - _ = shutdown_watcher() => { + _ = cancel.cancelled() => { // We were requested to shut down. tracing::info!("shutdown request received during handshake"); return Err(QueryError::Shutdown) @@ -473,7 +464,7 @@ impl PostgresBackend { let mut query_string = Bytes::new(); while let Some(msg) = tokio::select!( biased; - _ = shutdown_watcher() => { + _ = cancel.cancelled() => { // We were requested to shut down. tracing::info!("shutdown request received in run_message_loop"); return Err(QueryError::Shutdown) @@ -485,7 +476,7 @@ impl PostgresBackend { let result = self.process_message(handler, msg, &mut query_string).await; tokio::select!( biased; - _ = shutdown_watcher() => { + _ = cancel.cancelled() => { // We were requested to shut down. tracing::info!("shutdown request received during response flush"); diff --git a/libs/postgres_backend/tests/simple_select.rs b/libs/postgres_backend/tests/simple_select.rs index 80df9db858a9..7ec85f0dbe90 100644 --- a/libs/postgres_backend/tests/simple_select.rs +++ b/libs/postgres_backend/tests/simple_select.rs @@ -3,13 +3,14 @@ use once_cell::sync::Lazy; use postgres_backend::{AuthType, Handler, PostgresBackend, QueryError}; use pq_proto::{BeMessage, RowDescriptor}; use std::io::Cursor; -use std::{future, sync::Arc}; +use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite}; use tokio::net::{TcpListener, TcpStream}; use tokio_postgres::config::SslMode; use tokio_postgres::tls::MakeTlsConnect; use tokio_postgres::{Config, NoTls, SimpleQueryMessage}; use tokio_postgres_rustls::MakeRustlsConnect; +use tokio_util::sync::CancellationToken; // generate client, server test streams async fn make_tcp_pair() -> (TcpStream, TcpStream) { @@ -50,7 +51,7 @@ async fn simple_select() { tokio::spawn(async move { let mut handler = TestHandler {}; - pgbackend.run(&mut handler, future::pending::<()>).await + pgbackend.run(&mut handler, &CancellationToken::new()).await }); let conf = Config::new(); @@ -102,7 +103,7 @@ async fn simple_select_ssl() { tokio::spawn(async move { let mut handler = TestHandler {}; - pgbackend.run(&mut handler, future::pending::<()>).await + pgbackend.run(&mut handler, &CancellationToken::new()).await }); let client_cfg = rustls::ClientConfig::builder() diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 07365b5eb85e..975c91297060 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -267,7 +267,7 @@ async fn page_service_conn_main( let pgbackend = PostgresBackend::new_from_io(socket, peer_addr, auth_type, None)?; match pgbackend - .run(&mut conn_handler, task_mgr::shutdown_watcher) + .run(&mut conn_handler, &task_mgr::shutdown_token()) .await { Ok(()) => { diff --git a/proxy/src/console/mgmt.rs b/proxy/src/console/mgmt.rs index c7a2d467c016..befe7d75104b 100644 --- a/proxy/src/console/mgmt.rs +++ b/proxy/src/console/mgmt.rs @@ -6,8 +6,9 @@ use anyhow::Context; use once_cell::sync::Lazy; use postgres_backend::{AuthType, PostgresBackend, PostgresBackendTCP, QueryError}; use pq_proto::{BeMessage, SINGLE_COL_ROWDESC}; -use std::{convert::Infallible, future}; +use std::convert::Infallible; use tokio::net::{TcpListener, TcpStream}; +use tokio_util::sync::CancellationToken; use tracing::{error, info, info_span, Instrument}; static CPLANE_WAITERS: Lazy> = Lazy::new(Default::default); @@ -67,7 +68,9 @@ pub async fn task_main(listener: TcpListener) -> anyhow::Result { async fn handle_connection(socket: TcpStream) -> Result<(), QueryError> { let pgbackend = PostgresBackend::new(socket, AuthType::Trust, None)?; - pgbackend.run(&mut MgmtHandler, future::pending::<()>).await + pgbackend + .run(&mut MgmtHandler, &CancellationToken::new()) + .await } /// A message received by `mgmt` when a compute node is ready. diff --git a/safekeeper/src/wal_service.rs b/safekeeper/src/wal_service.rs index 4a97eb3993f3..091571111e5c 100644 --- a/safekeeper/src/wal_service.rs +++ b/safekeeper/src/wal_service.rs @@ -4,9 +4,10 @@ //! use anyhow::{Context, Result}; use postgres_backend::QueryError; -use std::{future, time::Duration}; +use std::time::Duration; use tokio::net::TcpStream; use tokio_io_timeout::TimeoutReader; +use tokio_util::sync::CancellationToken; use tracing::*; use utils::{auth::Scope, measured_stream::MeasuredStream}; @@ -100,7 +101,7 @@ async fn handle_socket( // libpq protocol between safekeeper and walproposer / pageserver // We don't use shutdown. pgbackend - .run(&mut conn_handler, future::pending::<()>) + .run(&mut conn_handler, &CancellationToken::new()) .await } diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index 7f5758599430..832fe06bf697 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -34,10 +34,8 @@ deranged = { version = "0.3", default-features = false, features = ["powerfmt", either = { version = "1" } fail = { version = "0.5", default-features = false, features = ["failpoints"] } futures-channel = { version = "0.3", features = ["sink"] } -futures-core = { version = "0.3" } futures-executor = { version = "0.3" } futures-io = { version = "0.3" } -futures-sink = { version = "0.3" } futures-util = { version = "0.3", features = ["channel", "io", "sink"] } getrandom = { version = "0.2", default-features = false, features = ["std"] } hashbrown = { version = "0.14", features = ["raw"] } From 9bb16c8780da435e6de9fac08e11d4e0c2f5c682 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 9 Jul 2024 20:58:48 +0200 Subject: [PATCH 077/194] fix(l0_flush): drops permit before fsync, potential cause for OOMs (#8327) ## Problem Slack thread: https://neondb.slack.com/archives/C033RQ5SPDH/p1720511577862519 We're seeing OOMs in staging on a pageserver that has l0_flush.mode=Direct enabled. There's a strong correlation between jumps in `maxrss_kb` and `pageserver_timeline_ephemeral_bytes`, so, it's quite likely that l0_flush.mode=Direct is the culprit. Notably, the expected max memory usage on that staging server by the l0_flush.mode=Direct is ~2GiB but we're seeing as much as 24GiB max RSS before the OOM kill. One hypothesis is that we're dropping the semaphore permit before all the dirtied pages have been flushed to disk. (The flushing to disk likely happens in the fsync inside the `.finish()` call, because we're using ext4 in data=ordered mode). ## Summary of changes Hold the permit until after we're done with `.finish()`. --- .../src/tenant/storage_layer/inmemory_layer.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pageserver/src/tenant/storage_layer/inmemory_layer.rs b/pageserver/src/tenant/storage_layer/inmemory_layer.rs index e1eaea90af57..5941a52e9825 100644 --- a/pageserver/src/tenant/storage_layer/inmemory_layer.rs +++ b/pageserver/src/tenant/storage_layer/inmemory_layer.rs @@ -715,16 +715,22 @@ impl InMemoryLayer { res?; } } - - // Hold the permit until the IO is done; if we didn't, one could drop this future, - // thereby releasing the permit, but the Vec remains allocated until the IO completes. - // => we'd have more concurrenct Vec than allowed as per the semaphore. - drop(_concurrency_permit); } } // MAX is used here because we identify L0 layers by full key range let delta_layer = delta_layer_writer.finish(Key::MAX, timeline, ctx).await?; + + // Hold the permit until all the IO is done, including the fsync in `delta_layer_writer.finish()``. + // + // If we didn't and our caller drops this future, tokio-epoll-uring would extend the lifetime of + // the `file_contents: Vec` until the IO is done, but not the permit's lifetime. + // Thus, we'd have more concurrenct `Vec` in existence than the semaphore allows. + // + // We hold across the fsync so that on ext4 mounted with data=ordered, all the kernel page cache pages + // we dirtied when writing to the filesystem have been flushed and marked !dirty. + drop(_concurrency_permit); + Ok(Some(delta_layer)) } } From 1a49f1c15c7e728812d7a46191b5d3f194d98999 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 9 Jul 2024 23:17:42 +0200 Subject: [PATCH 078/194] pageserver: move `page_service`'s `import basebackup` / `import wal` to mgmt API (#8292) I want to fix bugs in `page_service` ([issue](https://github.com/neondatabase/neon/issues/7427)) and the `import basebackup` / `import wal` stand in the way / make the refactoring more complicated. We don't use these methods anyway in practice, but, there have been some objections to removing the functionality completely. So, this PR preserves the existing functionality but moves it into the HTTP management API. Note that I don't try to fix existing bugs in the code, specifically not fixing * it only ever worked correctly for unsharded tenants * it doesn't clean up on error All errors are mapped to `ApiError::InternalServerError`. --- control_plane/src/pageserver.rs | 58 ++--- libs/utils/src/http/request.rs | 9 + pageserver/client/Cargo.toml | 2 +- pageserver/client/src/mgmt_api.rs | 77 ++++++- pageserver/src/bin/pageserver.rs | 1 - pageserver/src/http/routes.rs | 194 ++++++++++++++++ pageserver/src/metrics.rs | 2 - pageserver/src/page_service.rs | 357 +---------------------------- storage_controller/src/node.rs | 2 +- storage_controller/src/service.rs | 4 + test_runner/regress/test_import.py | 3 +- 11 files changed, 301 insertions(+), 408 deletions(-) diff --git a/control_plane/src/pageserver.rs b/control_plane/src/pageserver.rs index 983f78577ce4..f0403b179622 100644 --- a/control_plane/src/pageserver.rs +++ b/control_plane/src/pageserver.rs @@ -15,7 +15,6 @@ use std::time::Duration; use anyhow::{bail, Context}; use camino::Utf8PathBuf; -use futures::SinkExt; use pageserver_api::models::{ self, AuxFilePolicy, LocationConfig, TenantHistorySize, TenantInfo, TimelineInfo, }; @@ -566,60 +565,39 @@ impl PageServerNode { pg_wal: Option<(Lsn, PathBuf)>, pg_version: u32, ) -> anyhow::Result<()> { - let (client, conn) = self.page_server_psql_client().await?; - // The connection object performs the actual communication with the database, - // so spawn it off to run on its own. - tokio::spawn(async move { - if let Err(e) = conn.await { - eprintln!("connection error: {}", e); - } - }); - let client = std::pin::pin!(client); - // Init base reader let (start_lsn, base_tarfile_path) = base; let base_tarfile = tokio::fs::File::open(base_tarfile_path).await?; - let base_tarfile = tokio_util::io::ReaderStream::new(base_tarfile); + let base_tarfile = + mgmt_api::ReqwestBody::wrap_stream(tokio_util::io::ReaderStream::new(base_tarfile)); // Init wal reader if necessary let (end_lsn, wal_reader) = if let Some((end_lsn, wal_tarfile_path)) = pg_wal { let wal_tarfile = tokio::fs::File::open(wal_tarfile_path).await?; - let wal_reader = tokio_util::io::ReaderStream::new(wal_tarfile); + let wal_reader = + mgmt_api::ReqwestBody::wrap_stream(tokio_util::io::ReaderStream::new(wal_tarfile)); (end_lsn, Some(wal_reader)) } else { (start_lsn, None) }; - let copy_in = |reader, cmd| { - let client = &client; - async move { - let writer = client.copy_in(&cmd).await?; - let writer = std::pin::pin!(writer); - let mut writer = writer.sink_map_err(|e| { - std::io::Error::new(std::io::ErrorKind::Other, format!("{e}")) - }); - let mut reader = std::pin::pin!(reader); - writer.send_all(&mut reader).await?; - writer.into_inner().finish().await?; - anyhow::Ok(()) - } - }; - // Import base - copy_in( - base_tarfile, - format!( - "import basebackup {tenant_id} {timeline_id} {start_lsn} {end_lsn} {pg_version}" - ), - ) - .await?; - // Import wal if necessary - if let Some(wal_reader) = wal_reader { - copy_in( - wal_reader, - format!("import wal {tenant_id} {timeline_id} {start_lsn} {end_lsn}"), + self.http_client + .import_basebackup( + tenant_id, + timeline_id, + start_lsn, + end_lsn, + pg_version, + base_tarfile, ) .await?; + + // Import wal if necessary + if let Some(wal_reader) = wal_reader { + self.http_client + .import_wal(tenant_id, timeline_id, start_lsn, end_lsn, wal_reader) + .await?; } Ok(()) diff --git a/libs/utils/src/http/request.rs b/libs/utils/src/http/request.rs index 766bbfc9dfae..8b8ed5a67f39 100644 --- a/libs/utils/src/http/request.rs +++ b/libs/utils/src/http/request.rs @@ -74,6 +74,15 @@ pub fn parse_query_param>( .transpose() } +pub fn must_parse_query_param>( + request: &Request, + param_name: &str, +) -> Result { + parse_query_param(request, param_name)?.ok_or_else(|| { + ApiError::BadRequest(anyhow!("no {param_name} specified in query parameters")) + }) +} + pub async fn ensure_no_body(request: &mut Request) -> Result<(), ApiError> { match request.body_mut().data().await { Some(_) => Err(ApiError::BadRequest(anyhow!("Unexpected request body"))), diff --git a/pageserver/client/Cargo.toml b/pageserver/client/Cargo.toml index 0ed27602cd3c..a938367334fa 100644 --- a/pageserver/client/Cargo.toml +++ b/pageserver/client/Cargo.toml @@ -8,7 +8,7 @@ license.workspace = true pageserver_api.workspace = true thiserror.workspace = true async-trait.workspace = true -reqwest.workspace = true +reqwest = { workspace = true, features = [ "stream" ] } utils.workspace = true serde.workspace = true workspace_hack = { version = "0.1", path = "../../workspace_hack" } diff --git a/pageserver/client/src/mgmt_api.rs b/pageserver/client/src/mgmt_api.rs index 48b27775cb91..e3ddb446fa2c 100644 --- a/pageserver/client/src/mgmt_api.rs +++ b/pageserver/client/src/mgmt_api.rs @@ -9,6 +9,8 @@ use utils::{ lsn::Lsn, }; +pub use reqwest::Body as ReqwestBody; + pub mod util; #[derive(Debug, Clone)] @@ -20,6 +22,9 @@ pub struct Client { #[derive(thiserror::Error, Debug)] pub enum Error { + #[error("send request: {0}")] + SendRequest(reqwest::Error), + #[error("receive body: {0}")] ReceiveBody(reqwest::Error), @@ -173,19 +178,30 @@ impl Client { self.request(Method::GET, uri, ()).await } - async fn request_noerror( + fn start_request( &self, method: Method, uri: U, - body: B, - ) -> Result { + ) -> reqwest::RequestBuilder { let req = self.client.request(method, uri); - let req = if let Some(value) = &self.authorization_header { + if let Some(value) = &self.authorization_header { req.header(reqwest::header::AUTHORIZATION, value) } else { req - }; - req.json(&body).send().await.map_err(Error::ReceiveBody) + } + } + + async fn request_noerror( + &self, + method: Method, + uri: U, + body: B, + ) -> Result { + self.start_request(method, uri) + .json(&body) + .send() + .await + .map_err(Error::ReceiveBody) } async fn request( @@ -609,4 +625,53 @@ impl Client { }), } } + + pub async fn import_basebackup( + &self, + tenant_id: TenantId, + timeline_id: TimelineId, + base_lsn: Lsn, + end_lsn: Lsn, + pg_version: u32, + basebackup_tarball: ReqwestBody, + ) -> Result<()> { + let uri = format!( + "{}/v1/tenant/{tenant_id}/timeline/{timeline_id}/import_basebackup?base_lsn={base_lsn}&end_lsn={end_lsn}&pg_version={pg_version}", + self.mgmt_api_endpoint, + ); + self.start_request(Method::PUT, uri) + .body(basebackup_tarball) + .send() + .await + .map_err(Error::SendRequest)? + .error_from_body() + .await? + .json() + .await + .map_err(Error::ReceiveBody) + } + + pub async fn import_wal( + &self, + tenant_id: TenantId, + timeline_id: TimelineId, + start_lsn: Lsn, + end_lsn: Lsn, + wal_tarball: ReqwestBody, + ) -> Result<()> { + let uri = format!( + "{}/v1/tenant/{tenant_id}/timeline/{timeline_id}/import_wal?start_lsn={start_lsn}&end_lsn={end_lsn}", + self.mgmt_api_endpoint, + ); + self.start_request(Method::PUT, uri) + .body(wal_tarball) + .send() + .await + .map_err(Error::SendRequest)? + .error_from_body() + .await? + .json() + .await + .map_err(Error::ReceiveBody) + } } diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 2763352a213f..9f705f0bc923 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -660,7 +660,6 @@ fn start_pageserver( async move { page_service::libpq_listener_main( tenant_manager, - broker_client, pg_auth, pageserver_listener, conf.pg_auth_type, diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 893302b7d6d9..6f8f3e6389d5 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -10,6 +10,7 @@ use std::time::Duration; use anyhow::{anyhow, Context, Result}; use enumset::EnumSet; +use futures::StreamExt; use futures::TryFutureExt; use humantime::format_rfc3339; use hyper::header; @@ -44,12 +45,14 @@ use remote_storage::DownloadError; use remote_storage::GenericRemoteStorage; use remote_storage::TimeTravelError; use tenant_size_model::{svg::SvgBranchKind, SizeResult, StorageModel}; +use tokio_util::io::StreamReader; use tokio_util::sync::CancellationToken; use tracing::*; use utils::auth::JwtAuth; use utils::failpoint_support::failpoints_handler; use utils::http::endpoint::prometheus_metrics_handler; use utils::http::endpoint::request_span; +use utils::http::request::must_parse_query_param; use utils::http::request::{get_request_param, must_get_query_param, parse_query_param}; use crate::context::{DownloadBehavior, RequestContext}; @@ -2404,6 +2407,189 @@ async fn post_top_tenants( ) } +async fn put_tenant_timeline_import_basebackup( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { + let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; + let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; + let base_lsn: Lsn = must_parse_query_param(&request, "base_lsn")?; + let end_lsn: Lsn = must_parse_query_param(&request, "end_lsn")?; + let pg_version: u32 = must_parse_query_param(&request, "pg_version")?; + + check_permission(&request, Some(tenant_id))?; + + let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn); + + let span = info_span!("import_basebackup", tenant_id=%tenant_id, timeline_id=%timeline_id, base_lsn=%base_lsn, end_lsn=%end_lsn, pg_version=%pg_version); + async move { + let state = get_state(&request); + let tenant = state + .tenant_manager + .get_attached_tenant_shard(TenantShardId::unsharded(tenant_id))?; + + let broker_client = state.broker_client.clone(); + + let mut body = StreamReader::new(request.into_body().map(|res| { + res.map_err(|error| { + std::io::Error::new(std::io::ErrorKind::Other, anyhow::anyhow!(error)) + }) + })); + + tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?; + + let timeline = tenant + .create_empty_timeline(timeline_id, base_lsn, pg_version, &ctx) + .map_err(ApiError::InternalServerError) + .await?; + + // TODO mark timeline as not ready until it reaches end_lsn. + // We might have some wal to import as well, and we should prevent compute + // from connecting before that and writing conflicting wal. + // + // This is not relevant for pageserver->pageserver migrations, since there's + // no wal to import. But should be fixed if we want to import from postgres. + + // TODO leave clean state on error. For now you can use detach to clean + // up broken state from a failed import. + + // Import basebackup provided via CopyData + info!("importing basebackup"); + + timeline + .import_basebackup_from_tar(tenant.clone(), &mut body, base_lsn, broker_client, &ctx) + .await + .map_err(ApiError::InternalServerError)?; + + // Read the end of the tar archive. + read_tar_eof(body) + .await + .map_err(ApiError::InternalServerError)?; + + // TODO check checksum + // Meanwhile you can verify client-side by taking fullbackup + // and checking that it matches in size with what was imported. + // It wouldn't work if base came from vanilla postgres though, + // since we discard some log files. + + info!("done"); + json_response(StatusCode::OK, ()) + } + .instrument(span) + .await +} + +async fn put_tenant_timeline_import_wal( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { + let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; + let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; + let start_lsn: Lsn = must_parse_query_param(&request, "start_lsn")?; + let end_lsn: Lsn = must_parse_query_param(&request, "end_lsn")?; + + check_permission(&request, Some(tenant_id))?; + + let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn); + + let span = info_span!("import_wal", tenant_id=%tenant_id, timeline_id=%timeline_id, start_lsn=%start_lsn, end_lsn=%end_lsn); + async move { + let state = get_state(&request); + + let timeline = active_timeline_of_active_tenant(&state.tenant_manager, TenantShardId::unsharded(tenant_id), timeline_id).await?; + + let mut body = StreamReader::new(request.into_body().map(|res| { + res.map_err(|error| { + std::io::Error::new(std::io::ErrorKind::Other, anyhow::anyhow!(error)) + }) + })); + + let last_record_lsn = timeline.get_last_record_lsn(); + if last_record_lsn != start_lsn { + return Err(ApiError::InternalServerError(anyhow::anyhow!("Cannot import WAL from Lsn {start_lsn} because timeline does not start from the same lsn: {last_record_lsn}"))); + } + + // TODO leave clean state on error. For now you can use detach to clean + // up broken state from a failed import. + + // Import wal provided via CopyData + info!("importing wal"); + crate::import_datadir::import_wal_from_tar(&timeline, &mut body, start_lsn, end_lsn, &ctx).await.map_err(ApiError::InternalServerError)?; + info!("wal import complete"); + + // Read the end of the tar archive. + read_tar_eof(body).await.map_err(ApiError::InternalServerError)?; + + // TODO Does it make sense to overshoot? + if timeline.get_last_record_lsn() < end_lsn { + return Err(ApiError::InternalServerError(anyhow::anyhow!("Cannot import WAL from Lsn {start_lsn} because timeline does not start from the same lsn: {last_record_lsn}"))); + } + + // Flush data to disk, then upload to s3. No need for a forced checkpoint. + // We only want to persist the data, and it doesn't matter if it's in the + // shape of deltas or images. + info!("flushing layers"); + timeline.freeze_and_flush().await.map_err(|e| match e { + tenant::timeline::FlushLayerError::Cancelled => ApiError::ShuttingDown, + other => ApiError::InternalServerError(anyhow::anyhow!(other)), + })?; + + info!("done"); + + json_response(StatusCode::OK, ()) + }.instrument(span).await +} + +/// Read the end of a tar archive. +/// +/// A tar archive normally ends with two consecutive blocks of zeros, 512 bytes each. +/// `tokio_tar` already read the first such block. Read the second all-zeros block, +/// and check that there is no more data after the EOF marker. +/// +/// 'tar' command can also write extra blocks of zeros, up to a record +/// size, controlled by the --record-size argument. Ignore them too. +async fn read_tar_eof(mut reader: (impl tokio::io::AsyncRead + Unpin)) -> anyhow::Result<()> { + use tokio::io::AsyncReadExt; + let mut buf = [0u8; 512]; + + // Read the all-zeros block, and verify it + let mut total_bytes = 0; + while total_bytes < 512 { + let nbytes = reader.read(&mut buf[total_bytes..]).await?; + total_bytes += nbytes; + if nbytes == 0 { + break; + } + } + if total_bytes < 512 { + anyhow::bail!("incomplete or invalid tar EOF marker"); + } + if !buf.iter().all(|&x| x == 0) { + anyhow::bail!("invalid tar EOF marker"); + } + + // Drain any extra zero-blocks after the EOF marker + let mut trailing_bytes = 0; + let mut seen_nonzero_bytes = false; + loop { + let nbytes = reader.read(&mut buf).await?; + trailing_bytes += nbytes; + if !buf.iter().all(|&x| x == 0) { + seen_nonzero_bytes = true; + } + if nbytes == 0 { + break; + } + } + if seen_nonzero_bytes { + anyhow::bail!("unexpected non-zero bytes after the tar archive"); + } + if trailing_bytes % 512 != 0 { + anyhow::bail!("unexpected number of zeros ({trailing_bytes}), not divisible by tar block size (512 bytes), after the tar archive"); + } + Ok(()) +} + /// Common functionality of all the HTTP API handlers. /// /// - Adds a tracing span to each request (by `request_span`) @@ -2698,5 +2884,13 @@ pub fn make_router( "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/perf_info", |r| testing_api_handler("perf_info", r, perf_info), ) + .put( + "/v1/tenant/:tenant_id/timeline/:timeline_id/import_basebackup", + |r| api_handler(r, put_tenant_timeline_import_basebackup), + ) + .put( + "/v1/tenant/:tenant_id/timeline/:timeline_id/import_wal", + |r| api_handler(r, put_tenant_timeline_import_wal), + ) .any(handler_404)) } diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 59b729363147..e67fa656d02e 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -1473,8 +1473,6 @@ pub(crate) enum ComputeCommandKind { PageStream, Basebackup, Fullbackup, - ImportBasebackup, - ImportWal, LeaseLsn, Show, } diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 975c91297060..c10c2f2a0f9a 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -4,9 +4,7 @@ use anyhow::Context; use async_compression::tokio::write::GzipEncoder; use bytes::Buf; -use bytes::Bytes; use futures::stream::FuturesUnordered; -use futures::Stream; use futures::StreamExt; use pageserver_api::key::Key; use pageserver_api::models::TenantState; @@ -28,7 +26,6 @@ use std::borrow::Cow; use std::collections::HashMap; use std::io; use std::net::TcpListener; -use std::pin::pin; use std::str; use std::str::FromStr; use std::sync::Arc; @@ -37,7 +34,6 @@ use std::time::Instant; use std::time::SystemTime; use tokio::io::AsyncWriteExt; use tokio::io::{AsyncRead, AsyncWrite}; -use tokio_util::io::StreamReader; use tokio_util::sync::CancellationToken; use tracing::*; use utils::id::ConnectionId; @@ -53,7 +49,6 @@ use crate::auth::check_permission; use crate::basebackup; use crate::basebackup::BasebackupError; use crate::context::{DownloadBehavior, RequestContext}; -use crate::import_datadir::import_wal_from_tar; use crate::metrics; use crate::metrics::{ComputeCommandKind, COMPUTE_COMMANDS_COUNTERS, LIVE_CONNECTIONS}; use crate::pgdatadir_mapping::Version; @@ -66,7 +61,6 @@ use crate::tenant::mgr::GetTenantError; use crate::tenant::mgr::ShardResolveResult; use crate::tenant::mgr::ShardSelector; use crate::tenant::mgr::TenantManager; -use crate::tenant::timeline::FlushLayerError; use crate::tenant::timeline::WaitLsnError; use crate::tenant::GetTimelineError; use crate::tenant::PageReconstructError; @@ -82,56 +76,6 @@ use postgres_ffi::BLCKSZ; // is not yet in state [`TenantState::Active`]. const ACTIVE_TENANT_TIMEOUT: Duration = Duration::from_millis(30000); -/// Read the end of a tar archive. -/// -/// A tar archive normally ends with two consecutive blocks of zeros, 512 bytes each. -/// `tokio_tar` already read the first such block. Read the second all-zeros block, -/// and check that there is no more data after the EOF marker. -/// -/// 'tar' command can also write extra blocks of zeros, up to a record -/// size, controlled by the --record-size argument. Ignore them too. -async fn read_tar_eof(mut reader: (impl AsyncRead + Unpin)) -> anyhow::Result<()> { - use tokio::io::AsyncReadExt; - let mut buf = [0u8; 512]; - - // Read the all-zeros block, and verify it - let mut total_bytes = 0; - while total_bytes < 512 { - let nbytes = reader.read(&mut buf[total_bytes..]).await?; - total_bytes += nbytes; - if nbytes == 0 { - break; - } - } - if total_bytes < 512 { - anyhow::bail!("incomplete or invalid tar EOF marker"); - } - if !buf.iter().all(|&x| x == 0) { - anyhow::bail!("invalid tar EOF marker"); - } - - // Drain any extra zero-blocks after the EOF marker - let mut trailing_bytes = 0; - let mut seen_nonzero_bytes = false; - loop { - let nbytes = reader.read(&mut buf).await?; - trailing_bytes += nbytes; - if !buf.iter().all(|&x| x == 0) { - seen_nonzero_bytes = true; - } - if nbytes == 0 { - break; - } - } - if seen_nonzero_bytes { - anyhow::bail!("unexpected non-zero bytes after the tar archive"); - } - if trailing_bytes % 512 != 0 { - anyhow::bail!("unexpected number of zeros ({trailing_bytes}), not divisible by tar block size (512 bytes), after the tar archive"); - } - Ok(()) -} - /////////////////////////////////////////////////////////////////////////////// /// @@ -141,7 +85,6 @@ async fn read_tar_eof(mut reader: (impl AsyncRead + Unpin)) -> anyhow::Result<() /// pub async fn libpq_listener_main( tenant_manager: Arc, - broker_client: storage_broker::BrokerClientChannel, auth: Option>, listener: TcpListener, auth_type: AuthType, @@ -186,7 +129,6 @@ pub async fn libpq_listener_main( false, page_service_conn_main( tenant_manager.clone(), - broker_client.clone(), local_auth, socket, auth_type, @@ -209,7 +151,6 @@ pub async fn libpq_listener_main( #[instrument(skip_all, fields(peer_addr))] async fn page_service_conn_main( tenant_manager: Arc, - broker_client: storage_broker::BrokerClientChannel, auth: Option>, socket: tokio::net::TcpStream, auth_type: AuthType, @@ -262,8 +203,7 @@ async fn page_service_conn_main( // and create a child per-query context when it invokes process_query. // But it's in a shared crate, so, we store connection_ctx inside PageServerHandler // and create the per-query context in process_query ourselves. - let mut conn_handler = - PageServerHandler::new(tenant_manager, broker_client, auth, connection_ctx); + let mut conn_handler = PageServerHandler::new(tenant_manager, auth, connection_ctx); let pgbackend = PostgresBackend::new_from_io(socket, peer_addr, auth_type, None)?; match pgbackend @@ -294,7 +234,6 @@ struct HandlerTimeline { } struct PageServerHandler { - broker_client: storage_broker::BrokerClientChannel, auth: Option>, claims: Option, @@ -386,13 +325,11 @@ impl From for QueryError { impl PageServerHandler { pub fn new( tenant_manager: Arc, - broker_client: storage_broker::BrokerClientChannel, auth: Option>, connection_ctx: RequestContext, ) -> Self { PageServerHandler { tenant_manager, - broker_client, auth, claims: None, connection_ctx, @@ -475,73 +412,6 @@ impl PageServerHandler { ) } - fn copyin_stream<'a, IO>( - &'a self, - pgb: &'a mut PostgresBackend, - cancel: &'a CancellationToken, - ) -> impl Stream> + 'a - where - IO: AsyncRead + AsyncWrite + Send + Sync + Unpin, - { - async_stream::try_stream! { - loop { - let msg = tokio::select! { - biased; - - _ = cancel.cancelled() => { - // We were requested to shut down. - let msg = "pageserver is shutting down"; - let _ = pgb.write_message_noflush(&BeMessage::ErrorResponse(msg, None)); - Err(QueryError::Shutdown) - } - - msg = pgb.read_message() => { msg.map_err(QueryError::from)} - }; - - match msg { - Ok(Some(message)) => { - let copy_data_bytes = match message { - FeMessage::CopyData(bytes) => bytes, - FeMessage::CopyDone => { break }, - FeMessage::Sync => continue, - FeMessage::Terminate => { - let msg = "client terminated connection with Terminate message during COPY"; - let query_error = QueryError::Disconnected(ConnectionError::Io(io::Error::new(io::ErrorKind::ConnectionReset, msg))); - // error can't happen here, ErrorResponse serialization should be always ok - pgb.write_message_noflush(&BeMessage::ErrorResponse(msg, Some(query_error.pg_error_code()))).map_err(|e| e.into_io_error())?; - Err(io::Error::new(io::ErrorKind::ConnectionReset, msg))?; - break; - } - m => { - let msg = format!("unexpected message {m:?}"); - // error can't happen here, ErrorResponse serialization should be always ok - pgb.write_message_noflush(&BeMessage::ErrorResponse(&msg, None)).map_err(|e| e.into_io_error())?; - Err(io::Error::new(io::ErrorKind::Other, msg))?; - break; - } - }; - - yield copy_data_bytes; - } - Ok(None) => { - let msg = "client closed connection during COPY"; - let query_error = QueryError::Disconnected(ConnectionError::Io(io::Error::new(io::ErrorKind::ConnectionReset, msg))); - // error can't happen here, ErrorResponse serialization should be always ok - pgb.write_message_noflush(&BeMessage::ErrorResponse(msg, Some(query_error.pg_error_code()))).map_err(|e| e.into_io_error())?; - self.flush_cancellable(pgb, cancel).await.map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?; - Err(io::Error::new(io::ErrorKind::ConnectionReset, msg))?; - } - Err(QueryError::Disconnected(ConnectionError::Io(io_error))) => { - Err(io_error)?; - } - Err(other) => { - Err(io::Error::new(io::ErrorKind::Other, other.to_string()))?; - } - }; - } - } - } - #[instrument(skip_all)] async fn handle_pagerequests( &mut self, @@ -713,128 +583,6 @@ impl PageServerHandler { Ok(()) } - #[allow(clippy::too_many_arguments)] - #[instrument(skip_all, fields(%base_lsn, end_lsn=%_end_lsn, %pg_version))] - async fn handle_import_basebackup( - &self, - pgb: &mut PostgresBackend, - tenant_id: TenantId, - timeline_id: TimelineId, - base_lsn: Lsn, - _end_lsn: Lsn, - pg_version: u32, - ctx: RequestContext, - ) -> Result<(), QueryError> - where - IO: AsyncRead + AsyncWrite + Send + Sync + Unpin, - { - debug_assert_current_span_has_tenant_and_timeline_id_no_shard_id(); - - // Create empty timeline - info!("creating new timeline"); - let tenant = self - .get_active_tenant_with_timeout(tenant_id, ShardSelector::Zero, ACTIVE_TENANT_TIMEOUT) - .await?; - let timeline = tenant - .create_empty_timeline(timeline_id, base_lsn, pg_version, &ctx) - .await?; - - // TODO mark timeline as not ready until it reaches end_lsn. - // We might have some wal to import as well, and we should prevent compute - // from connecting before that and writing conflicting wal. - // - // This is not relevant for pageserver->pageserver migrations, since there's - // no wal to import. But should be fixed if we want to import from postgres. - - // TODO leave clean state on error. For now you can use detach to clean - // up broken state from a failed import. - - // Import basebackup provided via CopyData - info!("importing basebackup"); - pgb.write_message_noflush(&BeMessage::CopyInResponse)?; - self.flush_cancellable(pgb, &tenant.cancel).await?; - - let mut copyin_reader = pin!(StreamReader::new(self.copyin_stream(pgb, &tenant.cancel))); - timeline - .import_basebackup_from_tar( - tenant.clone(), - &mut copyin_reader, - base_lsn, - self.broker_client.clone(), - &ctx, - ) - .await?; - - // Read the end of the tar archive. - read_tar_eof(copyin_reader).await?; - - // TODO check checksum - // Meanwhile you can verify client-side by taking fullbackup - // and checking that it matches in size with what was imported. - // It wouldn't work if base came from vanilla postgres though, - // since we discard some log files. - - info!("done"); - Ok(()) - } - - #[instrument(skip_all, fields(shard_id, %start_lsn, %end_lsn))] - async fn handle_import_wal( - &self, - pgb: &mut PostgresBackend, - tenant_id: TenantId, - timeline_id: TimelineId, - start_lsn: Lsn, - end_lsn: Lsn, - ctx: RequestContext, - ) -> Result<(), QueryError> - where - IO: AsyncRead + AsyncWrite + Send + Sync + Unpin, - { - let timeline = self - .get_active_tenant_timeline(tenant_id, timeline_id, ShardSelector::Zero) - .await?; - let last_record_lsn = timeline.get_last_record_lsn(); - if last_record_lsn != start_lsn { - return Err(QueryError::Other( - anyhow::anyhow!("Cannot import WAL from Lsn {start_lsn} because timeline does not start from the same lsn: {last_record_lsn}")) - ); - } - - // TODO leave clean state on error. For now you can use detach to clean - // up broken state from a failed import. - - // Import wal provided via CopyData - info!("importing wal"); - pgb.write_message_noflush(&BeMessage::CopyInResponse)?; - self.flush_cancellable(pgb, &timeline.cancel).await?; - let mut copyin_reader = pin!(StreamReader::new(self.copyin_stream(pgb, &timeline.cancel))); - import_wal_from_tar(&timeline, &mut copyin_reader, start_lsn, end_lsn, &ctx).await?; - info!("wal import complete"); - - // Read the end of the tar archive. - read_tar_eof(copyin_reader).await?; - - // TODO Does it make sense to overshoot? - if timeline.get_last_record_lsn() < end_lsn { - return Err(QueryError::Other( - anyhow::anyhow!("Cannot import WAL from Lsn {start_lsn} because timeline does not start from the same lsn: {last_record_lsn}")) - ); - } - - // Flush data to disk, then upload to s3. No need for a forced checkpoint. - // We only want to persist the data, and it doesn't matter if it's in the - // shape of deltas or images. - info!("flushing layers"); - timeline.freeze_and_flush().await.map_err(|e| match e { - FlushLayerError::Cancelled => QueryError::Shutdown, - other => QueryError::Other(other.into()), - })?; - - info!("done"); - Ok(()) - } - /// Helper function to handle the LSN from client request. /// /// Each GetPage (and Exists and Nblocks) request includes information about @@ -1705,109 +1453,6 @@ where ) .await?; pgb.write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?; - } else if query_string.starts_with("import basebackup ") { - // Import the `base` section (everything but the wal) of a basebackup. - // Assumes the tenant already exists on this pageserver. - // - // Files are scheduled to be persisted to remote storage, and the - // caller should poll the http api to check when that is done. - // - // Example import command: - // 1. Get start/end LSN from backup_manifest file - // 2. Run: - // cat my_backup/base.tar | psql -h $PAGESERVER \ - // -c "import basebackup $TENANT $TIMELINE $START_LSN $END_LSN $PG_VERSION" - let params = &parts[2..]; - if params.len() != 5 { - return Err(QueryError::Other(anyhow::anyhow!( - "invalid param number for import basebackup command" - ))); - } - let tenant_id = TenantId::from_str(params[0]) - .with_context(|| format!("Failed to parse tenant id from {}", params[0]))?; - let timeline_id = TimelineId::from_str(params[1]) - .with_context(|| format!("Failed to parse timeline id from {}", params[1]))?; - let base_lsn = Lsn::from_str(params[2]) - .with_context(|| format!("Failed to parse Lsn from {}", params[2]))?; - let end_lsn = Lsn::from_str(params[3]) - .with_context(|| format!("Failed to parse Lsn from {}", params[3]))?; - let pg_version = u32::from_str(params[4]) - .with_context(|| format!("Failed to parse pg_version from {}", params[4]))?; - - tracing::Span::current() - .record("tenant_id", field::display(tenant_id)) - .record("timeline_id", field::display(timeline_id)); - - self.check_permission(Some(tenant_id))?; - - COMPUTE_COMMANDS_COUNTERS - .for_command(ComputeCommandKind::ImportBasebackup) - .inc(); - - match self - .handle_import_basebackup( - pgb, - tenant_id, - timeline_id, - base_lsn, - end_lsn, - pg_version, - ctx, - ) - .await - { - Ok(()) => pgb.write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?, - Err(e) => { - error!("error importing base backup between {base_lsn} and {end_lsn}: {e:?}"); - pgb.write_message_noflush(&BeMessage::ErrorResponse( - &e.to_string(), - Some(e.pg_error_code()), - ))? - } - }; - } else if query_string.starts_with("import wal ") { - // Import the `pg_wal` section of a basebackup. - // - // Files are scheduled to be persisted to remote storage, and the - // caller should poll the http api to check when that is done. - let params = &parts[2..]; - if params.len() != 4 { - return Err(QueryError::Other(anyhow::anyhow!( - "invalid param number for import wal command" - ))); - } - let tenant_id = TenantId::from_str(params[0]) - .with_context(|| format!("Failed to parse tenant id from {}", params[0]))?; - let timeline_id = TimelineId::from_str(params[1]) - .with_context(|| format!("Failed to parse timeline id from {}", params[1]))?; - let start_lsn = Lsn::from_str(params[2]) - .with_context(|| format!("Failed to parse Lsn from {}", params[2]))?; - let end_lsn = Lsn::from_str(params[3]) - .with_context(|| format!("Failed to parse Lsn from {}", params[3]))?; - - tracing::Span::current() - .record("tenant_id", field::display(tenant_id)) - .record("timeline_id", field::display(timeline_id)); - - self.check_permission(Some(tenant_id))?; - - COMPUTE_COMMANDS_COUNTERS - .for_command(ComputeCommandKind::ImportWal) - .inc(); - - match self - .handle_import_wal(pgb, tenant_id, timeline_id, start_lsn, end_lsn, ctx) - .await - { - Ok(()) => pgb.write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?, - Err(e) => { - error!("error importing WAL between {start_lsn} and {end_lsn}: {e:?}"); - pgb.write_message_noflush(&BeMessage::ErrorResponse( - &e.to_string(), - Some(e.pg_error_code()), - ))? - } - }; } else if query_string.to_ascii_lowercase().starts_with("set ") { // important because psycopg2 executes "SET datestyle TO 'ISO'" // on connect diff --git a/storage_controller/src/node.rs b/storage_controller/src/node.rs index 4d17dff9feaf..fff44aaf2670 100644 --- a/storage_controller/src/node.rs +++ b/storage_controller/src/node.rs @@ -226,7 +226,7 @@ impl Node { fn is_fatal(e: &mgmt_api::Error) -> bool { use mgmt_api::Error::*; match e { - ReceiveBody(_) | ReceiveErrorBody(_) => false, + SendRequest(_) | ReceiveBody(_) | ReceiveErrorBody(_) => false, ApiError(StatusCode::SERVICE_UNAVAILABLE, _) | ApiError(StatusCode::GATEWAY_TIMEOUT, _) | ApiError(StatusCode::REQUEST_TIMEOUT, _) => false, diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 78f0848c241e..aada1939eeea 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -151,6 +151,10 @@ struct ServiceState { /// controller API. fn passthrough_api_error(node: &Node, e: mgmt_api::Error) -> ApiError { match e { + mgmt_api::Error::SendRequest(e) => { + // Presume errors sending requests are connectivity/availability issues + ApiError::ResourceUnavailable(format!("{node} error sending request: {e}").into()) + } mgmt_api::Error::ReceiveErrorBody(str) => { // Presume errors receiving body are connectivity/availability issues ApiError::ResourceUnavailable( diff --git a/test_runner/regress/test_import.py b/test_runner/regress/test_import.py index d97e882a7093..4dae9176b83f 100644 --- a/test_runner/regress/test_import.py +++ b/test_runner/regress/test_import.py @@ -88,7 +88,8 @@ def test_import_from_vanilla(test_output_dir, pg_bin, vanilla_pg, neon_env_build env.pageserver.allowed_errors.extend( [ - ".*error importing base backup .*", + ".*Failed to import basebackup.*", + ".*unexpected non-zero bytes after the tar archive.*", ".*Timeline got dropped without initializing, cleaning its files.*", ".*InternalServerError.*timeline not found.*", ".*InternalServerError.*Tenant .* not found.*", From fe13fccdc2a0e097bb785edb4ff3913aee35789f Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Wed, 10 Jul 2024 09:10:29 +0100 Subject: [PATCH 079/194] proxy: pg17 fixes (#8321) ## Problem #7809 - we do not support sslnegotiation=direct #7810 - we do not support negotiating down the protocol extensions. ## Summary of changes 1. Same as postgres, check the first startup packet byte for tls header `0x16`, and check the ALPN. 2. Tell clients using protocol >3.0 to downgrade --- libs/postgres_backend/src/lib.rs | 12 ++- libs/pq_proto/src/framed.rs | 6 +- libs/pq_proto/src/lib.rs | 91 ++++++++++++++---- proxy/src/bin/pg_sni_router.rs | 3 +- proxy/src/config.rs | 12 ++- proxy/src/proxy/handshake.rs | 152 ++++++++++++++++++++++++++----- 6 files changed, 222 insertions(+), 54 deletions(-) diff --git a/libs/postgres_backend/src/lib.rs b/libs/postgres_backend/src/lib.rs index c79ee4e0533a..7c7c6535b338 100644 --- a/libs/postgres_backend/src/lib.rs +++ b/libs/postgres_backend/src/lib.rs @@ -663,11 +663,17 @@ impl PostgresBackend { assert!(self.state < ProtoState::Authentication); let have_tls = self.tls_config.is_some(); match msg { - FeStartupPacket::SslRequest => { + FeStartupPacket::SslRequest { direct } => { debug!("SSL requested"); - self.write_message(&BeMessage::EncryptionResponse(have_tls)) - .await?; + if !direct { + self.write_message(&BeMessage::EncryptionResponse(have_tls)) + .await?; + } else if !have_tls { + return Err(QueryError::Other(anyhow::anyhow!( + "direct SSL negotiation but no TLS support" + ))); + } if have_tls { self.start_tls().await?; diff --git a/libs/pq_proto/src/framed.rs b/libs/pq_proto/src/framed.rs index 6e97b8c2a02c..ccbb90e3842e 100644 --- a/libs/pq_proto/src/framed.rs +++ b/libs/pq_proto/src/framed.rs @@ -44,9 +44,9 @@ impl ConnectionError { /// Wraps async io `stream`, providing messages to write/flush + read Postgres /// messages. pub struct Framed { - stream: S, - read_buf: BytesMut, - write_buf: BytesMut, + pub stream: S, + pub read_buf: BytesMut, + pub write_buf: BytesMut, } impl Framed { diff --git a/libs/pq_proto/src/lib.rs b/libs/pq_proto/src/lib.rs index cee374201763..a01191bd5de3 100644 --- a/libs/pq_proto/src/lib.rs +++ b/libs/pq_proto/src/lib.rs @@ -39,14 +39,39 @@ pub enum FeMessage { PasswordMessage(Bytes), } +#[derive(Clone, Copy, PartialEq, PartialOrd)] +pub struct ProtocolVersion(u32); + +impl ProtocolVersion { + pub const fn new(major: u16, minor: u16) -> Self { + Self((major as u32) << 16 | minor as u32) + } + pub const fn minor(self) -> u16 { + self.0 as u16 + } + pub const fn major(self) -> u16 { + (self.0 >> 16) as u16 + } +} + +impl fmt::Debug for ProtocolVersion { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list() + .entry(&self.major()) + .entry(&self.minor()) + .finish() + } +} + #[derive(Debug)] pub enum FeStartupPacket { CancelRequest(CancelKeyData), - SslRequest, + SslRequest { + direct: bool, + }, GssEncRequest, StartupMessage { - major_version: u32, - minor_version: u32, + version: ProtocolVersion, params: StartupMessageParams, }, } @@ -301,11 +326,23 @@ impl FeStartupPacket { /// different from [`FeMessage::parse`] because startup messages don't have /// message type byte; otherwise, its comments apply. pub fn parse(buf: &mut BytesMut) -> Result, ProtocolError> { + /// const MAX_STARTUP_PACKET_LENGTH: usize = 10000; - const RESERVED_INVALID_MAJOR_VERSION: u32 = 1234; - const CANCEL_REQUEST_CODE: u32 = 5678; - const NEGOTIATE_SSL_CODE: u32 = 5679; - const NEGOTIATE_GSS_CODE: u32 = 5680; + const RESERVED_INVALID_MAJOR_VERSION: u16 = 1234; + /// + const CANCEL_REQUEST_CODE: ProtocolVersion = ProtocolVersion::new(1234, 5678); + /// + const NEGOTIATE_SSL_CODE: ProtocolVersion = ProtocolVersion::new(1234, 5679); + /// + const NEGOTIATE_GSS_CODE: ProtocolVersion = ProtocolVersion::new(1234, 5680); + + // + // First byte indicates standard SSL handshake message + // (It can't be a Postgres startup length because in network byte order + // that would be a startup packet hundreds of megabytes long) + if buf.first() == Some(&0x16) { + return Ok(Some(FeStartupPacket::SslRequest { direct: true })); + } // need at least 4 bytes with packet len if buf.len() < 4 { @@ -338,12 +375,10 @@ impl FeStartupPacket { let mut msg = buf.split_to(len).freeze(); msg.advance(4); // consume len - let request_code = msg.get_u32(); - let req_hi = request_code >> 16; - let req_lo = request_code & ((1 << 16) - 1); + let request_code = ProtocolVersion(msg.get_u32()); // StartupMessage, CancelRequest, SSLRequest etc are differentiated by request code. - let message = match (req_hi, req_lo) { - (RESERVED_INVALID_MAJOR_VERSION, CANCEL_REQUEST_CODE) => { + let message = match request_code { + CANCEL_REQUEST_CODE => { if msg.remaining() != 8 { return Err(ProtocolError::BadMessage( "CancelRequest message is malformed, backend PID / secret key missing" @@ -355,21 +390,22 @@ impl FeStartupPacket { cancel_key: msg.get_i32(), }) } - (RESERVED_INVALID_MAJOR_VERSION, NEGOTIATE_SSL_CODE) => { + NEGOTIATE_SSL_CODE => { // Requested upgrade to SSL (aka TLS) - FeStartupPacket::SslRequest + FeStartupPacket::SslRequest { direct: false } } - (RESERVED_INVALID_MAJOR_VERSION, NEGOTIATE_GSS_CODE) => { + NEGOTIATE_GSS_CODE => { // Requested upgrade to GSSAPI FeStartupPacket::GssEncRequest } - (RESERVED_INVALID_MAJOR_VERSION, unrecognized_code) => { + version if version.major() == RESERVED_INVALID_MAJOR_VERSION => { return Err(ProtocolError::Protocol(format!( - "Unrecognized request code {unrecognized_code}" + "Unrecognized request code {}", + version.minor() ))); } // TODO bail if protocol major_version is not 3? - (major_version, minor_version) => { + version => { // StartupMessage let s = str::from_utf8(&msg).map_err(|_e| { @@ -382,8 +418,7 @@ impl FeStartupPacket { })?; FeStartupPacket::StartupMessage { - major_version, - minor_version, + version, params: StartupMessageParams { params: msg.slice_ref(s.as_bytes()), }, @@ -522,6 +557,10 @@ pub enum BeMessage<'a> { RowDescription(&'a [RowDescriptor<'a>]), XLogData(XLogDataBody<'a>), NoticeResponse(&'a str), + NegotiateProtocolVersion { + version: ProtocolVersion, + options: &'a [&'a str], + }, KeepAlive(WalSndKeepAlive), } @@ -945,6 +984,18 @@ impl<'a> BeMessage<'a> { buf.put_u8(u8::from(req.request_reply)); }); } + + BeMessage::NegotiateProtocolVersion { version, options } => { + buf.put_u8(b'v'); + write_body(buf, |buf| { + buf.put_u32(version.0); + buf.put_u32(options.len() as u32); + for option in options.iter() { + write_cstr(option, buf)?; + } + Ok(()) + })? + } } Ok(()) } diff --git a/proxy/src/bin/pg_sni_router.rs b/proxy/src/bin/pg_sni_router.rs index 44e880838e07..d7a3eb9a4d18 100644 --- a/proxy/src/bin/pg_sni_router.rs +++ b/proxy/src/bin/pg_sni_router.rs @@ -216,10 +216,11 @@ async fn ssl_handshake( use pq_proto::FeStartupPacket::*; match msg { - SslRequest => { + SslRequest { direct: false } => { stream .write_message(&pq_proto::BeMessage::EncryptionResponse(true)) .await?; + // Upgrade raw stream into a secure TLS-backed stream. // NOTE: We've consumed `tls`; this fact will be used later. diff --git a/proxy/src/config.rs b/proxy/src/config.rs index af5511d7ec24..650491976053 100644 --- a/proxy/src/config.rs +++ b/proxy/src/config.rs @@ -75,6 +75,9 @@ impl TlsConfig { } } +/// +pub const PG_ALPN_PROTOCOL: &[u8] = b"postgresql"; + /// Configure TLS for the main endpoint. pub fn configure_tls( key_path: &str, @@ -111,16 +114,17 @@ pub fn configure_tls( let cert_resolver = Arc::new(cert_resolver); // allow TLS 1.2 to be compatible with older client libraries - let config = rustls::ServerConfig::builder_with_protocol_versions(&[ + let mut config = rustls::ServerConfig::builder_with_protocol_versions(&[ &rustls::version::TLS13, &rustls::version::TLS12, ]) .with_no_client_auth() - .with_cert_resolver(cert_resolver.clone()) - .into(); + .with_cert_resolver(cert_resolver.clone()); + + config.alpn_protocols = vec![PG_ALPN_PROTOCOL.to_vec()]; Ok(TlsConfig { - config, + config: Arc::new(config), common_names, cert_resolver, }) diff --git a/proxy/src/proxy/handshake.rs b/proxy/src/proxy/handshake.rs index dd935cc24528..d488aea9275b 100644 --- a/proxy/src/proxy/handshake.rs +++ b/proxy/src/proxy/handshake.rs @@ -1,11 +1,17 @@ -use pq_proto::{BeMessage as Be, CancelKeyData, FeStartupPacket, StartupMessageParams}; +use bytes::Buf; +use pq_proto::{ + framed::Framed, BeMessage as Be, CancelKeyData, FeStartupPacket, ProtocolVersion, + StartupMessageParams, +}; use thiserror::Error; use tokio::io::{AsyncRead, AsyncWrite}; -use tracing::info; +use tracing::{info, warn}; use crate::{ - config::TlsConfig, + auth::endpoint_sni, + config::{TlsConfig, PG_ALPN_PROTOCOL}, error::ReportableError, + metrics::Metrics, proxy::ERR_INSECURE_CONNECTION, stream::{PqStream, Stream, StreamUpgradeError}, }; @@ -68,6 +74,9 @@ pub async fn handshake( // Client may try upgrading to each protocol only once let (mut tried_ssl, mut tried_gss) = (false, false); + const PG_PROTOCOL_EARLIEST: ProtocolVersion = ProtocolVersion::new(3, 0); + const PG_PROTOCOL_LATEST: ProtocolVersion = ProtocolVersion::new(3, 0); + let mut stream = PqStream::new(Stream::from_raw(stream)); loop { let msg = stream.read_startup_packet().await?; @@ -75,40 +84,96 @@ pub async fn handshake( use FeStartupPacket::*; match msg { - SslRequest => match stream.get_ref() { + SslRequest { direct } => match stream.get_ref() { Stream::Raw { .. } if !tried_ssl => { tried_ssl = true; // We can't perform TLS handshake without a config - let enc = tls.is_some(); - stream.write_message(&Be::EncryptionResponse(enc)).await?; + let have_tls = tls.is_some(); + if !direct { + stream + .write_message(&Be::EncryptionResponse(have_tls)) + .await?; + } else if !have_tls { + return Err(HandshakeError::ProtocolViolation); + } + if let Some(tls) = tls.take() { // Upgrade raw stream into a secure TLS-backed stream. // NOTE: We've consumed `tls`; this fact will be used later. - let (raw, read_buf) = stream.into_inner(); - // TODO: Normally, client doesn't send any data before - // server says TLS handshake is ok and read_buf is empy. - // However, you could imagine pipelining of postgres - // SSLRequest + TLS ClientHello in one hunk similar to - // pipelining in our node js driver. We should probably - // support that by chaining read_buf with the stream. + let Framed { + stream: raw, + read_buf, + write_buf, + } = stream.framed; + + let Stream::Raw { raw } = raw else { + return Err(HandshakeError::StreamUpgradeError( + StreamUpgradeError::AlreadyTls, + )); + }; + + let mut read_buf = read_buf.reader(); + let mut res = Ok(()); + let accept = tokio_rustls::TlsAcceptor::from(tls.to_server_config()) + .accept_with(raw, |session| { + // push the early data to the tls session + while !read_buf.get_ref().is_empty() { + match session.read_tls(&mut read_buf) { + Ok(_) => {} + Err(e) => { + res = Err(e); + break; + } + } + } + }); + + res?; + + let read_buf = read_buf.into_inner(); if !read_buf.is_empty() { return Err(HandshakeError::EarlyData); } - let tls_stream = raw - .upgrade(tls.to_server_config(), record_handshake_error) - .await?; + + let tls_stream = accept.await.inspect_err(|_| { + if record_handshake_error { + Metrics::get().proxy.tls_handshake_failures.inc() + } + })?; + + let conn_info = tls_stream.get_ref().1; + + // check the ALPN, if exists, as required. + match conn_info.alpn_protocol() { + None | Some(PG_ALPN_PROTOCOL) => {} + Some(other) => { + // try parse ep for better error + let ep = conn_info.server_name().and_then(|sni| { + endpoint_sni(sni, &tls.common_names).ok().flatten() + }); + let alpn = String::from_utf8_lossy(other); + warn!(?ep, %alpn, "unexpected ALPN"); + return Err(HandshakeError::ProtocolViolation); + } + } let (_, tls_server_end_point) = tls .cert_resolver - .resolve(tls_stream.get_ref().1.server_name()) + .resolve(conn_info.server_name()) .ok_or(HandshakeError::MissingCertificate)?; - stream = PqStream::new(Stream::Tls { - tls: Box::new(tls_stream), - tls_server_end_point, - }); + stream = PqStream { + framed: Framed { + stream: Stream::Tls { + tls: Box::new(tls_stream), + tls_server_end_point, + }, + read_buf, + write_buf, + }, + }; } } _ => return Err(HandshakeError::ProtocolViolation), @@ -122,7 +187,9 @@ pub async fn handshake( } _ => return Err(HandshakeError::ProtocolViolation), }, - StartupMessage { params, .. } => { + StartupMessage { params, version } + if PG_PROTOCOL_EARLIEST <= version && version <= PG_PROTOCOL_LATEST => + { // Check that the config has been consumed during upgrade // OR we didn't provide it at all (for dev purposes). if tls.is_some() { @@ -131,9 +198,48 @@ pub async fn handshake( .await?; } - info!(session_type = "normal", "successful handshake"); + info!(?version, session_type = "normal", "successful handshake"); break Ok(HandshakeData::Startup(stream, params)); } + // downgrade protocol version + StartupMessage { params, version } + if version.major() == 3 && version > PG_PROTOCOL_LATEST => + { + warn!(?version, "unsupported minor version"); + + // no protocol extensions are supported. + // + let mut unsupported = vec![]; + for (k, _) in params.iter() { + if k.starts_with("_pq_.") { + unsupported.push(k); + } + } + + // TODO: remove unsupported options so we don't send them to compute. + + stream + .write_message(&Be::NegotiateProtocolVersion { + version: PG_PROTOCOL_LATEST, + options: &unsupported, + }) + .await?; + + info!( + ?version, + session_type = "normal", + "successful handshake; unsupported minor version requested" + ); + break Ok(HandshakeData::Startup(stream, params)); + } + StartupMessage { version, .. } => { + warn!( + ?version, + session_type = "normal", + "unsuccessful handshake; unsupported version" + ); + return Err(HandshakeError::ProtocolViolation); + } CancelRequest(cancel_key_data) => { info!(session_type = "cancellation", "successful handshake"); break Ok(HandshakeData::Cancel(cancel_key_data)); From e89ec55ea571c1f7ca0d722cd2ade07b6c2753cb Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 10 Jul 2024 14:14:10 +0100 Subject: [PATCH 080/194] tests: stabilize test_sharding_split_compaction (#8318) ## Problem This test incorrectly assumed that a post-split compaction would only drop content. This was easily destabilized by any changes to image generation rules. ## Summary of changes - Before split, do a full image layer generation pass, to guarantee that post-split compaction should only drop data, never create it. - Fix the force_image_layer_creation mode of compaction that we use from tests like this: previously it would try and generate image layers even if one already existed with the same layer key, which caused compaction to fail. --- pageserver/src/tenant/timeline.rs | 19 ++++++++++++++++++- .../src/tenant/timeline/layer_manager.rs | 8 ++++++++ test_runner/regress/test_sharding.py | 6 ++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 541704e8d668..762e903bf85d 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -66,12 +66,12 @@ use std::{ ops::{Deref, Range}, }; -use crate::pgdatadir_mapping::MAX_AUX_FILE_V2_DELTAS; use crate::{ aux_file::AuxFileSizeEstimator, tenant::{ layer_map::{LayerMap, SearchResult}, metadata::TimelineMetadata, + storage_layer::PersistentLayerDesc, }, }; use crate::{ @@ -98,6 +98,7 @@ use crate::{ metrics::ScanLatencyOngoingRecording, tenant::timeline::logical_size::CurrentLogicalSize, }; use crate::{pgdatadir_mapping::LsnForTimestamp, tenant::tasks::BackgroundLoopKind}; +use crate::{pgdatadir_mapping::MAX_AUX_FILE_V2_DELTAS, tenant::storage_layer::PersistentLayerKey}; use crate::{ pgdatadir_mapping::{AuxFilesDirectory, DirectoryKind}, virtual_file::{MaybeFatalIo, VirtualFile}, @@ -4572,6 +4573,22 @@ impl Timeline { start = img_range.end; continue; } + } else if let ImageLayerCreationMode::Force = mode { + // When forced to create image layers, we might try and create them where they already + // exist. This mode is only used in tests/debug. + let layers = self.layers.read().await; + if layers.contains_key(&PersistentLayerKey { + key_range: img_range.clone(), + lsn_range: PersistentLayerDesc::image_layer_lsn_range(lsn), + is_delta: false, + }) { + tracing::info!( + "Skipping image layer at {lsn} {}..{}, already exists", + img_range.start, + img_range.end + ); + continue; + } } let image_layer_writer = ImageLayerWriter::new( diff --git a/pageserver/src/tenant/timeline/layer_manager.rs b/pageserver/src/tenant/timeline/layer_manager.rs index 948237e06a5e..a43ff873acb5 100644 --- a/pageserver/src/tenant/timeline/layer_manager.rs +++ b/pageserver/src/tenant/timeline/layer_manager.rs @@ -339,6 +339,10 @@ impl LayerManager { self.layer_fmgr.contains(layer) } + pub(crate) fn contains_key(&self, key: &PersistentLayerKey) -> bool { + self.layer_fmgr.contains_key(key) + } + pub(crate) fn all_persistent_layers(&self) -> Vec { self.layer_fmgr.0.keys().cloned().collect_vec() } @@ -363,6 +367,10 @@ impl LayerFileManager { .clone() } + fn contains_key(&self, key: &PersistentLayerKey) -> bool { + self.0.contains_key(key) + } + pub(crate) fn insert(&mut self, layer: T) { let present = self.0.insert(layer.layer_desc().key(), layer.clone()); if present.is_some() && cfg!(debug_assertions) { diff --git a/test_runner/regress/test_sharding.py b/test_runner/regress/test_sharding.py index d414f986e655..4471237900b8 100644 --- a/test_runner/regress/test_sharding.py +++ b/test_runner/regress/test_sharding.py @@ -225,6 +225,12 @@ def test_sharding_split_compaction(neon_env_builder: NeonEnvBuilder, failpoint: workload.validate() workload.stop() + # Do a full image layer generation before splitting, so that when we compact after splitting + # we should only see sizes decrease (from post-split drops/rewrites), not increase (from image layer generation) + env.get_tenant_pageserver(tenant_id).http_client().timeline_compact( + tenant_id, timeline_id, force_image_layer_creation=True, wait_until_uploaded=True + ) + # Split one shard into two shards = env.storage_controller.tenant_shard_split(tenant_id, shard_count=2) From 1afab13ccb95ed083397c5bff1e31ae1631b1091 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Wed, 10 Jul 2024 15:05:25 +0100 Subject: [PATCH 081/194] proxy: remove some trace logs (#8334) --- proxy/src/http.rs | 41 +---------------------------------------- proxy/src/logging.rs | 3 ++- 2 files changed, 3 insertions(+), 41 deletions(-) diff --git a/proxy/src/http.rs b/proxy/src/http.rs index fc7400869ffa..dd7164181d94 100644 --- a/proxy/src/http.rs +++ b/proxy/src/http.rs @@ -4,14 +4,11 @@ pub mod health_server; -use std::{str::FromStr, sync::Arc, time::Duration}; +use std::time::Duration; -use futures::FutureExt; pub use reqwest::{Request, Response, StatusCode}; pub use reqwest_middleware::{ClientWithMiddleware, Error}; pub use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; -use tokio::time::Instant; -use tracing::trace; use crate::{ metrics::{ConsoleRequest, Metrics}, @@ -24,8 +21,6 @@ use reqwest_middleware::RequestBuilder; /// We deliberately don't want to replace this with a public static. pub fn new_client() -> ClientWithMiddleware { let client = reqwest::ClientBuilder::new() - .dns_resolver(Arc::new(GaiResolver::default())) - .connection_verbose(true) .build() .expect("Failed to create http client"); @@ -36,8 +31,6 @@ pub fn new_client() -> ClientWithMiddleware { pub fn new_client_with_timeout(default_timout: Duration) -> ClientWithMiddleware { let timeout_client = reqwest::ClientBuilder::new() - .dns_resolver(Arc::new(GaiResolver::default())) - .connection_verbose(true) .timeout(default_timout) .build() .expect("Failed to create http client with timeout"); @@ -103,38 +96,6 @@ impl Endpoint { } } -use hyper_util::client::legacy::connect::dns::{ - GaiResolver as HyperGaiResolver, Name as HyperName, -}; -use reqwest::dns::{Addrs, Name, Resolve, Resolving}; -/// https://docs.rs/reqwest/0.11.18/src/reqwest/dns/gai.rs.html -use tower_service::Service; -#[derive(Debug)] -pub struct GaiResolver(HyperGaiResolver); - -impl Default for GaiResolver { - fn default() -> Self { - Self(HyperGaiResolver::new()) - } -} - -impl Resolve for GaiResolver { - fn resolve(&self, name: Name) -> Resolving { - let this = &mut self.0.clone(); - let hyper_name = HyperName::from_str(name.as_str()).expect("name should be valid"); - let start = Instant::now(); - Box::pin( - Service::::call(this, hyper_name).map(move |result| { - let resolve_duration = start.elapsed(); - trace!(duration = ?resolve_duration, addr = %name.as_str(), "resolve host complete"); - result - .map(|addrs| -> Addrs { Box::new(addrs) }) - .map_err(|err| -> Box { Box::new(err) }) - }), - ) - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/proxy/src/logging.rs b/proxy/src/logging.rs index 3405b8cbc672..3b30ad8b4663 100644 --- a/proxy/src/logging.rs +++ b/proxy/src/logging.rs @@ -15,7 +15,8 @@ use tracing_subscriber::{ pub async fn init() -> anyhow::Result { let env_filter = EnvFilter::builder() .with_default_directive(LevelFilter::INFO.into()) - .from_env_lossy(); + .from_env_lossy() + .add_directive("azure_core::policies::transport=off".parse().unwrap()); let fmt_layer = tracing_subscriber::fmt::layer() .with_ansi(false) From 98387d6fb1a125a5e9676534cb46dca88e3252fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Jul 2024 18:12:57 +0000 Subject: [PATCH 082/194] build(deps-dev): bump zipp from 3.8.1 to 3.19.1 Bumps [zipp](https://github.com/jaraco/zipp) from 3.8.1 to 3.19.1. - [Release notes](https://github.com/jaraco/zipp/releases) - [Changelog](https://github.com/jaraco/zipp/blob/main/NEWS.rst) - [Commits](https://github.com/jaraco/zipp/compare/v3.8.1...v3.19.1) --- updated-dependencies: - dependency-name: zipp dependency-type: indirect ... Signed-off-by: dependabot[bot] --- poetry.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index bf16aaf55d26..809114141188 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3133,18 +3133,18 @@ multidict = ">=4.0" [[package]] name = "zipp" -version = "3.8.1" +version = "3.19.1" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "zipp-3.8.1-py3-none-any.whl", hash = "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"}, - {file = "zipp-3.8.1.tar.gz", hash = "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2"}, + {file = "zipp-3.19.1-py3-none-any.whl", hash = "sha256:2828e64edb5386ea6a52e7ba7cdb17bb30a73a858f5eb6eb93d8d36f5ea26091"}, + {file = "zipp-3.19.1.tar.gz", hash = "sha256:35427f6d5594f4acf82d25541438348c26736fa9b3afa2754bcd63cdb99d8e8f"}, ] [package.extras] -docs = ["jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx"] -testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] [[package]] name = "zstandard" From e78341e1c220625d9bfa3f08632bd5cfb8e6a876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Wed, 10 Jul 2024 18:09:19 +0200 Subject: [PATCH 083/194] Remove ImageCompressionAlgorithm::DisabledNoDecompress (#8300) Removes the `ImageCompressionAlgorithm::DisabledNoDecompress` variant. We now assume any blob with the specific bits set is actually a compressed blob. The `ImageCompressionAlgorithm::Disabled` variant still remains and is the new default. Reverts large parts of #8238 , as originally intended in that PR. Part of #5431 --- libs/pageserver_api/src/models.rs | 14 ---------- pageserver/src/config.rs | 2 +- pageserver/src/tenant/blob_io.rs | 11 ++------ pageserver/src/tenant/block_io.rs | 10 +------ .../src/tenant/storage_layer/delta_layer.rs | 2 +- .../src/tenant/storage_layer/image_layer.rs | 28 ++++++------------- pageserver/src/tenant/storage_layer/layer.rs | 1 - 7 files changed, 15 insertions(+), 53 deletions(-) diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index 49c942938dfd..d360cc6e870f 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -440,9 +440,6 @@ pub enum CompactionAlgorithm { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum ImageCompressionAlgorithm { - /// Disabled for writes, and never decompress during reading. - /// Never set this after you've enabled compression once! - DisabledNoDecompress, // Disabled for writes, support decompressing during read path Disabled, /// Zstandard compression. Level 0 means and None mean the same (default level). Levels can be negative as well. @@ -452,12 +449,6 @@ pub enum ImageCompressionAlgorithm { }, } -impl ImageCompressionAlgorithm { - pub fn allow_decompression(&self) -> bool { - !matches!(self, ImageCompressionAlgorithm::DisabledNoDecompress) - } -} - impl FromStr for ImageCompressionAlgorithm { type Err = anyhow::Error; fn from_str(s: &str) -> Result { @@ -466,7 +457,6 @@ impl FromStr for ImageCompressionAlgorithm { .next() .ok_or_else(|| anyhow::anyhow!("empty string"))?; match first { - "disabled-no-decompress" => Ok(ImageCompressionAlgorithm::DisabledNoDecompress), "disabled" => Ok(ImageCompressionAlgorithm::Disabled), "zstd" => { let level = if let Some(v) = components.next() { @@ -1683,10 +1673,6 @@ mod tests { ImageCompressionAlgorithm::from_str("disabled").unwrap(), Disabled ); - assert_eq!( - ImageCompressionAlgorithm::from_str("disabled-no-decompress").unwrap(), - DisabledNoDecompress - ); assert_eq!( ImageCompressionAlgorithm::from_str("zstd").unwrap(), Zstd { level: None } diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index b7c9af224404..17bc427b2cf1 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -92,7 +92,7 @@ pub mod defaults { pub const DEFAULT_MAX_VECTORED_READ_BYTES: usize = 128 * 1024; // 128 KiB pub const DEFAULT_IMAGE_COMPRESSION: ImageCompressionAlgorithm = - ImageCompressionAlgorithm::DisabledNoDecompress; + ImageCompressionAlgorithm::Disabled; pub const DEFAULT_VALIDATE_VECTORED_GET: bool = true; diff --git a/pageserver/src/tenant/blob_io.rs b/pageserver/src/tenant/blob_io.rs index 0705182d5db2..e98ed66ef998 100644 --- a/pageserver/src/tenant/blob_io.rs +++ b/pageserver/src/tenant/blob_io.rs @@ -273,12 +273,8 @@ impl BlobWriter { srcbuf: B, ctx: &RequestContext, ) -> (B::Buf, Result) { - self.write_blob_maybe_compressed( - srcbuf, - ctx, - ImageCompressionAlgorithm::DisabledNoDecompress, - ) - .await + self.write_blob_maybe_compressed(srcbuf, ctx, ImageCompressionAlgorithm::Disabled) + .await } /// Write a blob of data. Returns the offset that it was written to, @@ -340,8 +336,7 @@ impl BlobWriter { (BYTE_UNCOMPRESSED, len, slice.into_inner()) } } - ImageCompressionAlgorithm::Disabled - | ImageCompressionAlgorithm::DisabledNoDecompress => { + ImageCompressionAlgorithm::Disabled => { (BYTE_UNCOMPRESSED, len, srcbuf.slice_full().into_inner()) } }; diff --git a/pageserver/src/tenant/block_io.rs b/pageserver/src/tenant/block_io.rs index 3324e840ecf1..601b09515519 100644 --- a/pageserver/src/tenant/block_io.rs +++ b/pageserver/src/tenant/block_io.rs @@ -202,18 +202,10 @@ pub struct FileBlockReader<'a> { impl<'a> FileBlockReader<'a> { pub fn new(file: &'a VirtualFile, file_id: FileId) -> Self { - Self::new_with_compression(file, file_id, false) - } - - pub fn new_with_compression( - file: &'a VirtualFile, - file_id: FileId, - compressed_reads: bool, - ) -> Self { FileBlockReader { file_id, file, - compressed_reads, + compressed_reads: true, } } diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 685f6dce60e7..000076d7c09d 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -453,7 +453,7 @@ impl DeltaLayerWriterInner { ) -> (Vec, anyhow::Result<()>) { assert!(self.lsn_range.start <= lsn); // We don't want to use compression in delta layer creation - let compression = ImageCompressionAlgorithm::DisabledNoDecompress; + let compression = ImageCompressionAlgorithm::Disabled; let (val, res) = self .blob_writer .write_blob_maybe_compressed(val, ctx, compression) diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index 4a1b3a02377a..50aacbd9ad46 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -165,7 +165,6 @@ pub struct ImageLayerInner { file_id: FileId, max_vectored_read_bytes: Option, - compressed_reads: bool, } impl std::fmt::Debug for ImageLayerInner { @@ -179,8 +178,7 @@ impl std::fmt::Debug for ImageLayerInner { impl ImageLayerInner { pub(super) async fn dump(&self, ctx: &RequestContext) -> anyhow::Result<()> { - let block_reader = - FileBlockReader::new_with_compression(&self.file, self.file_id, self.compressed_reads); + let block_reader = FileBlockReader::new(&self.file, self.file_id); let tree_reader = DiskBtreeReader::<_, KEY_SIZE>::new( self.index_start_blk, self.index_root_blk, @@ -268,10 +266,9 @@ impl ImageLayer { async fn load_inner(&self, ctx: &RequestContext) -> Result { let path = self.path(); - let loaded = - ImageLayerInner::load(&path, self.desc.image_layer_lsn(), None, None, false, ctx) - .await - .and_then(|res| res)?; + let loaded = ImageLayerInner::load(&path, self.desc.image_layer_lsn(), None, None, ctx) + .await + .and_then(|res| res)?; // not production code let actual_layer_name = LayerName::from_str(path.file_name().unwrap()).unwrap(); @@ -380,7 +377,6 @@ impl ImageLayerInner { lsn: Lsn, summary: Option, max_vectored_read_bytes: Option, - support_compressed_reads: bool, ctx: &RequestContext, ) -> Result, anyhow::Error> { let file = match VirtualFile::open(path, ctx).await { @@ -424,7 +420,6 @@ impl ImageLayerInner { file, file_id, max_vectored_read_bytes, - compressed_reads: support_compressed_reads, key_range: actual_summary.key_range, })) } @@ -435,8 +430,7 @@ impl ImageLayerInner { reconstruct_state: &mut ValueReconstructState, ctx: &RequestContext, ) -> anyhow::Result { - let block_reader = - FileBlockReader::new_with_compression(&self.file, self.file_id, self.compressed_reads); + let block_reader = FileBlockReader::new(&self.file, self.file_id); let tree_reader = DiskBtreeReader::new(self.index_start_blk, self.index_root_blk, &block_reader); @@ -496,14 +490,12 @@ impl ImageLayerInner { &self, ctx: &RequestContext, ) -> anyhow::Result> { - let block_reader = - FileBlockReader::new_with_compression(&self.file, self.file_id, self.compressed_reads); + let block_reader = FileBlockReader::new(&self.file, self.file_id); let tree_reader = DiskBtreeReader::new(self.index_start_blk, self.index_root_blk, &block_reader); let mut result = Vec::new(); let mut stream = Box::pin(tree_reader.into_stream(&[0; KEY_SIZE], ctx)); - let block_reader = - FileBlockReader::new_with_compression(&self.file, self.file_id, self.compressed_reads); + let block_reader = FileBlockReader::new(&self.file, self.file_id); let cursor = block_reader.block_cursor(); while let Some(item) = stream.next().await { // TODO: dedup code with get_reconstruct_value @@ -538,8 +530,7 @@ impl ImageLayerInner { .into(), ); - let block_reader = - FileBlockReader::new_with_compression(&self.file, self.file_id, self.compressed_reads); + let block_reader = FileBlockReader::new(&self.file, self.file_id); let tree_reader = DiskBtreeReader::new(self.index_start_blk, self.index_root_blk, block_reader); @@ -700,8 +691,7 @@ impl ImageLayerInner { #[cfg(test)] pub(crate) fn iter<'a>(&'a self, ctx: &'a RequestContext) -> ImageLayerIterator<'a> { - let block_reader = - FileBlockReader::new_with_compression(&self.file, self.file_id, self.compressed_reads); + let block_reader = FileBlockReader::new(&self.file, self.file_id); let tree_reader = DiskBtreeReader::new(self.index_start_blk, self.index_root_blk, block_reader); ImageLayerIterator { diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index afd11780e77d..02069c29d264 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -1685,7 +1685,6 @@ impl DownloadedLayer { lsn, summary, Some(owner.conf.max_vectored_read_bytes), - owner.conf.image_compression.allow_decompression(), ctx, ) .await From 9f4511c5545e86a492966abb4887bcac22fd01d4 Mon Sep 17 00:00:00 2001 From: Alex Chi Z Date: Wed, 10 Jul 2024 14:11:27 -0400 Subject: [PATCH 084/194] feat(pageserver): add k-merge layer iterator with lazy loading (#8053) Part of https://github.com/neondatabase/neon/issues/8002. This pull request adds a k-merge iterator for bottom-most compaction. ## Summary of changes * Added back lsn_range / key_range in delta layer inner. This was removed due to https://github.com/neondatabase/neon/pull/8050, but added back because iterators need that information to process lazy loading. * Added lazy-loading k-merge iterator. * Added iterator wrapper as a unified iterator type for image+delta iterator. The current status and test should cover the use case for L0 compaction so that the L0 compaction process can bypass page cache and have a fixed amount of memory usage. The next step is to integrate this with the new bottom-most compaction. --------- Signed-off-by: Alex Chi Z Co-authored-by: Christian Schwarz --- pageserver/src/tenant/storage_layer.rs | 3 + .../src/tenant/storage_layer/delta_layer.rs | 30 +- .../src/tenant/storage_layer/image_layer.rs | 10 + .../tenant/storage_layer/merge_iterator.rs | 412 ++++++++++++++++++ 4 files changed, 452 insertions(+), 3 deletions(-) create mode 100644 pageserver/src/tenant/storage_layer/merge_iterator.rs diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index 9607546ce0f2..62730f88b260 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -7,6 +7,9 @@ pub(crate) mod layer; mod layer_desc; mod layer_name; +#[cfg(test)] +pub mod merge_iterator; + use crate::context::{AccessStatsBehavior, RequestContext}; use crate::repository::Value; use crate::task_mgr::TaskKind; diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 000076d7c09d..dfd0196c87e9 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -223,6 +223,11 @@ pub struct DeltaLayerInner { file: VirtualFile, file_id: FileId, + #[allow(dead_code)] + layer_key_range: Range, + #[allow(dead_code)] + layer_lsn_range: Range, + max_vectored_read_bytes: Option, } @@ -742,6 +747,16 @@ impl DeltaLayer { } impl DeltaLayerInner { + #[cfg(test)] + pub(crate) fn key_range(&self) -> &Range { + &self.layer_key_range + } + + #[cfg(test)] + pub(crate) fn lsn_range(&self) -> &Range { + &self.layer_lsn_range + } + /// Returns nested result following Result, Critical>: /// - inner has the success or transient failure /// - outer has the permanent failure @@ -790,6 +805,8 @@ impl DeltaLayerInner { index_start_blk: actual_summary.index_start_blk, index_root_blk: actual_summary.index_root_blk, max_vectored_read_bytes, + layer_key_range: actual_summary.key_range, + layer_lsn_range: actual_summary.lsn_range, })) } @@ -1639,7 +1656,7 @@ impl<'a> DeltaLayerIterator<'a> { } #[cfg(test)] -mod test { +pub(crate) mod test { use std::collections::BTreeMap; use itertools::MinMaxResult; @@ -2217,13 +2234,20 @@ mod test { } } - async fn produce_delta_layer( + pub(crate) fn sort_delta( + (k1, l1, _): &(Key, Lsn, Value), + (k2, l2, _): &(Key, Lsn, Value), + ) -> std::cmp::Ordering { + (k1, l1).cmp(&(k2, l2)) + } + + pub(crate) async fn produce_delta_layer( tenant: &Tenant, tline: &Arc, mut deltas: Vec<(Key, Lsn, Value)>, ctx: &RequestContext, ) -> anyhow::Result { - deltas.sort_by(|(k1, l1, _), (k2, l2, _)| (k1, l1).cmp(&(k2, l2))); + deltas.sort_by(sort_delta); let (key_start, _, _) = deltas.first().unwrap(); let (key_max, _, _) = deltas.first().unwrap(); let lsn_min = deltas.iter().map(|(_, lsn, _)| lsn).min().unwrap(); diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index 50aacbd9ad46..1e03e1a58c92 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -369,6 +369,16 @@ impl ImageLayer { } impl ImageLayerInner { + #[cfg(test)] + pub(crate) fn key_range(&self) -> &Range { + &self.key_range + } + + #[cfg(test)] + pub(crate) fn lsn(&self) -> Lsn { + self.lsn + } + /// Returns nested result following Result, Critical>: /// - inner has the success or transient failure /// - outer has the permanent failure diff --git a/pageserver/src/tenant/storage_layer/merge_iterator.rs b/pageserver/src/tenant/storage_layer/merge_iterator.rs new file mode 100644 index 000000000000..36386c87c999 --- /dev/null +++ b/pageserver/src/tenant/storage_layer/merge_iterator.rs @@ -0,0 +1,412 @@ +use std::{ + cmp::Ordering, + collections::{binary_heap, BinaryHeap}, +}; + +use pageserver_api::key::Key; +use utils::lsn::Lsn; + +use crate::{context::RequestContext, repository::Value}; + +use super::{ + delta_layer::{DeltaLayerInner, DeltaLayerIterator}, + image_layer::{ImageLayerInner, ImageLayerIterator}, +}; + +#[derive(Clone, Copy)] +enum LayerRef<'a> { + Image(&'a ImageLayerInner), + Delta(&'a DeltaLayerInner), +} + +impl<'a> LayerRef<'a> { + fn iter(self, ctx: &'a RequestContext) -> LayerIterRef<'a> { + match self { + Self::Image(x) => LayerIterRef::Image(x.iter(ctx)), + Self::Delta(x) => LayerIterRef::Delta(x.iter(ctx)), + } + } +} + +enum LayerIterRef<'a> { + Image(ImageLayerIterator<'a>), + Delta(DeltaLayerIterator<'a>), +} + +impl LayerIterRef<'_> { + async fn next(&mut self) -> anyhow::Result> { + match self { + Self::Delta(x) => x.next().await, + Self::Image(x) => x.next().await, + } + } +} + +/// This type plays several roles at once +/// 1. Unified iterator for image and delta layers. +/// 2. `Ord` for use in [`MergeIterator::heap`] (for the k-merge). +/// 3. Lazy creation of the real delta/image iterator. +enum IteratorWrapper<'a> { + NotLoaded { + ctx: &'a RequestContext, + first_key_lower_bound: (Key, Lsn), + layer: LayerRef<'a>, + }, + Loaded { + iter: PeekableLayerIterRef<'a>, + }, +} + +struct PeekableLayerIterRef<'a> { + iter: LayerIterRef<'a>, + peeked: Option<(Key, Lsn, Value)>, // None == end +} + +impl<'a> PeekableLayerIterRef<'a> { + async fn create(mut iter: LayerIterRef<'a>) -> anyhow::Result { + let peeked = iter.next().await?; + Ok(Self { iter, peeked }) + } + + fn peek(&self) -> &Option<(Key, Lsn, Value)> { + &self.peeked + } + + async fn next(&mut self) -> anyhow::Result> { + let result = self.peeked.take(); + self.peeked = self.iter.next().await?; + Ok(result) + } +} + +impl<'a> std::cmp::PartialEq for IteratorWrapper<'a> { + fn eq(&self, other: &Self) -> bool { + self.cmp(other) == Ordering::Equal + } +} + +impl<'a> std::cmp::Eq for IteratorWrapper<'a> {} + +impl<'a> std::cmp::PartialOrd for IteratorWrapper<'a> { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl<'a> std::cmp::Ord for IteratorWrapper<'a> { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + use std::cmp::Ordering; + let a = self.peek_next_key_lsn(); + let b = other.peek_next_key_lsn(); + match (a, b) { + (Some((k1, l1)), Some((k2, l2))) => { + let loaded_1 = if self.is_loaded() { 1 } else { 0 }; + let loaded_2 = if other.is_loaded() { 1 } else { 0 }; + // When key_lsn are the same, the unloaded iter will always appear before the loaded one. + // And note that we do a reverse at the end of the comparison, so it works with the max heap. + (k1, l1, loaded_1).cmp(&(k2, l2, loaded_2)) + } + (Some(_), None) => Ordering::Less, + (None, Some(_)) => Ordering::Greater, + (None, None) => Ordering::Equal, + } + .reverse() + } +} + +impl<'a> IteratorWrapper<'a> { + pub fn create_from_image_layer( + image_layer: &'a ImageLayerInner, + ctx: &'a RequestContext, + ) -> Self { + Self::NotLoaded { + layer: LayerRef::Image(image_layer), + first_key_lower_bound: (image_layer.key_range().start, image_layer.lsn()), + ctx, + } + } + + pub fn create_from_delta_layer( + delta_layer: &'a DeltaLayerInner, + ctx: &'a RequestContext, + ) -> Self { + Self::NotLoaded { + layer: LayerRef::Delta(delta_layer), + first_key_lower_bound: (delta_layer.key_range().start, delta_layer.lsn_range().start), + ctx, + } + } + + fn peek_next_key_lsn(&self) -> Option<(&Key, Lsn)> { + match self { + Self::Loaded { iter } => iter.peek().as_ref().map(|(key, lsn, _)| (key, *lsn)), + Self::NotLoaded { + first_key_lower_bound: (key, lsn), + .. + } => Some((key, *lsn)), + } + } + + // CORRECTNESS: this function must always take `&mut self`, never `&self`. + // + // The reason is that `impl Ord for Self` evaluates differently after this function + // returns. We're called through a `PeekMut::deref_mut`, which causes heap repair when + // the PeekMut gets returned. So, it's critical that we actually run through `PeekMut::deref_mut` + // and not just `PeekMut::deref` + // If we don't take `&mut self` + async fn load(&mut self) -> anyhow::Result<()> { + assert!(!self.is_loaded()); + let Self::NotLoaded { + ctx, + first_key_lower_bound, + layer, + } = self + else { + unreachable!() + }; + let iter = layer.iter(ctx); + let iter = PeekableLayerIterRef::create(iter).await?; + if let Some((k1, l1, _)) = iter.peek() { + let (k2, l2) = first_key_lower_bound; + debug_assert!((k1, l1) >= (k2, l2)); + } + *self = Self::Loaded { iter }; + Ok(()) + } + + fn is_loaded(&self) -> bool { + matches!(self, Self::Loaded { .. }) + } + + /// Correctness: must load the iterator before using. + /// + /// Given this iterator wrapper is private to the merge iterator, users won't be able to mis-use it. + /// The public interfaces to use are [`crate::tenant::storage_layer::delta_layer::DeltaLayerIterator`] and + /// [`crate::tenant::storage_layer::image_layer::ImageLayerIterator`]. + async fn next(&mut self) -> anyhow::Result> { + let Self::Loaded { iter } = self else { + panic!("must load the iterator before using") + }; + iter.next().await + } +} + +pub struct MergeIterator<'a> { + heap: BinaryHeap>, +} + +impl<'a> MergeIterator<'a> { + pub fn create( + deltas: &[&'a DeltaLayerInner], + images: &[&'a ImageLayerInner], + ctx: &'a RequestContext, + ) -> Self { + let mut heap = Vec::with_capacity(images.len() + deltas.len()); + for image in images { + heap.push(IteratorWrapper::create_from_image_layer(image, ctx)); + } + for delta in deltas { + heap.push(IteratorWrapper::create_from_delta_layer(delta, ctx)); + } + Self { + heap: BinaryHeap::from(heap), + } + } + + pub async fn next(&mut self) -> anyhow::Result> { + while let Some(mut iter) = self.heap.peek_mut() { + if !iter.is_loaded() { + // Once we load the iterator, we can know the real first key-value pair in the iterator. + // We put it back into the heap so that a potentially unloaded layer may have a key between + // [potential_first_key, loaded_first_key). + iter.load().await?; + continue; + } + let Some(item) = iter.next().await? else { + // If the iterator returns None, we pop this iterator. Actually, in the current implementation, + // we order None > Some, and all the rest of the iterators should return None. + binary_heap::PeekMut::pop(iter); + continue; + }; + return Ok(Some(item)); + } + Ok(None) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use itertools::Itertools; + use pageserver_api::key::Key; + use utils::lsn::Lsn; + + use crate::{ + tenant::{ + harness::{TenantHarness, TIMELINE_ID}, + storage_layer::delta_layer::test::{produce_delta_layer, sort_delta}, + }, + DEFAULT_PG_VERSION, + }; + + async fn assert_merge_iter_equal( + merge_iter: &mut MergeIterator<'_>, + expect: &[(Key, Lsn, Value)], + ) { + let mut expect_iter = expect.iter(); + loop { + let o1 = merge_iter.next().await.unwrap(); + let o2 = expect_iter.next(); + assert_eq!(o1.is_some(), o2.is_some()); + if o1.is_none() && o2.is_none() { + break; + } + let (k1, l1, v1) = o1.unwrap(); + let (k2, l2, v2) = o2.unwrap(); + assert_eq!(&k1, k2); + assert_eq!(l1, *l2); + assert_eq!(&v1, v2); + } + } + + #[tokio::test] + async fn merge_in_between() { + use crate::repository::Value; + use bytes::Bytes; + + let harness = TenantHarness::create("merge_iterator_delta_merge").unwrap(); + let (tenant, ctx) = harness.load().await; + + let tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) + .await + .unwrap(); + + fn get_key(id: u32) -> Key { + let mut key = Key::from_hex("000000000033333333444444445500000000").unwrap(); + key.field6 = id; + key + } + let test_deltas1 = vec![ + ( + get_key(0), + Lsn(0x10), + Value::Image(Bytes::copy_from_slice(b"test")), + ), + ( + get_key(5), + Lsn(0x10), + Value::Image(Bytes::copy_from_slice(b"test")), + ), + ]; + let resident_layer_1 = produce_delta_layer(&tenant, &tline, test_deltas1.clone(), &ctx) + .await + .unwrap(); + let test_deltas2 = vec![ + ( + get_key(3), + Lsn(0x10), + Value::Image(Bytes::copy_from_slice(b"test")), + ), + ( + get_key(4), + Lsn(0x10), + Value::Image(Bytes::copy_from_slice(b"test")), + ), + ]; + let resident_layer_2 = produce_delta_layer(&tenant, &tline, test_deltas2.clone(), &ctx) + .await + .unwrap(); + let mut merge_iter = MergeIterator::create( + &[ + resident_layer_2.get_as_delta(&ctx).await.unwrap(), + resident_layer_1.get_as_delta(&ctx).await.unwrap(), + ], + &[], + &ctx, + ); + let mut expect = Vec::new(); + expect.extend(test_deltas1); + expect.extend(test_deltas2); + expect.sort_by(sort_delta); + assert_merge_iter_equal(&mut merge_iter, &expect).await; + } + + #[tokio::test] + async fn delta_merge() { + use crate::repository::Value; + use bytes::Bytes; + + let harness = TenantHarness::create("merge_iterator_delta_merge").unwrap(); + let (tenant, ctx) = harness.load().await; + + let tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) + .await + .unwrap(); + + fn get_key(id: u32) -> Key { + let mut key = Key::from_hex("000000000033333333444444445500000000").unwrap(); + key.field6 = id; + key + } + const N: usize = 1000; + let test_deltas1 = (0..N) + .map(|idx| { + ( + get_key(idx as u32 / 10), + Lsn(0x20 * ((idx as u64) % 10 + 1)), + Value::Image(Bytes::from(format!("img{idx:05}"))), + ) + }) + .collect_vec(); + let resident_layer_1 = produce_delta_layer(&tenant, &tline, test_deltas1.clone(), &ctx) + .await + .unwrap(); + let test_deltas2 = (0..N) + .map(|idx| { + ( + get_key(idx as u32 / 10), + Lsn(0x20 * ((idx as u64) % 10 + 1) + 0x10), + Value::Image(Bytes::from(format!("img{idx:05}"))), + ) + }) + .collect_vec(); + let resident_layer_2 = produce_delta_layer(&tenant, &tline, test_deltas2.clone(), &ctx) + .await + .unwrap(); + let test_deltas3 = (0..N) + .map(|idx| { + ( + get_key(idx as u32 / 10 + N as u32), + Lsn(0x10 * ((idx as u64) % 10 + 1)), + Value::Image(Bytes::from(format!("img{idx:05}"))), + ) + }) + .collect_vec(); + let resident_layer_3 = produce_delta_layer(&tenant, &tline, test_deltas3.clone(), &ctx) + .await + .unwrap(); + let mut merge_iter = MergeIterator::create( + &[ + resident_layer_1.get_as_delta(&ctx).await.unwrap(), + resident_layer_2.get_as_delta(&ctx).await.unwrap(), + resident_layer_3.get_as_delta(&ctx).await.unwrap(), + ], + &[], + &ctx, + ); + let mut expect = Vec::new(); + expect.extend(test_deltas1); + expect.extend(test_deltas2); + expect.extend(test_deltas3); + expect.sort_by(sort_delta); + assert_merge_iter_equal(&mut merge_iter, &expect).await; + + // TODO: test layers are loaded only when needed, reducing num of active iterators in k-merge + } + + // TODO: image layer merge, delta+image mixed merge + // TODO: is it possible to have duplicated delta at same LSN now? we might need to test that +} From 24f8133e890f6c44089291f4211171a3d4428738 Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 10 Jul 2024 19:38:14 +0100 Subject: [PATCH 085/194] safekeeper: add eviction_min_resident to stop evictions thrashing (#8335) ## Problem - The condition for eviction is not time-based: it is possible for a timeline to be restored in response to a client, that client times out, and then as soon as the timeline is restored it is immediately evicted again. - There is no delay on eviction at startup of the safekeeper, so when it starts up and sees many idle timelines, it does many evictions which will likely be immediately restored when someone uses the timeline. ## Summary of changes - Add `eviction_min_resident` parameter, and use it in `ready_for_eviction` to avoid evictions if the timeline has been resident for less than this period. - This also implicitly delays evictions at startup for `eviction_min_resident` - Set this to a very low number for the existing eviction test, which expects immediate eviction. The default period is 15 minutes. The general reasoning for that is that in the worst case where we thrash ~10k timelines on one safekeeper, downloading 16MB for each one, we should set a period that would not overwhelm the node's bandwidth. --- safekeeper/src/bin/safekeeper.rs | 11 ++++++++-- safekeeper/src/lib.rs | 7 +++++++ safekeeper/src/timeline_eviction.rs | 4 ++++ safekeeper/src/timeline_manager.rs | 5 +++++ .../tests/walproposer_sim/safekeeper.rs | 1 + test_runner/fixtures/neon_fixtures.py | 21 +++++++++++++++++-- test_runner/regress/test_wal_acceptor.py | 21 ++++++++++--------- 7 files changed, 56 insertions(+), 14 deletions(-) diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index 4d580e57ed7e..9eb6546d6bae 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -27,8 +27,8 @@ use utils::pid_file; use metrics::set_build_info_metric; use safekeeper::defaults::{ - DEFAULT_CONTROL_FILE_SAVE_INTERVAL, DEFAULT_HEARTBEAT_TIMEOUT, DEFAULT_HTTP_LISTEN_ADDR, - DEFAULT_MAX_OFFLOADER_LAG_BYTES, DEFAULT_PARTIAL_BACKUP_CONCURRENCY, + DEFAULT_CONTROL_FILE_SAVE_INTERVAL, DEFAULT_EVICTION_MIN_RESIDENT, DEFAULT_HEARTBEAT_TIMEOUT, + DEFAULT_HTTP_LISTEN_ADDR, DEFAULT_MAX_OFFLOADER_LAG_BYTES, DEFAULT_PARTIAL_BACKUP_CONCURRENCY, DEFAULT_PARTIAL_BACKUP_TIMEOUT, DEFAULT_PG_LISTEN_ADDR, }; use safekeeper::http; @@ -194,6 +194,12 @@ struct Args { /// Number of allowed concurrent uploads of partial segments to remote storage. #[arg(long, default_value = DEFAULT_PARTIAL_BACKUP_CONCURRENCY)] partial_backup_concurrency: usize, + /// How long a timeline must be resident before it is eligible for eviction. + /// Usually, timeline eviction has to wait for `partial_backup_timeout` before being eligible for eviction, + /// but if a timeline is un-evicted and then _not_ written to, it would immediately flap to evicting again, + /// if it weren't for `eviction_min_resident` preventing that. + #[arg(long, value_parser = humantime::parse_duration, default_value = DEFAULT_EVICTION_MIN_RESIDENT)] + eviction_min_resident: Duration, } // Like PathBufValueParser, but allows empty string. @@ -348,6 +354,7 @@ async fn main() -> anyhow::Result<()> { delete_offloaded_wal: args.delete_offloaded_wal, control_file_save_interval: args.control_file_save_interval, partial_backup_concurrency: args.partial_backup_concurrency, + eviction_min_resident: args.eviction_min_resident, }; // initialize sentry if SENTRY_DSN is provided diff --git a/safekeeper/src/lib.rs b/safekeeper/src/lib.rs index 5cd676d8570c..af83feb77fac 100644 --- a/safekeeper/src/lib.rs +++ b/safekeeper/src/lib.rs @@ -53,6 +53,11 @@ pub mod defaults { pub const DEFAULT_PARTIAL_BACKUP_TIMEOUT: &str = "15m"; pub const DEFAULT_CONTROL_FILE_SAVE_INTERVAL: &str = "300s"; pub const DEFAULT_PARTIAL_BACKUP_CONCURRENCY: &str = "5"; + + // By default, our required residency before eviction is the same as the period that passes + // before uploading a partial segment, so that in normal operation the eviction can happen + // as soon as we have done the partial segment upload. + pub const DEFAULT_EVICTION_MIN_RESIDENT: &str = DEFAULT_PARTIAL_BACKUP_TIMEOUT; } #[derive(Debug, Clone)] @@ -93,6 +98,7 @@ pub struct SafeKeeperConf { pub delete_offloaded_wal: bool, pub control_file_save_interval: Duration, pub partial_backup_concurrency: usize, + pub eviction_min_resident: Duration, } impl SafeKeeperConf { @@ -136,6 +142,7 @@ impl SafeKeeperConf { delete_offloaded_wal: false, control_file_save_interval: Duration::from_secs(1), partial_backup_concurrency: 1, + eviction_min_resident: Duration::ZERO, } } } diff --git a/safekeeper/src/timeline_eviction.rs b/safekeeper/src/timeline_eviction.rs index b303d41b7bab..e4ab65290d52 100644 --- a/safekeeper/src/timeline_eviction.rs +++ b/safekeeper/src/timeline_eviction.rs @@ -5,6 +5,7 @@ use anyhow::Context; use camino::Utf8PathBuf; use remote_storage::RemotePath; +use std::time::Instant; use tokio::{ fs::File, io::{AsyncRead, AsyncWriteExt}, @@ -48,6 +49,7 @@ impl Manager { .flush_lsn .segment_number(self.wal_seg_size) == self.last_removed_segno + 1 + && self.resident_since.elapsed() >= self.conf.eviction_min_resident } /// Evict the timeline to remote storage. @@ -91,6 +93,8 @@ impl Manager { return; } + self.resident_since = Instant::now(); + info!("successfully restored evicted timeline"); } } diff --git a/safekeeper/src/timeline_manager.rs b/safekeeper/src/timeline_manager.rs index 62142162de8c..debf8c824f2d 100644 --- a/safekeeper/src/timeline_manager.rs +++ b/safekeeper/src/timeline_manager.rs @@ -186,6 +186,10 @@ pub(crate) struct Manager { // misc pub(crate) access_service: AccessService, pub(crate) partial_backup_rate_limiter: RateLimiter, + + // Anti-flapping state: we evict timelines eagerly if they are inactive, but should not + // evict them if they go inactive very soon after being restored. + pub(crate) resident_since: std::time::Instant, } /// This task gets spawned alongside each timeline and is responsible for managing the timeline's @@ -350,6 +354,7 @@ impl Manager { access_service: AccessService::new(manager_tx), tli, partial_backup_rate_limiter, + resident_since: std::time::Instant::now(), } } diff --git a/safekeeper/tests/walproposer_sim/safekeeper.rs b/safekeeper/tests/walproposer_sim/safekeeper.rs index 6bbf96d71df4..0c6d97ddfaad 100644 --- a/safekeeper/tests/walproposer_sim/safekeeper.rs +++ b/safekeeper/tests/walproposer_sim/safekeeper.rs @@ -188,6 +188,7 @@ pub fn run_server(os: NodeOs, disk: Arc) -> Result<()> { delete_offloaded_wal: false, control_file_save_interval: Duration::from_secs(1), partial_backup_concurrency: 1, + eviction_min_resident: Duration::ZERO, }; let mut global = GlobalMap::new(disk, conf.clone())?; diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index cae2e422c198..5ca31644a910 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -492,6 +492,7 @@ def __init__( pageserver_virtual_file_io_engine: Optional[str] = None, pageserver_aux_file_policy: Optional[AuxFileStore] = None, pageserver_default_tenant_config_compaction_algorithm: Optional[Dict[str, Any]] = None, + safekeeper_extra_opts: Optional[list[str]] = None, ): self.repo_dir = repo_dir self.rust_log_override = rust_log_override @@ -557,6 +558,8 @@ def __init__( self.pageserver_aux_file_policy = pageserver_aux_file_policy + self.safekeeper_extra_opts = safekeeper_extra_opts + assert test_name.startswith( "test_" ), "Unexpectedly instantiated from outside a test function" @@ -1193,7 +1196,9 @@ def __init__(self, config: NeonEnvBuilder): sk_cfg[ "remote_storage" ] = self.safekeepers_remote_storage.to_toml_inline_table().strip() - self.safekeepers.append(Safekeeper(env=self, id=id, port=port)) + self.safekeepers.append( + Safekeeper(env=self, id=id, port=port, extra_opts=config.safekeeper_extra_opts) + ) cfg["safekeepers"].append(sk_cfg) log.info(f"Config: {cfg}") @@ -4016,16 +4021,28 @@ class Safekeeper(LogUtils): id: int running: bool = False - def __init__(self, env: NeonEnv, port: SafekeeperPort, id: int, running: bool = False): + def __init__( + self, + env: NeonEnv, + port: SafekeeperPort, + id: int, + running: bool = False, + extra_opts: Optional[List[str]] = None, + ): self.env = env self.port = port self.id = id self.running = running self.logfile = Path(self.data_dir) / f"safekeeper-{id}.log" + self.extra_opts = extra_opts def start( self, extra_opts: Optional[List[str]] = None, timeout_in_seconds: Optional[int] = None ) -> "Safekeeper": + if extra_opts is None: + # Apply either the extra_opts passed in, or the ones from our constructor: we do not merge the two. + extra_opts = self.extra_opts + assert self.running is False self.env.neon_cli.safekeeper_start( self.id, extra_opts=extra_opts, timeout_in_seconds=timeout_in_seconds diff --git a/test_runner/regress/test_wal_acceptor.py b/test_runner/regress/test_wal_acceptor.py index febfc102930a..7efd86e3497d 100644 --- a/test_runner/regress/test_wal_acceptor.py +++ b/test_runner/regress/test_wal_acceptor.py @@ -2191,24 +2191,25 @@ def test_s3_eviction( ): neon_env_builder.num_safekeepers = 3 neon_env_builder.enable_safekeeper_remote_storage(RemoteStorageKind.LOCAL_FS) - env = neon_env_builder.init_start( - initial_tenant_conf={ - "checkpoint_timeout": "100ms", - } - ) - extra_opts = [ + neon_env_builder.safekeeper_extra_opts = [ "--enable-offload", "--partial-backup-timeout", "50ms", "--control-file-save-interval", "1s", + # Safekeepers usually wait a while before evicting something: for this test we want them to + # evict things as soon as they are inactive. + "--eviction-min-resident=100ms", ] if delete_offloaded_wal: - extra_opts.append("--delete-offloaded-wal") + neon_env_builder.safekeeper_extra_opts.append("--delete-offloaded-wal") - for sk in env.safekeepers: - sk.stop().start(extra_opts=extra_opts) + env = neon_env_builder.init_start( + initial_tenant_conf={ + "checkpoint_timeout": "100ms", + } + ) n_timelines = 5 @@ -2263,7 +2264,7 @@ def test_s3_eviction( # restarting random safekeepers for sk in env.safekeepers: if random.random() < restart_chance: - sk.stop().start(extra_opts=extra_opts) + sk.stop().start() time.sleep(0.5) # require at least one successful eviction in at least one safekeeper From 6bbd34a216accdea3c6a3bd30df8ab28386afdde Mon Sep 17 00:00:00 2001 From: Stas Kelvich Date: Thu, 11 Jul 2024 10:20:14 +0300 Subject: [PATCH 086/194] Enable core dumps for postgres (#8272) Set core rmilit to ulimited in compute_ctl, so that all child processes inherit it. We could also set rlimit in relevant startup script, but that way we would depend on external setup and might inadvertently disable it again (core dumping worked in pods, but not in VMs with inittab-based startup). --- Cargo.lock | 10 ++++++++++ compute_tools/Cargo.toml | 1 + compute_tools/src/bin/compute_ctl.rs | 4 ++++ compute_tools/src/compute.rs | 2 +- 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 776d95c3c745..9fb3f5385dcb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1236,6 +1236,7 @@ dependencies = [ "regex", "remote_storage", "reqwest 0.12.4", + "rlimit", "rust-ini", "serde", "serde_json", @@ -4901,6 +4902,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "rlimit" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3560f70f30a0f16d11d01ed078a07740fe6b489667abc7c7b029155d9f21c3d8" +dependencies = [ + "libc", +] + [[package]] name = "routerify" version = "3.0.0" diff --git a/compute_tools/Cargo.toml b/compute_tools/Cargo.toml index 8f96530a9deb..8ceb8f2ad216 100644 --- a/compute_tools/Cargo.toml +++ b/compute_tools/Cargo.toml @@ -44,3 +44,4 @@ vm_monitor = { version = "0.1", path = "../libs/vm_monitor/" } zstd = "0.13" bytes = "1.0" rust-ini = "0.20.0" +rlimit = "0.10.1" diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index 7bf5db5a57be..f4c396a85d84 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -64,6 +64,7 @@ use compute_tools::monitor::launch_monitor; use compute_tools::params::*; use compute_tools::spec::*; use compute_tools::swap::resize_swap; +use rlimit::{setrlimit, Resource}; // this is an arbitrary build tag. Fine as a default / for testing purposes // in-case of not-set environment var @@ -72,6 +73,9 @@ const BUILD_TAG_DEFAULT: &str = "latest"; fn main() -> Result<()> { let (build_tag, clap_args) = init()?; + // enable core dumping for all child processes + setrlimit(Resource::CORE, rlimit::INFINITY, rlimit::INFINITY)?; + let (pg_handle, start_pg_result) = { // Enter startup tracing context let _startup_context_guard = startup_context_from_env(); diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index eced6fc0b2e7..1112795d3034 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -1116,7 +1116,7 @@ impl ComputeNode { // EKS worker nodes have following core dump settings: // /proc/sys/kernel/core_pattern -> core // /proc/sys/kernel/core_uses_pid -> 1 - // ulimint -c -> unlimited + // ulimit -c -> unlimited // which results in core dumps being written to postgres data directory as core.. // // Use that as a default location and pattern, except macos where core dumps are written From 69b6675da04ff81f6e2bfe5071e414cc2831e3ed Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 11 Jul 2024 08:23:51 +0100 Subject: [PATCH 087/194] rfcs: add RFC for timeline archival (#8221) A design for a cheap low-resource state for idle timelines: - #8088 --- docs/rfcs/034-timeline-archive.md | 507 ++++++++++++++++++++++++++++++ 1 file changed, 507 insertions(+) create mode 100644 docs/rfcs/034-timeline-archive.md diff --git a/docs/rfcs/034-timeline-archive.md b/docs/rfcs/034-timeline-archive.md new file mode 100644 index 000000000000..c8342169622e --- /dev/null +++ b/docs/rfcs/034-timeline-archive.md @@ -0,0 +1,507 @@ +# Timeline Archival + +## Summary + +This RFC describes a mechanism for pageservers to eliminate local storage + compute work +for timelines which are not in use, in response to external API calls to "archive" a timeline. + +The archived state roughly corresponds to fully offloading a timeline to object storage, such +that its cost is purely the cost of that object storage. + +## Motivation + +Archived timelines serve multiple purposes: +- Act as a 'snapshot' for workloads that would like to retain restorable copies of their + database from longer ago than their PITR window. +- Enable users to create huge numbers of branches (e.g. one per github PR) without having + to diligently clean them up later to avoid overloading the pageserver (currently we support + up to ~500 branches per tenant). + +### Prior art + +Most storage and database systems have some form of snapshot, which can be implemented several ways: +1. full copies of data (e.g. an EBS snapshot to S3) +2. shallow snapshots which are CoW relative to the original version of the data, e.g. on a typical NFS appliance, or a filesystem like CephFS. +3. a series of snapshots which are CoW or de-duplicated relative to one another. + +Today's Neon branches are approximately like `2.`, although due to implementation details branches +often end up storing much more data than they really need, as parent branches assume that all data +at the branch point is needed. The layers pinned in the parent branch may have a much larger size +than the physical size of a compressed image layer representing the data at the branch point. + +## Requirements + +- Enter & exit the archived state in response to external admin API calls +- API calls to modify the archived state are atomic and durable +- An archived timeline should eventually (once out of PITR window) use an efficient compressed + representation, and avoid retaining arbitrarily large data in its parent branch. +- Remote object GETs during tenant start may be O(N) with the number of _active_ branches, + but must not scale with the number of _archived_ branches. +- Background I/O for archived branches should only be done a limited number of times to evolve them + to a long-term-efficient state (e.g. rewriting to image layers). There should be no ongoing "housekeeping" + overhead for archived branches, including operations related to calculating sizes for billing. +- The pageserver should put no load on the safekeeper for archived branches. +- Performance of un-archiving a branch must make good use of S3/disk bandwidth to restore the branch + to a performant state in a short time (linear with the branch's logical size) + +## Non Goals + +- Archived branches are not a literal `fullbackup` postgres snapshot: they are still stored + in Neon's internal format. +- Compute cold starts after activating an archived branch will not have comparable performance to + cold starts on an active branch. +- Archived branches will not use any new/additional compression or de-duplication beyond what + is already implemented for image layers (zstd per page). +- The pageserver will not "auto start" archived branches in response to page_service API requests: they + are only activated explicitly via the HTTP API. +- We will not implement a total offload of archived timelines from safekeepers: their control file (small) will + remain on local disk, although existing eviction mechanisms will remove any segments from local disk. +- We will not expose any prometheus metrics for archived timelines, or make them visible in any + detailed HTTP APIs other than the specific API for listing archived timelines. +- A parent branch may not be archived unless all its children are. + +## Impacted Components + +pageserver, storage controller + +## Terminology + +**Archived**: a branch is _archived_ when an HTTP API request to archive it has succeeded: the caller +may assume that this branch is now very cheap to store, although this may not be physically so until the +branch proceeds to the offloaded state. + +**Active** branches are branches which are available for use by page_service clients, and have a relatively +high cost due to consuming local storage. + +**Offloaded** branches are a subset of _archived_ branches, which have had their local state removed such +that they now consume minimal runtime resources and have a cost similar to the cost of object storage. + +**Activate** (verb): transition from Archived to Active + +**Archive** (verb): transition from Active to Archived + +**Offload** (verb): transition from Archived to Offloaded + +**Offload manifest**: an object stored in S3 that describes timelines which pageservers do not load. + +**Warm up** (verb): operation done on an active branch, by downloading its active layers. Once a branch is +warmed up, good performance will be available to page_service clients. + +## Implementation + +### High level flow + +We may think of a timeline which is archived and then activated as proceeding through a series of states: + +```mermaid +stateDiagram + [*] --> Active(warm) + Active(warm) --> Archived + Archived --> Offloaded + Archived --> Active(warm) + Offloaded --> Active(cold) + Active(cold) --> Active(warm) +``` + +Note that the transition from Archived to Active(warm) is expected to be fairly rare: the most common lifecycles +of branches will be: +- Very frequent: Short lived branches: Active -> Deleted +- Frequent: Long-lived branches: Active -> Archived -> Offloaded -> Deleted +- Rare: Branches used to restore old state: Active ->Archived -> Offloaded -> Active + +These states are _not_ all stored as a single physical state on the timeline, but rather represent the combination +of: +- the timeline's lifecycle state: active or archived, stored in the timeline's index +- its offload state: whether pageserver has chosen to drop local storage of the timeline and write it into the + manifest of offloaded timelines. +- cache state (whether it's warm or cold). + +### Storage format changes + +There are two storage format changes: +1. `index_part.json` gets a new attribute `state` that describes whether the timeline is to + be considered active or archived. +2. A new tenant-level _manifest_ object `tenant_manifest-v1.json` describes which timelines a tenant does not need to load + at startup (and is available for storing other small, rarely changing tenant-wide attributes in future) + +The manifest object will have a format like this: +``` +{ + "offload_timelines": [ + { + "timeline_id": ... + "last_record_lsn": ... + "last_record_lsn_time": ... + "pitr_interval": ... + "last_gc_lsn": ... # equal to last_record_lsn if this branch has no history (i.e. a snapshot) + "logical_size": ... # The size at last_record_lsn + "physical_size" ... + "parent": Option<{ + "timeline_id"... + "lsn"... # Branch point LSN on the parent + "requires_data": bool # True if this branch depends on layers in its parent, identify it here + + }> + } + ] +} +``` + +The information about a timeline in its offload state is intentionally minimal: just enough to decide: +- Whether it requires [archive optimization](#archive-branch-optimization) by rewriting as a set of image layers: we may infer this + by checking if now > last_record_lsn_time - pitr_interval, and pitr_lsn < last_record_lsn. +- Whether a parent branch should include this offloaded branch in its GC inputs to avoid removing + layers that the archived branch depends on +- Whether requests to delete this `timeline_id` should be executed (i.e. if a deletion request + is received for a timeline_id that isn't in the site of live `Timelines` or in the manifest, then + we don't need to go to S3 for the deletion. +- How much archived space to report in consumption metrics + +The contents of the manifest's offload list will also be stored as an attribute of `Tenant`, such that the total +set of timelines may be found by the union of `Tenant::timelines` (non-offloaded timelines) and `Tenant::offloaded` +(offloaded timelines). + +For split-brain protection, the manifest object will be written with a generation suffix, in the same way as +index_part objects are (see [generation numbers RFC](025-generation-numbers.md)). This will add some complexity, but +give us total safety against two pageservers with the same tenant attached fighting over the object. Existing code +for finding the latest generation and for cleaning up old generations (in the scrubber) will be generalized to cover +the manifest file. + +### API & Timeline state + +Timelines will store a lifecycle state (enum of Active or Archived) in their IndexPart. This will +be controlled by a new per-timeline `configure` endpoint. This is intentionally generic naming, which +may be used in future to control other per-timeline attributes (e.g. in future we may make PITR interval +a per-timeline configuration). + +`PUT /v1/tenants/{tenant_id}/timelines/{timeline_id}/configure` +``` +{ + 'state': 'active|archive' +} +``` + +When archiving a timeline, this API will complete as soon as the timeline's state has been set in index_part, and that index has been uploaded. + +When activating a timeline, this API will complete as soon as the timeline's state has been set in index_part, +**and** the `Timeline` object has been instantiated and activated. This will require reading the timeline's +index, but not any data: it should be about as fast as a couple of small S3 requests. + +The API will be available with identical path via the storage controller: calling this on a sharded tenant +will simply map the API call to all the shards. + +Archived timelines may never have descendent timelines which are active. This will be enforced at the API level, +such that activating a timeline requires that all its ancestors are active, and archiving a timeline requires +that all its descendents are archived. It is the callers responsibility to walk the hierarchy of timelines +in the proper order if they would like to archive whole trees of branches. + +Because archive timelines will be excluded from the usual timeline listing APIs, a new API specifically +for archived timelines will be added: this is for use in support/debug: + +``` +GET /v1/tenants/{tenant_id}/archived_timelines + +{ + ...same per-timeline content as the tenant manifest... +} + +``` + +### Tenant attach changes + +Currently, during Tenant::spawn we list all the timelines in the S3 bucket, and then for each timeline +we load their index_part.json. To avoid the number of GETs scaling linearly with the number of archived +timelines, we must have a single object that tells us which timelines do not need to be loaded. The +number of ListObjects requests while listing timelines will still scale O(N), but this is less problematic +because each request covers 1000 timelines. + +This is **not** literally the same as the set of timelines who have state=archived. Rather, it is +the set of timelines which have been offloaded in the background after their state was set to archived. + +We may simply skip loading these timelines: there will be no special state of `Timeline`, they just won't +exist from the perspective of an active `Tenant` apart from in deletion: timeline deletion will need +to check for offloaded timelines as well as active timelines, to avoid wrongly returning 404 on trying +to delete an offloaded timeline. + +### Warm-up API + +`PUT /v1/tenants/{tenant_id}/timelines/{timeline_id}/download?wait_ms=1234` + +This API will be similar to the existing `download_remote_layers` API, but smarter: +- It will not download _all_ remote layers, just the visible set (i.e. layers needed for a read) +- It will download layers in the visible set until reaching `wait_ms`, then return a struct describing progress + of downloads, so that the caller can poll. + +The _visible set_ mentioned above will be calculated by the pageserver in the background, by taking the set +of readable LSNs (i.e. branch points and heads of branches), and walking the layer map to work out which layers +can possibly be read from these LSNs. This concept of layer visibility is more generally useful for cache +eviction and heatmaps, as well as in this specific case of warming up a timeline. + +The caller does not have to wait for the warm up API, or call it at all. But it is strongly advised +to call it, because otherwise populating local contents for a timeline can take a long time when waiting +for SQL queries to coincidentally hit all the layers, and during that time query latency remains quite +volatile. + +### Background work + +Archived branches are not subject to normal compaction. Instead, when the compaction loop encounters +an archived branch, it will consider rewriting the branch to just image layers if the branch has no history +([archive branch optimization](#archive-branch-optimization)), or offloading the timeline from local disk +if its state permits that. + +Additionally, the tenant compaction task will walk the state of already offloaded timelines to consider +optimizing their storage, e.g. if a timeline had some history when offloaded, but since then its PITR +has elapsed and it can now be rewritten to image layers. + +#### Archive branch offload + +Recall that when we archive a timeline via the HTTP API, this only sets a state: it doesn't do +any actual work. + +This work is done in the background compaction loop. It makes sense to tag this work on to the compaction +loop, because it is spiritually aligned: offloading data for archived branches improves storage efficiency. + +The condition for offload is simple: + - a `Timeline` object exists with state `Archived` + - the timeline does not have any non-offloaded children. + + Regarding the condition that children must be offloaded, this will always be eventually true, because + we enforce at the API level that children of archived timelines must themselves be archived, and all + archived timelines will eventually be offloaded. + +Offloading a timeline is simple: +- Read the timeline's attributes that we will store in its offloaded state (especially its logical size) +- Call `shutdown()` on the timeline and remove it from the `Tenant` (as if we were about to delete it) +- Erase all the timeline's content from local storage (`remove_dir_all` on its path) +- Write the tenant manifest to S3 to prevent this timeline being loaded on next start. + +#### Archive branch optimization (flattening) + +When we offloaded a branch, it might have had some history that prevented rewriting it to a single +point in time set of image layers. For example, a branch might have several days of writes and a 7 +day PITR: when we archive it, it still has those days of history. + +Once the PITR has expired, we have an opportunity to reduce the physical footprint of the branch by: +- Writing compressed image layers within the archived branch, as these are more efficient as a way of storing + a point in time compared with delta layers +- Updating the branch's offload metadata to indicate that this branch no longer depends on its ancestor + for data, i.e. the ancestor is free to GC layers files at+below the branch point + +Fully compacting an archived branch into image layers at a single LSN may be thought of as *flattening* the +branch, such that it is now a one-dimensional keyspace rather than a two-dimensional key/lsn space. It becomes +a true snapshot at that LSN. + +It is not always more efficient to flatten a branch than to keep some extra history on the parent: this +is described in more detail in [optimizations](#delaying-storage-optimization-if-retaining-parent-layers-is-cheaper) + +Archive branch optimization should be done _before_ background offloads during compaction, because there may +be timelines which are ready to be offloaded but also would benefit from the optimization step before +being offloaded. For example, a branch which has already fallen out of PITR window and has no history +of its own may be immediately re-written as a series of image layers before being offloaded. + +### Consumption metrics + +Archived timelines and offloaded timelines will be excluded from the synthetic size calculation, in anticipating +that billing structures based on consumption metrics are highly likely to apply different $/GB rates to archived +vs. ordinary content. + +Archived and offloaded timelines' logical size will be reported under the existing `timeline_logical_size` +variant of `MetricsKey`: receivers are then free to bill on this metric as they please. + +### Secondary locations + +Archived timelines (including offloaded timelines) will be excluded from heatmaps, and thereby +when a timeline is archived, after the next cycle of heatmap upload & secondary download, its contents +will be dropped from secondary locations. + +### Sharding + +Archiving or activating a timeline will be done symmetrically across all shards in a tenant, in +the same way that timeline creation and deletion is done. There are no special rules about ordering: +the storage controller may dispatch concurrent calls to all shards when archiving or activating a timeline. + +Since consumption metrics are only transmitted from shard zero, the state of archival on this shard +will be authoritative for consumption metrics. + +## Error cases + +### Errors in sharded tenants + +If one shard in a tenant fails an operation but others succeed, the tenant may end up in a mixed +state, where a timeline is archived on some shards but not on others. + +We will not bother implementing a rollback mechanism for this: errors in archiving/activating a timeline +are either transient (e.g. S3 unavailable, shutting down), or the fault of the caller (NotFound, BadRequest). +In the transient case callers are expected to retry until success, or to make appropriate API calls to clear +up their mistake. We rely on this good behavior of callers to eventually get timelines into a consistent +state across all shards. If callers do leave a timeline in an inconsistent state across shards, this doesn't +break anything, it's just "weird". + +This is similar to the status quo for timeline creation and deletion: callers are expected to retry +these operations until they succeed. + +### Archiving/activating + +Archiving/activating a timeline can fail in a limited number of ways: +1. I/O error storing/reading the timeline's updated index + - These errors are always retryable: a fundamental design assumption of the pageserver is that remote + storage errors are always transient. +2. NotFound if the timeline doesn't exist + - Callers of the API are expected to avoid calling deletion and archival APIs concurrently. + - The storage controller has runtime locking to prevent races such as deleting a timeline while + archiving it. +3. BadRequest if the rules around ancestors/descendents of archived timelines would be violated + - Callers are expected to do their own checks to avoid hitting this case. If they make + a mistake and encounter this error, they should give up. + +### Offloading + +Offloading can only fail if remote storage is unavailable, which would prevent us from writing the +tenant manifest. In such error cases, we give up in the expectation that offloading will be tried +again at the next iteration of the compaction loop. + +### Archive branch optimization + +Optimization is a special form of compaction, so can encounter all the same errors as regular compaction +can: it should return Result<(), CompactionError>, and as with compaction it will be retried on +the next iteration of the compaction loop. + +## Optimizations + +### Delaying storage optimization if retaining parent layers is cheaper + +Optimizing archived branches to image layers and thereby enabling parent branch GC to progress +is a safe default: archived branches cannot over-fill a pageserver's local disk, and once they +are offloaded to S3 they're totally safe, inert things. + +However, in some cases it can be advantageous to retain extra history on their parent branch rather +than flattening the archived branch. For example, if a 1TB parent branch is rather slow-changing (1GB +of data per day), and archive branches are being created nightly, then writing out full 1TB image layers +for each nightly branch is inefficient compared with just keeping more history on the main branch. + +Getting this right requires consideration of: +- Compaction: if keeping more history on the main branch is going to prompt the main branch's compaction to + write out extra image layers, then it might make more sense to just write out the image layers on + the archived branch. +- Metadata bloat: keeping extra history on a parent branch doesn't just cost GB of storage, it makes + the layer map (and index_part) bigger. There are practical limits beyond which writing an indefinitely + large layer map can cause problems elsewhere. + +This optimization can probably be implemented quite cheaply with some basic heuristics like: +- don't bother doing optimization on an archive branch if the LSN distance between + its branch point and the end of the PITR window is <5% of the logical size of the archive branch. +- ...but, Don't keep more history on the main branch than double the PITR + +### Creating a timeline in archived state (a snapshot) + +Sometimes, one might want to create a branch with no history, which will not be written to +before it is archived. This is a snapshot, although we do not require a special snapshot API, +since a snapshot can be represented as a timeline with no history. + +This can be accomplished by simply creating a timeline and then immediately archiving it, but +that is somewhat wasteful: this timeline it will spin up various tasks and open a connection to the storage +broker to try and ingest WAL, before being shutdown in the subsequent archival call. To explicitly +support this common special case, we may add a parameter to the timeline creation API which +creates a timeline directly into the archived state. + +Such a timeline creation will do exactly two I/Os at creation time: +- write the index_part object to record the timeline's existence +- when the timeline is offloaded in the next iteration of the compaction loop (~20s later), + write the tenant manifest. + +Later, when the timeline falls off the end of the PITR interval, the usual offload logic will wake +up the 'snapshot' branch and write out image layers. + +## Future Work + +### Enabling `fullbackup` dumps from archive branches + +It would be useful to be able to export an archive branch to another system, or for use in a local +postgres database. + +This could be implemented as a general capability for all branches, in which case it would "just work" +for archive branches by activating them. However, downloading all the layers in a branch just to generate +a fullbackup is a bit inefficient: we could implement a special case for flattened archived branches +which streams image layers from S3 and outputs the fullbackup stream without writing the layers out to disk. + +Implementing `fullbackup` is a bit more complicated than this because of sharding, but solving that problem +is unrelated to the topic of archived branches (it probably involves having each shard write out a fullbackup +stream to S3 in an intermediate format and, then having one node stitch them together). + +### Tagging layers from archived branches + +When we know a layer is an image layer written for an archived branch that has fallen off the PITR window, +we may add tags to the S3 objects to enable writing lifecycle policies that transition such layers to even +cheaper storage. + +This could be done for all archived layers, or it could be driven by the archival API, to give the pageserver +external hints on which branches are likely to be reactivated, and which branches are good candidates for +tagging for low performance storage. + +Tagging+lifecycles is just one mechanism: one might also directly use S3 storage classes. Other clouds' object +stores have similar mechanisms. + +### Storing sequences of archive branches as deltas + +When archived branches are used as scheduled snapshots, we could store them even more efficiently +by encoding them as deltas relative to each other (i.e. for nightly snapshots, when we do the +storage optimization for Tuesday's snapshot, we would read Monday's snapshot and store only the modified +pages). This is the kind of encoding that many backup storage systems use. + +The utility of this depends a lot on the churn rate of the data, and the cost of doing the delta encoding +vs. just writing out a simple stream of the entire database. For smaller databases, writing out a full +copy is pretty trivial (e.g. writing a compressed copy of a 10GiB database to S3 can take under 10 seconds, +so the complexity tradeoff of diff-encoding it is dubious). + +One does not necessarily have to read-back the previous snapshot in order to encoded the next one: if the +pageserver knows about the schedule, it can intentionally retain extra history on the main branch so that +we can say: "A branch exists from Monday night. I have Monday night's data still active in the main branch, +so now I can read at the Monday LSN and the Tuesday LSN, calculate the delta, and store it as Tuesday's +delta snapshot". + +Clearly this all requires careful housekeeping to retain the relationship between branches that depend on +each other: perhaps this would be done by making the archive branches have child/parent relationships with +each other, or perhaps we would permit them to remain children of their original parent, but additionally +have a relationship with the snapshot they're encoded relative to. + +Activating a branch that is diff-encoded may require activating several earlier branches too, so figuring +out how frequently to write a full copy is important. This is essentially a zoomed-out version of what +we do with delta layers and image layers within a timeline, except each "layer" is a whole timeline. + + +## FAQ/Alternatives + +### Store all timelines in the tenant manifest + +Rather than special-casing offloaded timelines in the offload manifest, we could store a total +manifest of all timelines, eliminating the need for the pageserver to list timelines in S3 on +startup. + +That would be a more invasive change (require hooking in to timeline creation), and would +generate much more I/O to this manifest for tenants that had many branches _and_ frequent +create/delete cycles for short lived branches. Restricting the manifest to offloaded timelines +means that we only have to cope with the rate at which long-lived timelines are archived, rather +than the rate at which sort lived timelines are created & destroyed. + +### Automatically archiving/activating timelines without external API calls + +We could implement TTL driven offload of timelines, waking them up when a page request +arrives. + +This has downsides: +- Opacity: if we do TTL-driven offload inside the pageserver, then the end user doesn't + know which of their branches are in this state, and might get a surprise when they try + to use such a branch. +- Price fluctuation: if the archival of a branch is used in end user pricing, then users + prefer clarity & consistency. Ideally a branch's storage should cost the same from the moment it + is created, rather than having a usage-dependency storage price. +- Complexity: enabling the page service to call up into the Tenant to activate a timeline + would be awkward, compared with an external entry point. + +### Make offloaded a state of Timeline + +To reduce the operator-facing complexity of having some timelines APIs that only return +non-offloaded timelines, we could build the offloaded state into the Timeline type. + +`timeline.rs` is already one of the most egregiously long source files in the tree, so +this is rejected on the basis that we need to avoid making that complexity worse. \ No newline at end of file From c11b9cb43dfccffd2ce0c48a31119d29ecd28b0f Mon Sep 17 00:00:00 2001 From: Peter Bendel Date: Thu, 11 Jul 2024 11:07:12 +0200 Subject: [PATCH 088/194] Run Performance bench on more platforms (#8312) ## Problem https://github.com/neondatabase/cloud/issues/14721 ## Summary of changes add one more platform to benchmarking job https://github.com/neondatabase/neon/blob/57535c039c938f7c179693d9db8b052912019823/.github/workflows/benchmarking.yml#L57C3-L126 Run with pg 16, provisioner k8-neonvm by default on the new platform. Adjust some test cases to - not depend on database client <-> database server latency by pushing loops into server side pl/pgSQL functions - increase statement and test timeouts First successful run of these job steps https://github.com/neondatabase/neon/actions/runs/9869817756/job/27254280428 --- .github/workflows/benchmarking.yml | 21 +++++++++++++---- test_runner/performance/test_hot_page.py | 28 +++++++++++++++++------ test_runner/performance/test_hot_table.py | 21 +++++++++++++---- 3 files changed, 55 insertions(+), 15 deletions(-) diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index 899cae2b8658..d038f64f15b0 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -56,15 +56,26 @@ concurrency: jobs: bench: if: ${{ github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null }} + strategy: + matrix: + include: + - DEFAULT_PG_VERSION: 14 + PLATFORM: "neon-staging" + region_id: ${{ github.event.inputs.region_id || 'aws-us-east-2' }} + provisioner: 'k8s-pod' + - DEFAULT_PG_VERSION: 16 + PLATFORM: "azure-staging" + region_id: 'azure-eastus2' + provisioner: 'k8s-neonvm' env: TEST_PG_BENCH_DURATIONS_MATRIX: "300" TEST_PG_BENCH_SCALES_MATRIX: "10,100" POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install - DEFAULT_PG_VERSION: 14 + DEFAULT_PG_VERSION: ${{ matrix.DEFAULT_PG_VERSION }} TEST_OUTPUT: /tmp/test_output BUILD_TYPE: remote SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }} - PLATFORM: "neon-staging" + PLATFORM: ${{ matrix.PLATFORM }} runs-on: [ self-hosted, us-east-2, x64 ] container: @@ -85,9 +96,10 @@ jobs: id: create-neon-project uses: ./.github/actions/neon-project-create with: - region_id: ${{ github.event.inputs.region_id || 'aws-us-east-2' }} + region_id: ${{ matrix.region_id }} postgres_version: ${{ env.DEFAULT_PG_VERSION }} api_key: ${{ secrets.NEON_STAGING_API_KEY }} + provisioner: ${{ matrix.provisioner }} - name: Run benchmark uses: ./.github/actions/run-python-test-set @@ -96,13 +108,14 @@ jobs: test_selection: performance run_in_parallel: false save_perf_report: ${{ env.SAVE_PERF_REPORT }} + pg_version: ${{ env.DEFAULT_PG_VERSION }} # Set --sparse-ordering option of pytest-order plugin # to ensure tests are running in order of appears in the file. # It's important for test_perf_pgbench.py::test_pgbench_remote_* tests extra_params: -m remote_cluster --sparse-ordering - --timeout 5400 + --timeout 14400 --ignore test_runner/performance/test_perf_olap.py --ignore test_runner/performance/test_perf_pgvector_queries.py --ignore test_runner/performance/test_logical_replication.py diff --git a/test_runner/performance/test_hot_page.py b/test_runner/performance/test_hot_page.py index d9785dd87e04..5e97c7cddf13 100644 --- a/test_runner/performance/test_hot_page.py +++ b/test_runner/performance/test_hot_page.py @@ -16,20 +16,34 @@ ) def test_hot_page(env: PgCompare): # Update the same page many times, then measure read performance - num_writes = 1000000 with closing(env.pg.connect()) as conn: with conn.cursor() as cur: cur.execute("drop table if exists t, f;") + num_writes = 1000000 - # Write many updates to the same row + # Use a PL/pgSQL block to perform many updates to the same row + # without depending on the latency between database client and postgres + # server + # - however a single staement should not run into a timeout so we increase it + cur.execute("SET statement_timeout = '4h';") with env.record_duration("write"): - cur.execute("create table t (i integer);") - cur.execute("insert into t values (0);") - for i in range(num_writes): - cur.execute(f"update t set i = {i};") + cur.execute( + f""" + DO $$ + BEGIN + create table t (i integer); + insert into t values (0); - # Write 3-4 MB to evict t from compute cache + FOR j IN 1..{num_writes} LOOP + update t set i = j; + END LOOP; + END $$; + """ + ) + + # Write ca 350 MB to evict t from compute shared buffers (128 MB) + # however it will still be in LFC, so I do not really understand the point of this test cur.execute("create table f (i integer);") cur.execute("insert into f values (generate_series(1,100000));") diff --git a/test_runner/performance/test_hot_table.py b/test_runner/performance/test_hot_table.py index 5fcffc8afb7a..9a78c92ec0e1 100644 --- a/test_runner/performance/test_hot_table.py +++ b/test_runner/performance/test_hot_table.py @@ -16,8 +16,8 @@ ) def test_hot_table(env: PgCompare): # Update a small table many times, then measure read performance - num_rows = 100000 # Slightly larger than shared buffers size TODO validate - num_writes = 1000000 + num_rows = 100000 # initial table size only about 4 MB + num_writes = 10000000 # write approximately 349 MB blocks > 128 MB shared_buffers num_reads = 10 with closing(env.pg.connect()) as conn: @@ -28,8 +28,21 @@ def test_hot_table(env: PgCompare): with env.record_duration("write"): cur.execute("create table t (i integer primary key);") cur.execute(f"insert into t values (generate_series(1,{num_rows}));") - for i in range(num_writes): - cur.execute(f"update t set i = {i + num_rows} WHERE i = {i};") + # PL/pgSQL block to perform updates (and avoid latency between client and server) + # - however a single staement should not run into a timeout so we increase it + cur.execute("SET statement_timeout = '4h';") + cur.execute( + f""" + DO $$ + DECLARE + r integer := {num_rows}; + BEGIN + FOR j IN 1..{num_writes} LOOP + UPDATE t SET i = j + r WHERE i = j; + END LOOP; + END $$; + """ + ) # Read the table with env.record_duration("read"): From e26ef640c1004306c7be192e7afece93f2f529c0 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Thu, 11 Jul 2024 15:17:07 +0200 Subject: [PATCH 089/194] pageserver: remove `trace_read_requests` (#8338) `trace_read_requests` is a per `Tenant`-object option. But the `handle_pagerequests` loop doesn't know which `Tenant` object (i.e., which shard) the request is for. The remaining use of the `Tenant` object is to check `tenant.cancel`. That check is incorrect [if the pageserver hosts multiple shards](https://github.com/neondatabase/neon/issues/7427#issuecomment-2220577518). I'll fix that in a future PR where I completely eliminate the holding of `Tenant/Timeline` objects across requests. See [my code RFC](https://github.com/neondatabase/neon/pull/8286) for the high level idea. Note that we can always bring the tracing functionality if we need it. But since it's actually about logging the `page_service` wire bytes, it should be a `page_service`-level config option, not per-Tenant. And for enabling tracing on a single connection, we can implement a `set pageserver_trace_connection;` option. --- Cargo.lock | 11 -- Cargo.toml | 1 - control_plane/src/pageserver.rs | 10 -- libs/pageserver_api/src/models.rs | 1 - libs/utils/src/id.rs | 11 -- pageserver/src/config.rs | 45 ----- pageserver/src/http/openapi_spec.yml | 2 - pageserver/src/lib.rs | 1 - pageserver/src/page_service.rs | 19 -- pageserver/src/tenant.rs | 8 - pageserver/src/tenant/config.rs | 10 -- pageserver/src/trace.rs | 36 ---- test_runner/fixtures/compare_fixtures.py | 2 - .../regress/test_attach_tenant_config.py | 1 - test_runner/regress/test_read_trace.py | 39 ---- trace/Cargo.toml | 13 -- trace/src/main.rs | 167 ------------------ 17 files changed, 377 deletions(-) delete mode 100644 pageserver/src/trace.rs delete mode 100644 test_runner/regress/test_read_trace.py delete mode 100644 trace/Cargo.toml delete mode 100644 trace/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 9fb3f5385dcb..4b1525edeee5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6510,17 +6510,6 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" -[[package]] -name = "trace" -version = "0.1.0" -dependencies = [ - "anyhow", - "clap", - "pageserver_api", - "utils", - "workspace_hack", -] - [[package]] name = "tracing" version = "0.1.37" diff --git a/Cargo.toml b/Cargo.toml index fc3dd5180922..6bad8e3b20ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,6 @@ members = [ "storage_controller", "storage_scrubber", "workspace_hack", - "trace", "libs/compute_api", "libs/pageserver_api", "libs/postgres_ffi", diff --git a/control_plane/src/pageserver.rs b/control_plane/src/pageserver.rs index f0403b179622..5f2373e95a68 100644 --- a/control_plane/src/pageserver.rs +++ b/control_plane/src/pageserver.rs @@ -349,11 +349,6 @@ impl PageServerNode { .map(|x| x.parse::()) .transpose() .context("Failed to parse 'max_lsn_wal_lag' as non zero integer")?, - trace_read_requests: settings - .remove("trace_read_requests") - .map(|x| x.parse::()) - .transpose() - .context("Failed to parse 'trace_read_requests' as bool")?, eviction_policy: settings .remove("eviction_policy") .map(serde_json::from_str) @@ -454,11 +449,6 @@ impl PageServerNode { .map(|x| x.parse::()) .transpose() .context("Failed to parse 'max_lsn_wal_lag' as non zero integer")?, - trace_read_requests: settings - .remove("trace_read_requests") - .map(|x| x.parse::()) - .transpose() - .context("Failed to parse 'trace_read_requests' as bool")?, eviction_policy: settings .remove("eviction_policy") .map(serde_json::from_str) diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index d360cc6e870f..6abdcb88d0fb 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -294,7 +294,6 @@ pub struct TenantConfig { pub walreceiver_connect_timeout: Option, pub lagging_wal_timeout: Option, pub max_lsn_wal_lag: Option, - pub trace_read_requests: Option, pub eviction_policy: Option, pub min_resident_size_override: Option, pub evictions_low_residence_duration_metric_threshold: Option, diff --git a/libs/utils/src/id.rs b/libs/utils/src/id.rs index 0409001f4f78..db468e30548b 100644 --- a/libs/utils/src/id.rs +++ b/libs/utils/src/id.rs @@ -302,17 +302,6 @@ pub struct TenantId(Id); id_newtype!(TenantId); -/// Neon Connection Id identifies long-lived connections (for example a pagestream -/// connection with the page_service). Is used for better logging and tracing -/// -/// NOTE: It (de)serializes as an array of hex bytes, so the string representation would look -/// like `[173,80,132,115,129,226,72,254,170,201,135,108,199,26,228,24]`. -/// See [`Id`] for alternative ways to serialize it. -#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, PartialOrd, Ord)] -pub struct ConnectionId(Id); - -id_newtype!(ConnectionId); - // A pair uniquely identifying Neon instance. #[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct TenantTimelineId { diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index 17bc427b2cf1..5b103b551fb1 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -12,7 +12,6 @@ use serde::de::IntoDeserializer; use std::env; use storage_broker::Uri; use utils::crashsafe::path_with_suffix_extension; -use utils::id::ConnectionId; use utils::logging::SecretString; use once_cell::sync::OnceCell; @@ -870,22 +869,6 @@ impl PageServerConf { ) } - pub fn traces_path(&self) -> Utf8PathBuf { - self.workdir.join("traces") - } - - pub fn trace_path( - &self, - tenant_shard_id: &TenantShardId, - timeline_id: &TimelineId, - connection_id: &ConnectionId, - ) -> Utf8PathBuf { - self.traces_path() - .join(tenant_shard_id.to_string()) - .join(timeline_id.to_string()) - .join(connection_id.to_string()) - } - /// Turns storage remote path of a file into its local path. pub fn local_path(&self, remote_path: &RemotePath) -> Utf8PathBuf { remote_path.with_base(&self.workdir) @@ -1560,34 +1543,6 @@ broker_endpoint = '{broker_endpoint}' Ok(()) } - #[test] - fn parse_tenant_config() -> anyhow::Result<()> { - let tempdir = tempdir()?; - let (workdir, pg_distrib_dir) = prepare_fs(&tempdir)?; - - let broker_endpoint = "http://127.0.0.1:7777"; - let trace_read_requests = true; - - let config_string = format!( - r#"{ALL_BASE_VALUES_TOML} -pg_distrib_dir='{pg_distrib_dir}' -broker_endpoint = '{broker_endpoint}' - -[tenant_config] -trace_read_requests = {trace_read_requests}"#, - ); - - let toml = config_string.parse()?; - - let conf = PageServerConf::parse_and_validate(&toml, &workdir)?; - assert_eq!( - conf.default_tenant_conf.trace_read_requests, trace_read_requests, - "Tenant config from pageserver config file should be parsed and udpated values used as defaults for all tenants", - ); - - Ok(()) - } - #[test] fn parse_incorrect_tenant_config() -> anyhow::Result<()> { let config_string = r#" diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index 5ba329f05ece..ae109ec1e75f 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -873,8 +873,6 @@ components: type: string max_lsn_wal_lag: type: integer - trace_read_requests: - type: boolean heatmap_period: type: string TenantConfigResponse: diff --git a/pageserver/src/lib.rs b/pageserver/src/lib.rs index ac6b9b4f2a60..63c677574fe5 100644 --- a/pageserver/src/lib.rs +++ b/pageserver/src/lib.rs @@ -23,7 +23,6 @@ pub mod span; pub(crate) mod statvfs; pub mod task_mgr; pub mod tenant; -pub mod trace; pub mod utilization; pub mod virtual_file; pub mod walingest; diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index c10c2f2a0f9a..f94b0d335e8e 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -36,7 +36,6 @@ use tokio::io::AsyncWriteExt; use tokio::io::{AsyncRead, AsyncWrite}; use tokio_util::sync::CancellationToken; use tracing::*; -use utils::id::ConnectionId; use utils::sync::gate::GateGuard; use utils::{ auth::{Claims, Scope, SwappableJwtAuth}, @@ -66,7 +65,6 @@ use crate::tenant::GetTimelineError; use crate::tenant::PageReconstructError; use crate::tenant::Tenant; use crate::tenant::Timeline; -use crate::trace::Tracer; use pageserver_api::key::rel_block_to_key; use pageserver_api::reltag::SlruKind; use postgres_ffi::pg_constants::DEFAULTTABLESPACE_OID; @@ -430,18 +428,6 @@ impl PageServerHandler { .get_active_tenant_with_timeout(tenant_id, ShardSelector::First, ACTIVE_TENANT_TIMEOUT) .await?; - // Make request tracer if needed - let mut tracer = if tenant.get_trace_read_requests() { - let connection_id = ConnectionId::generate(); - let path = - tenant - .conf - .trace_path(&tenant.tenant_shard_id(), &timeline_id, &connection_id); - Some(Tracer::new(path)) - } else { - None - }; - // switch client to COPYBOTH pgb.write_message_noflush(&BeMessage::CopyBothResponse)?; self.flush_cancellable(pgb, &tenant.cancel).await?; @@ -473,11 +459,6 @@ impl PageServerHandler { trace!("query: {copy_data_bytes:?}"); fail::fail_point!("ps::handle-pagerequest-message"); - // Trace request if needed - if let Some(t) = tracer.as_mut() { - t.trace(©_data_bytes) - } - let neon_fe_msg = PagestreamFeMessage::parse(&mut copy_data_bytes.reader(), protocol_version)?; diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index eef8dc104c69..bf23513527fb 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -2341,13 +2341,6 @@ impl Tenant { .unwrap_or(self.conf.default_tenant_conf.pitr_interval) } - pub fn get_trace_read_requests(&self) -> bool { - let tenant_conf = self.tenant_conf.load().tenant_conf.clone(); - tenant_conf - .trace_read_requests - .unwrap_or(self.conf.default_tenant_conf.trace_read_requests) - } - pub fn get_min_resident_size_override(&self) -> Option { let tenant_conf = self.tenant_conf.load().tenant_conf.clone(); tenant_conf @@ -3718,7 +3711,6 @@ pub(crate) mod harness { walreceiver_connect_timeout: Some(tenant_conf.walreceiver_connect_timeout), lagging_wal_timeout: Some(tenant_conf.lagging_wal_timeout), max_lsn_wal_lag: Some(tenant_conf.max_lsn_wal_lag), - trace_read_requests: Some(tenant_conf.trace_read_requests), eviction_policy: Some(tenant_conf.eviction_policy), min_resident_size_override: tenant_conf.min_resident_size_override, evictions_low_residence_duration_metric_threshold: Some( diff --git a/pageserver/src/tenant/config.rs b/pageserver/src/tenant/config.rs index 5b532e483004..48ff17db9460 100644 --- a/pageserver/src/tenant/config.rs +++ b/pageserver/src/tenant/config.rs @@ -335,7 +335,6 @@ pub struct TenantConf { /// A lagging safekeeper will be changed after `lagging_wal_timeout` time elapses since the last WAL update, /// to avoid eager reconnects. pub max_lsn_wal_lag: NonZeroU64, - pub trace_read_requests: bool, pub eviction_policy: EvictionPolicy, pub min_resident_size_override: Option, // See the corresponding metric's help string. @@ -436,10 +435,6 @@ pub struct TenantConfOpt { #[serde(default)] pub max_lsn_wal_lag: Option, - #[serde(skip_serializing_if = "Option::is_none")] - #[serde(default)] - pub trace_read_requests: Option, - #[serde(skip_serializing_if = "Option::is_none")] #[serde(default)] pub eviction_policy: Option, @@ -519,9 +514,6 @@ impl TenantConfOpt { .lagging_wal_timeout .unwrap_or(global_conf.lagging_wal_timeout), max_lsn_wal_lag: self.max_lsn_wal_lag.unwrap_or(global_conf.max_lsn_wal_lag), - trace_read_requests: self - .trace_read_requests - .unwrap_or(global_conf.trace_read_requests), eviction_policy: self.eviction_policy.unwrap_or(global_conf.eviction_policy), min_resident_size_override: self .min_resident_size_override @@ -581,7 +573,6 @@ impl Default for TenantConf { .expect("cannot parse default walreceiver lagging wal timeout"), max_lsn_wal_lag: NonZeroU64::new(DEFAULT_MAX_WALRECEIVER_LSN_WAL_LAG) .expect("cannot parse default max walreceiver Lsn wal lag"), - trace_read_requests: false, eviction_policy: EvictionPolicy::NoEviction, min_resident_size_override: None, evictions_low_residence_duration_metric_threshold: humantime::parse_duration( @@ -659,7 +650,6 @@ impl From for models::TenantConfig { walreceiver_connect_timeout: value.walreceiver_connect_timeout.map(humantime), lagging_wal_timeout: value.lagging_wal_timeout.map(humantime), max_lsn_wal_lag: value.max_lsn_wal_lag, - trace_read_requests: value.trace_read_requests, eviction_policy: value.eviction_policy, min_resident_size_override: value.min_resident_size_override, evictions_low_residence_duration_metric_threshold: value diff --git a/pageserver/src/trace.rs b/pageserver/src/trace.rs deleted file mode 100644 index 18ec2691981f..000000000000 --- a/pageserver/src/trace.rs +++ /dev/null @@ -1,36 +0,0 @@ -use bytes::Bytes; -use camino::Utf8PathBuf; -use std::{ - fs::{create_dir_all, File}, - io::{BufWriter, Write}, -}; - -pub struct Tracer { - writer: BufWriter, -} - -impl Drop for Tracer { - fn drop(&mut self) { - self.flush() - } -} - -impl Tracer { - pub fn new(path: Utf8PathBuf) -> Self { - let parent = path.parent().expect("failed to parse parent path"); - create_dir_all(parent).expect("failed to create trace dir"); - - let file = File::create(path).expect("failed to create trace file"); - Tracer { - writer: BufWriter::new(file), - } - } - - pub fn trace(&mut self, msg: &Bytes) { - self.writer.write_all(msg).expect("failed to write trace"); - } - - pub fn flush(&mut self) { - self.writer.flush().expect("failed to flush trace file"); - } -} diff --git a/test_runner/fixtures/compare_fixtures.py b/test_runner/fixtures/compare_fixtures.py index 429b6af54811..08215438e1d7 100644 --- a/test_runner/fixtures/compare_fixtures.py +++ b/test_runner/fixtures/compare_fixtures.py @@ -109,8 +109,6 @@ def __init__( # Create tenant tenant_conf: Dict[str, str] = {} - if False: # TODO add pytest setting for this - tenant_conf["trace_read_requests"] = "true" self.tenant, _ = self.env.neon_cli.create_tenant(conf=tenant_conf) # Create timeline diff --git a/test_runner/regress/test_attach_tenant_config.py b/test_runner/regress/test_attach_tenant_config.py index f2ee2b70aac6..a7eda73d4ce8 100644 --- a/test_runner/regress/test_attach_tenant_config.py +++ b/test_runner/regress/test_attach_tenant_config.py @@ -168,7 +168,6 @@ def test_fully_custom_config(positive_env: NeonEnv): "refill_amount": 1000, "max": 1000, }, - "trace_read_requests": True, "walreceiver_connect_timeout": "13m", "image_layer_creation_check_threshold": 1, "switch_aux_file_policy": "cross-validation", diff --git a/test_runner/regress/test_read_trace.py b/test_runner/regress/test_read_trace.py deleted file mode 100644 index cc5853b7277f..000000000000 --- a/test_runner/regress/test_read_trace.py +++ /dev/null @@ -1,39 +0,0 @@ -from contextlib import closing - -from fixtures.common_types import Lsn -from fixtures.neon_fixtures import NeonEnvBuilder -from fixtures.pageserver.utils import wait_for_last_record_lsn -from fixtures.utils import query_scalar - - -# This test demonstrates how to collect a read trace. It's useful until -# it gets replaced by a test that actually does stuff with the trace. -# -# Additionally, tests that pageserver is able to create tenants with custom configs. -def test_read_request_tracing(neon_env_builder: NeonEnvBuilder): - neon_env_builder.num_safekeepers = 1 - env = neon_env_builder.init_start( - initial_tenant_conf={ - "trace_read_requests": "true", - } - ) - - tenant_id = env.initial_tenant - timeline_id = env.initial_timeline - endpoint = env.endpoints.create_start("main") - - with closing(endpoint.connect()) as conn: - with conn.cursor() as cur: - cur.execute("create table t (i integer);") - cur.execute(f"insert into t values (generate_series(1,{10000}));") - cur.execute("select count(*) from t;") - current_lsn = Lsn(query_scalar(cur, "SELECT pg_current_wal_flush_lsn()")) - # wait until pageserver receives that data - pageserver_http = env.pageserver.http_client() - wait_for_last_record_lsn(pageserver_http, tenant_id, timeline_id, current_lsn) - - # Stop postgres so we drop the connection and flush the traces - endpoint.stop() - - trace_path = env.pageserver.workdir / "traces" / str(tenant_id) / str(timeline_id) - assert trace_path.exists() diff --git a/trace/Cargo.toml b/trace/Cargo.toml deleted file mode 100644 index d6eed3f49c33..000000000000 --- a/trace/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "trace" -version = "0.1.0" -edition.workspace = true -license.workspace = true - -[dependencies] -clap.workspace = true -anyhow.workspace = true - -pageserver_api.workspace = true -utils.workspace = true -workspace_hack.workspace = true diff --git a/trace/src/main.rs b/trace/src/main.rs deleted file mode 100644 index 79e1df988dae..000000000000 --- a/trace/src/main.rs +++ /dev/null @@ -1,167 +0,0 @@ -//! A tool for working with read traces generated by the pageserver. -use std::collections::HashMap; -use std::path::PathBuf; -use std::str::FromStr; -use std::{ - fs::{read_dir, File}, - io::BufReader, -}; - -use pageserver_api::models::{ - PagestreamFeMessage, PagestreamGetPageRequest, PagestreamProtocolVersion, -}; -use utils::id::{ConnectionId, TenantId, TimelineId}; - -use clap::{Parser, Subcommand}; - -/// Utils for working with pageserver read traces. For generating -/// traces, see the `trace_read_requests` tenant config option. -#[derive(Parser, Debug)] -#[command(author, version, about, long_about = None)] -struct Args { - /// Path of trace directory - #[arg(short, long)] - path: PathBuf, - - #[command(subcommand)] - command: Command, -} - -/// What to do with the read trace -#[derive(Subcommand, Debug)] -enum Command { - /// List traces in the directory - List, - - /// Print the traces in text format - Dump, - - /// Print stats and anomalies about the traces - Analyze, -} - -// HACK This function will change and improve as we see what kind of analysis is useful. -// Currently it collects the difference in blkno of consecutive GetPage requests, -// and counts the frequency of each value. This information is useful in order to: -// - see how sequential a workload is by seeing how often the delta is 1 -// - detect any prefetching anomalies by looking for negative deltas during seqscan -fn analyze_trace(mut reader: R) { - let mut total = 0; // Total requests traced - let mut cross_rel = 0; // Requests that ask for different rel than previous request - let mut deltas = HashMap::::new(); // Consecutive blkno differences - let mut prev: Option = None; - - // Compute stats - while let Ok(msg) = PagestreamFeMessage::parse(&mut reader, PagestreamProtocolVersion::V2) { - match msg { - PagestreamFeMessage::Exists(_) => {} - PagestreamFeMessage::Nblocks(_) => {} - PagestreamFeMessage::GetSlruSegment(_) => {} - PagestreamFeMessage::GetPage(req) => { - total += 1; - - if let Some(prev) = prev { - if prev.rel == req.rel { - let delta = (req.blkno as i32) - (prev.blkno as i32); - deltas.entry(delta).and_modify(|c| *c += 1).or_insert(1); - } else { - cross_rel += 1; - } - } - prev = Some(req); - } - PagestreamFeMessage::DbSize(_) => {} - }; - } - - // Print stats. - let mut other = deltas.len(); - deltas.retain(|_, count| *count > 300); - other -= deltas.len(); - dbg!(total); - dbg!(cross_rel); - dbg!(other); - dbg!(deltas); -} - -fn dump_trace(mut reader: R) { - while let Ok(msg) = PagestreamFeMessage::parse(&mut reader, PagestreamProtocolVersion::V2) { - println!("{msg:?}"); - } -} - -#[derive(Debug)] -struct TraceFile { - #[allow(dead_code)] - pub tenant_id: TenantId, - - #[allow(dead_code)] - pub timeline_id: TimelineId, - - #[allow(dead_code)] - pub connection_id: ConnectionId, - - pub path: PathBuf, -} - -fn get_trace_files(traces_dir: &PathBuf) -> anyhow::Result> { - let mut trace_files = Vec::::new(); - - // Trace files are organized as {tenant_id}/{timeline_id}/{connection_id} - for tenant_dir in read_dir(traces_dir)? { - let entry = tenant_dir?; - let path = entry.path(); - let tenant_id = TenantId::from_str(path.file_name().unwrap().to_str().unwrap())?; - - for timeline_dir in read_dir(path)? { - let entry = timeline_dir?; - let path = entry.path(); - let timeline_id = TimelineId::from_str(path.file_name().unwrap().to_str().unwrap())?; - - for trace_dir in read_dir(path)? { - let entry = trace_dir?; - let path = entry.path(); - let connection_id = - ConnectionId::from_str(path.file_name().unwrap().to_str().unwrap())?; - - trace_files.push(TraceFile { - tenant_id, - timeline_id, - connection_id, - path, - }); - } - } - } - - Ok(trace_files) -} - -fn main() -> anyhow::Result<()> { - let args = Args::parse(); - - match args.command { - Command::List => { - for trace_file in get_trace_files(&args.path)? { - println!("{trace_file:?}"); - } - } - Command::Dump => { - for trace_file in get_trace_files(&args.path)? { - let file = File::open(trace_file.path.clone())?; - let reader = BufReader::new(file); - dump_trace(reader); - } - } - Command::Analyze => { - for trace_file in get_trace_files(&args.path)? { - println!("analyzing {trace_file:?}"); - let file = File::open(trace_file.path.clone())?; - let reader = BufReader::new(file); - analyze_trace(reader); - } - } - } - - Ok(()) -} From d9a82468e27e185fb1f18d4da0d63ac18e37ac2d Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Thu, 11 Jul 2024 15:43:28 +0100 Subject: [PATCH 090/194] storage_controller: fix ReconcilerWaiter::get_status (#8341) ## Problem SeqWait::would_wait_for returns Ok in the case when we would not wait for the sequence number and Err otherwise. ReconcilerWaiter::get_status uses it the wrong way around. This can cause the storage controller to go into a busy loop and make it look unavailable to the k8s controller. ## Summary of changes Use `SeqWait::would_wait_for` correctly. --- storage_controller/src/tenant_shard.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage_controller/src/tenant_shard.rs b/storage_controller/src/tenant_shard.rs index 3fcf31ac1028..2ddab58aafbb 100644 --- a/storage_controller/src/tenant_shard.rs +++ b/storage_controller/src/tenant_shard.rs @@ -383,9 +383,9 @@ impl ReconcilerWaiter { } pub(crate) fn get_status(&self) -> ReconcilerStatus { - if self.seq_wait.would_wait_for(self.seq).is_err() { + if self.seq_wait.would_wait_for(self.seq).is_ok() { ReconcilerStatus::Done - } else if self.error_seq_wait.would_wait_for(self.seq).is_err() { + } else if self.error_seq_wait.would_wait_for(self.seq).is_ok() { ReconcilerStatus::Failed } else { ReconcilerStatus::InProgress From 0159ae9536d6b9e0a9cb27b0ced3fd244faf63d0 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 11 Jul 2024 17:05:35 +0100 Subject: [PATCH 091/194] safekeeper: eviction metrics (#8348) ## Problem Follow up to https://github.com/neondatabase/neon/pull/8335, to improve observability of how many evict/restores we are doing. ## Summary of changes - Add `safekeeper_eviction_events_started_total` and `safekeeper_eviction_events_completed_total`, with a "kind" label of evict or restore. This gives us rates, and also ability to calculate how many are in progress. - Generalize SafekeeperMetrics test type to use the same helpers as pageserver, and enable querying any metric. - Read the new metrics at the end of the eviction test. --- Cargo.lock | 2 + safekeeper/Cargo.toml | 2 + safekeeper/src/metrics.rs | 26 +++++++++++++ safekeeper/src/timeline_eviction.rs | 19 ++++++++++ test_runner/fixtures/safekeeper/http.py | 48 +++++++++++------------- test_runner/regress/test_wal_acceptor.py | 24 +++++++++++- 6 files changed, 92 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4b1525edeee5..b31ac69e6c50 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5206,6 +5206,8 @@ dependencies = [ "sha2", "signal-hook", "storage_broker", + "strum", + "strum_macros", "thiserror", "tokio", "tokio-io-timeout", diff --git a/safekeeper/Cargo.toml b/safekeeper/Cargo.toml index a650d5e20753..9f32016fd97b 100644 --- a/safekeeper/Cargo.toml +++ b/safekeeper/Cargo.toml @@ -41,6 +41,8 @@ serde.workspace = true serde_json.workspace = true serde_with.workspace = true signal-hook.workspace = true +strum.workspace = true +strum_macros.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["fs"] } tokio-util = { workspace = true } diff --git a/safekeeper/src/metrics.rs b/safekeeper/src/metrics.rs index 539ecf826bf8..aa2bafbe9229 100644 --- a/safekeeper/src/metrics.rs +++ b/safekeeper/src/metrics.rs @@ -205,6 +205,32 @@ pub static WAL_BACKUP_TASKS: Lazy = Lazy::new(|| { .expect("Failed to register safekeeper_wal_backup_tasks_finished_total counter") }); +// Metrics collected on operations on the storage repository. +#[derive(strum_macros::EnumString, strum_macros::Display, strum_macros::IntoStaticStr)] +#[strum(serialize_all = "kebab_case")] +pub(crate) enum EvictionEvent { + Evict, + Restore, +} + +pub(crate) static EVICTION_EVENTS_STARTED: Lazy = Lazy::new(|| { + register_int_counter_vec!( + "safekeeper_eviction_events_started_total", + "Number of eviction state changes, incremented when they start", + &["kind"] + ) + .expect("Failed to register metric") +}); + +pub(crate) static EVICTION_EVENTS_COMPLETED: Lazy = Lazy::new(|| { + register_int_counter_vec!( + "safekeeper_eviction_events_completed_total", + "Number of eviction state changes, incremented when they complete", + &["kind"] + ) + .expect("Failed to register metric") +}); + pub const LABEL_UNKNOWN: &str = "unknown"; /// Labels for traffic metrics. diff --git a/safekeeper/src/timeline_eviction.rs b/safekeeper/src/timeline_eviction.rs index e4ab65290d52..0b8d58ee8a52 100644 --- a/safekeeper/src/timeline_eviction.rs +++ b/safekeeper/src/timeline_eviction.rs @@ -14,6 +14,7 @@ use tracing::{debug, info, instrument, warn}; use utils::crashsafe::durable_rename; use crate::{ + metrics::{EvictionEvent, EVICTION_EVENTS_COMPLETED, EVICTION_EVENTS_STARTED}, timeline_manager::{Manager, StateSnapshot}, wal_backup, wal_backup_partial::{self, PartialRemoteSegment}, @@ -66,6 +67,15 @@ impl Manager { info!("starting eviction, using {:?}", partial_backup_uploaded); + EVICTION_EVENTS_STARTED + .with_label_values(&[EvictionEvent::Evict.into()]) + .inc(); + let _guard = scopeguard::guard((), |_| { + EVICTION_EVENTS_COMPLETED + .with_label_values(&[EvictionEvent::Evict.into()]) + .inc(); + }); + if let Err(e) = do_eviction(self, &partial_backup_uploaded).await { warn!("failed to evict timeline: {:?}", e); return; @@ -88,6 +98,15 @@ impl Manager { info!("starting uneviction, using {:?}", partial_backup_uploaded); + EVICTION_EVENTS_STARTED + .with_label_values(&[EvictionEvent::Restore.into()]) + .inc(); + let _guard = scopeguard::guard((), |_| { + EVICTION_EVENTS_COMPLETED + .with_label_values(&[EvictionEvent::Restore.into()]) + .inc(); + }); + if let Err(e) = do_uneviction(self, &partial_backup_uploaded).await { warn!("failed to unevict timeline: {:?}", e); return; diff --git a/test_runner/fixtures/safekeeper/http.py b/test_runner/fixtures/safekeeper/http.py index 11e6fef28f17..a51b89744b63 100644 --- a/test_runner/fixtures/safekeeper/http.py +++ b/test_runner/fixtures/safekeeper/http.py @@ -1,6 +1,5 @@ import json -import re -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Any, Dict, List, Optional, Tuple, Union import pytest @@ -8,6 +7,7 @@ from fixtures.common_types import Lsn, TenantId, TimelineId from fixtures.log_helper import log +from fixtures.metrics import Metrics, MetricsGetter, parse_metrics # Walreceiver as returned by sk's timeline status endpoint. @@ -31,15 +31,26 @@ class SafekeeperTimelineStatus: walreceivers: List[Walreceiver] -@dataclass -class SafekeeperMetrics: +class SafekeeperMetrics(Metrics): + # Helpers to get metrics from tests without hardcoding the metric names there. # These are metrics from Prometheus which uses float64 internally. # As a consequence, values may differ from real original int64s. - flush_lsn_inexact: Dict[Tuple[TenantId, TimelineId], int] = field(default_factory=dict) - commit_lsn_inexact: Dict[Tuple[TenantId, TimelineId], int] = field(default_factory=dict) + + def __init__(self, m: Metrics): + self.metrics = m.metrics + + def flush_lsn_inexact(self, tenant_id: TenantId, timeline_id: TimelineId): + return self.query_one( + "safekeeper_flush_lsn", {"tenant_id": str(tenant_id), "timeline_id": str(timeline_id)} + ).value + + def commit_lsn_inexact(self, tenant_id: TenantId, timeline_id: TimelineId): + return self.query_one( + "safekeeper_commit_lsn", {"tenant_id": str(tenant_id), "timeline_id": str(timeline_id)} + ).value -class SafekeeperHttpClient(requests.Session): +class SafekeeperHttpClient(requests.Session, MetricsGetter): HTTPError = requests.HTTPError def __init__(self, port: int, auth_token: Optional[str] = None, is_testing_enabled=False): @@ -209,28 +220,11 @@ def tenant_delete_force(self, tenant_id: TenantId) -> Dict[Any, Any]: return res_json def get_metrics_str(self) -> str: + """You probably want to use get_metrics() instead.""" request_result = self.get(f"http://localhost:{self.port}/metrics") request_result.raise_for_status() return request_result.text def get_metrics(self) -> SafekeeperMetrics: - all_metrics_text = self.get_metrics_str() - - metrics = SafekeeperMetrics() - for match in re.finditer( - r'^safekeeper_flush_lsn{tenant_id="([0-9a-f]+)",timeline_id="([0-9a-f]+)"} (\S+)$', - all_metrics_text, - re.MULTILINE, - ): - metrics.flush_lsn_inexact[(TenantId(match.group(1)), TimelineId(match.group(2)))] = int( - match.group(3) - ) - for match in re.finditer( - r'^safekeeper_commit_lsn{tenant_id="([0-9a-f]+)",timeline_id="([0-9a-f]+)"} (\S+)$', - all_metrics_text, - re.MULTILINE, - ): - metrics.commit_lsn_inexact[ - (TenantId(match.group(1)), TimelineId(match.group(2))) - ] = int(match.group(3)) - return metrics + res = self.get_metrics_str() + return SafekeeperMetrics(parse_metrics(res)) diff --git a/test_runner/regress/test_wal_acceptor.py b/test_runner/regress/test_wal_acceptor.py index 7efd86e3497d..e0ad4fdd5cf9 100644 --- a/test_runner/regress/test_wal_acceptor.py +++ b/test_runner/regress/test_wal_acceptor.py @@ -147,8 +147,8 @@ def collect_metrics(message: str) -> List[TimelineMetrics]: last_record_lsn=Lsn(timeline_detail["last_record_lsn"]), ) for sk_m in sk_metrics: - m.flush_lsns.append(Lsn(sk_m.flush_lsn_inexact[(tenant_id, timeline_id)])) - m.commit_lsns.append(Lsn(sk_m.commit_lsn_inexact[(tenant_id, timeline_id)])) + m.flush_lsns.append(Lsn(int(sk_m.flush_lsn_inexact(tenant_id, timeline_id)))) + m.commit_lsns.append(Lsn(int(sk_m.commit_lsn_inexact(tenant_id, timeline_id)))) for flush_lsn, commit_lsn in zip(m.flush_lsns, m.commit_lsns): # Invariant. May be < when transaction is in progress. @@ -2274,3 +2274,23 @@ def test_s3_eviction( and sk.log_contains("successfully restored evicted timeline") for sk in env.safekeepers ) + + assert any( + sk.http_client().get_metric_value( + "safekeeper_eviction_events_started_total", {"kind": "evict"} + ) + or 0 > 0 + and sk.http_client().get_metric_value( + "safekeeper_eviction_events_completed_total", {"kind": "evict"} + ) + or 0 > 0 + and sk.http_client().get_metric_value( + "safekeeper_eviction_events_started_total", {"kind": "restore"} + ) + or 0 > 0 + and sk.http_client().get_metric_value( + "safekeeper_eviction_events_completed_total", {"kind": "restore"} + ) + or 0 > 0 + for sk in env.safekeepers + ) From 814c8e8f683ee8fdddc86de99bf33900f423b4d4 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 11 Jul 2024 17:05:47 +0100 Subject: [PATCH 092/194] storage controller: add node deletion API (#8226) ## Problem In anticipation of later adding a really nice drain+delete API, I initially only added an intentionally basic `/drop` API that is just about usable for deleting nodes in a pinch, but requires some ugly storage controller restarts to persuade it to restart secondaries. ## Summary of changes I started making a few tiny fixes, and ended up writing the delete API... - Quality of life nit: ordering of node + tenant listings in storcon_cli - Papercut: Fix the attach_hook using the wrong operation type for reporting slow locks - Make Service::spawn tolerate `generation_pageserver` columns that point to nonexistent node IDs. I started out thinking of this as a general resilience thing, but when implementing the delete API I realized it was actually a legitimate end state after the delete API is called (as that API doesn't wait for all reconciles to succeed). - Add a `DELETE` API for nodes, which does not gracefully drain, but does reschedule everything. This becomes safe to use when the system is in any state, but will incur availability gaps for any tenants that weren't already live-migrated away. If tenants have already been drained, this becomes a totally clean + safe way to decom a node. - Add a test and a storcon_cli wrapper for it This is meant to be a robust initial API that lets us remove nodes without doing ugly things like restarting the storage controller -- it's not quite a totally graceful node-draining routine yet. There's more work in https://github.com/neondatabase/neon/issues/8333 to get to our end-end state. --- control_plane/storcon_cli/src/main.rs | 19 ++- storage_controller/src/http.rs | 11 ++ storage_controller/src/service.rs | 121 +++++++++++++++++- storage_controller/src/tenant_shard.rs | 19 ++- test_runner/fixtures/neon_fixtures.py | 8 ++ test_runner/regress/test_compatibility.py | 25 ++++ .../regress/test_storage_controller.py | 88 +++++++++++++ 7 files changed, 277 insertions(+), 14 deletions(-) diff --git a/control_plane/storcon_cli/src/main.rs b/control_plane/storcon_cli/src/main.rs index b2c5dfe58a7f..815f5c940f4c 100644 --- a/control_plane/storcon_cli/src/main.rs +++ b/control_plane/storcon_cli/src/main.rs @@ -56,6 +56,10 @@ enum Command { #[arg(long)] scheduling: Option, }, + NodeDelete { + #[arg(long)] + node_id: NodeId, + }, /// Modify a tenant's policies in the storage controller TenantPolicy { #[arg(long)] @@ -357,13 +361,16 @@ async fn main() -> anyhow::Result<()> { tracing::info!("Delete status: {}", status); } Command::Nodes {} => { - let resp = storcon_client + let mut resp = storcon_client .dispatch::<(), Vec>( Method::GET, "control/v1/node".to_string(), None, ) .await?; + + resp.sort_by(|a, b| a.listen_http_addr.cmp(&b.listen_http_addr)); + let mut table = comfy_table::Table::new(); table.set_header(["Id", "Hostname", "Scheduling", "Availability"]); for node in resp { @@ -395,13 +402,16 @@ async fn main() -> anyhow::Result<()> { .await?; } Command::Tenants {} => { - let resp = storcon_client + let mut resp = storcon_client .dispatch::<(), Vec>( Method::GET, "control/v1/tenant".to_string(), None, ) .await?; + + resp.sort_by(|a, b| a.tenant_id.cmp(&b.tenant_id)); + let mut table = comfy_table::Table::new(); table.set_header([ "TenantId", @@ -650,6 +660,11 @@ async fn main() -> anyhow::Result<()> { .dispatch::<(), ()>(Method::POST, format!("debug/v1/node/{node_id}/drop"), None) .await?; } + Command::NodeDelete { node_id } => { + storcon_client + .dispatch::<(), ()>(Method::DELETE, format!("control/v1/node/{node_id}"), None) + .await?; + } Command::TenantSetTimeBasedEviction { tenant_id, period, diff --git a/storage_controller/src/http.rs b/storage_controller/src/http.rs index 7446ad53a231..3a62c0dd4ffb 100644 --- a/storage_controller/src/http.rs +++ b/storage_controller/src/http.rs @@ -456,6 +456,14 @@ async fn handle_node_drop(req: Request) -> Result, ApiError json_response(StatusCode::OK, state.service.node_drop(node_id).await?) } +async fn handle_node_delete(req: Request) -> Result, ApiError> { + check_permissions(&req, Scope::Admin)?; + + let state = get_state(&req); + let node_id: NodeId = parse_request_param(&req, "node_id")?; + json_response(StatusCode::OK, state.service.node_delete(node_id).await?) +} + async fn handle_node_configure(mut req: Request) -> Result, ApiError> { check_permissions(&req, Scope::Admin)?; @@ -878,6 +886,9 @@ pub fn make_router( .post("/control/v1/node", |r| { named_request_span(r, handle_node_register, RequestName("control_v1_node")) }) + .delete("/control/v1/node/:node_id", |r| { + named_request_span(r, handle_node_delete, RequestName("control_v1_node_delete")) + }) .get("/control/v1/node", |r| { named_request_span(r, handle_node_list, RequestName("control_v1_node")) }) diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index aada1939eeea..b6e2b5319132 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -2,6 +2,7 @@ use std::{ borrow::Cow, cmp::Ordering, collections::{BTreeMap, HashMap, HashSet}, + ops::Deref, path::PathBuf, str::FromStr, sync::Arc, @@ -115,12 +116,14 @@ enum TenantOperations { SecondaryDownload, TimelineCreate, TimelineDelete, + AttachHook, } #[derive(Clone, strum_macros::Display)] enum NodeOperations { Register, Configure, + Delete, } pub const RECONCILER_CONCURRENCY_DEFAULT: usize = 128; @@ -845,9 +848,10 @@ impl Service { tenant_id=%result.tenant_shard_id.tenant_id, shard_id=%result.tenant_shard_id.shard_slug(), sequence=%result.sequence ))] - fn process_result(&self, result: ReconcileResult) { + fn process_result(&self, mut result: ReconcileResult) { let mut locked = self.inner.write().unwrap(); - let Some(tenant) = locked.tenants.get_mut(&result.tenant_shard_id) else { + let (nodes, tenants, _scheduler) = locked.parts_mut(); + let Some(tenant) = tenants.get_mut(&result.tenant_shard_id) else { // A reconciliation result might race with removing a tenant: drop results for // tenants that aren't in our map. return; @@ -864,6 +868,13 @@ impl Service { // Let the TenantShard know it is idle. tenant.reconcile_complete(result.sequence); + // In case a node was deleted while this reconcile is in flight, filter it out of the update we will + // make to the tenant + result + .observed + .locations + .retain(|node_id, _loc| nodes.contains_key(node_id)); + match result.result { Ok(()) => { for (node_id, loc) in &result.observed.locations { @@ -873,6 +884,7 @@ impl Service { tracing::info!("Setting observed location {} to None", node_id,) } } + tenant.observed = result.observed; tenant.waiter.advance(result.sequence); } @@ -1109,8 +1121,16 @@ impl Service { // We will populate intent properly later in [`Self::startup_reconcile`], initially populate // it with what we can infer: the node for which a generation was most recently issued. let mut intent = IntentState::new(); - if let Some(generation_pageserver) = tsp.generation_pageserver { - intent.set_attached(&mut scheduler, Some(NodeId(generation_pageserver as u64))); + if let Some(generation_pageserver) = tsp.generation_pageserver.map(|n| NodeId(n as u64)) + { + if nodes.contains_key(&generation_pageserver) { + intent.set_attached(&mut scheduler, Some(generation_pageserver)); + } else { + // If a node was removed before being completely drained, it is legal for it to leave behind a `generation_pageserver` referring + // to a non-existent node, because node deletion doesn't block on completing the reconciliations that will issue new generations + // on different pageservers. + tracing::warn!("Tenant shard {tenant_shard_id} references non-existent node {generation_pageserver} in database, will be rescheduled"); + } } let new_tenant = TenantShard::from_persistent(tsp, intent)?; @@ -1237,7 +1257,7 @@ impl Service { let _tenant_lock = trace_exclusive_lock( &self.tenant_op_locks, attach_req.tenant_shard_id.tenant_id, - TenantOperations::ShardSplit, + TenantOperations::AttachHook, ) .await; @@ -4210,8 +4230,6 @@ impl Service { /// This is for debug/support only: we simply drop all state for a tenant, without /// detaching or deleting it on pageservers. We do not try and re-schedule any /// tenants that were on this node. - /// - /// TODO: proper node deletion API that unhooks things more gracefully pub(crate) async fn node_drop(&self, node_id: NodeId) -> Result<(), ApiError> { self.persistence.delete_node(node_id).await?; @@ -4219,6 +4237,7 @@ impl Service { for shard in locked.tenants.values_mut() { shard.deref_node(node_id); + shard.observed.locations.remove(&node_id); } let mut nodes = (*locked.nodes).clone(); @@ -4230,6 +4249,94 @@ impl Service { Ok(()) } + /// If a node has any work on it, it will be rescheduled: this is "clean" in the sense + /// that we don't leave any bad state behind in the storage controller, but unclean + /// in the sense that we are not carefully draining the node. + pub(crate) async fn node_delete(&self, node_id: NodeId) -> Result<(), ApiError> { + let _node_lock = + trace_exclusive_lock(&self.node_op_locks, node_id, NodeOperations::Delete).await; + + // 1. Atomically update in-memory state: + // - set the scheduling state to Pause to make subsequent scheduling ops skip it + // - update shards' intents to exclude the node, and reschedule any shards whose intents we modified. + // - drop the node from the main nodes map, so that when running reconciles complete they do not + // re-insert references to this node into the ObservedState of shards + // - drop the node from the scheduler + { + let mut locked = self.inner.write().unwrap(); + let (nodes, tenants, scheduler) = locked.parts_mut(); + + { + let mut nodes_mut = (*nodes).deref().clone(); + match nodes_mut.get_mut(&node_id) { + Some(node) => { + // We do not bother setting this in the database, because we're about to delete the row anyway, and + // if we crash it would not be desirable to leave the node paused after a restart. + node.set_scheduling(NodeSchedulingPolicy::Pause); + } + None => { + tracing::info!( + "Node not found: presuming this is a retry and returning success" + ); + return Ok(()); + } + } + + *nodes = Arc::new(nodes_mut); + } + + for (tenant_shard_id, shard) in tenants { + if shard.deref_node(node_id) { + // FIXME: we need to build a ScheduleContext that reflects this shard's peers, otherwise + // it won't properly do anti-affinity. + let mut schedule_context = ScheduleContext::default(); + + if let Err(e) = shard.schedule(scheduler, &mut schedule_context) { + // TODO: implement force flag to remove a node even if we can't reschedule + // a tenant + tracing::error!("Refusing to delete node, shard {tenant_shard_id} can't be rescheduled: {e}"); + return Err(e.into()); + } else { + tracing::info!( + "Rescheduled shard {tenant_shard_id} away from node during deletion" + ) + } + + self.maybe_reconcile_shard(shard, nodes); + } + + // Here we remove an existing observed location for the node we're removing, and it will + // not be re-added by a reconciler's completion because we filter out removed nodes in + // process_result. + // + // Note that we update the shard's observed state _after_ calling maybe_reconcile_shard: that + // means any reconciles we spawned will know about the node we're deleting, enabling them + // to do live migrations if it's still online. + shard.observed.locations.remove(&node_id); + } + + scheduler.node_remove(node_id); + + { + let mut nodes_mut = (**nodes).clone(); + nodes_mut.remove(&node_id); + *nodes = Arc::new(nodes_mut); + } + } + + // Note: some `generation_pageserver` columns on tenant shards in the database may still refer to + // the removed node, as this column means "The pageserver to which this generation was issued", and + // their generations won't get updated until the reconcilers moving them away from this node complete. + // That is safe because in Service::spawn we only use generation_pageserver if it refers to a node + // that exists. + + // 2. Actually delete the node from the database and from in-memory state + tracing::info!("Deleting node from database"); + self.persistence.delete_node(node_id).await?; + + Ok(()) + } + pub(crate) async fn node_list(&self) -> Result, ApiError> { let nodes = { self.inner diff --git a/storage_controller/src/tenant_shard.rs b/storage_controller/src/tenant_shard.rs index 2ddab58aafbb..2574dc297ae9 100644 --- a/storage_controller/src/tenant_shard.rs +++ b/storage_controller/src/tenant_shard.rs @@ -1229,18 +1229,27 @@ impl TenantShard { } } - // If we had any state at all referring to this node ID, drop it. Does not - // attempt to reschedule. - pub(crate) fn deref_node(&mut self, node_id: NodeId) { + /// If we had any state at all referring to this node ID, drop it. Does not + /// attempt to reschedule. + /// + /// Returns true if we modified the node's intent state. + pub(crate) fn deref_node(&mut self, node_id: NodeId) -> bool { + let mut intent_modified = false; + + // Drop if this node was our attached intent if self.intent.attached == Some(node_id) { self.intent.attached = None; + intent_modified = true; } + // Drop from the list of secondaries, and check if we modified it + let had_secondaries = self.intent.secondary.len(); self.intent.secondary.retain(|n| n != &node_id); - - self.observed.locations.remove(&node_id); + intent_modified |= self.intent.secondary.len() != had_secondaries; debug_assert!(!self.intent.all_pageservers().contains(&node_id)); + + intent_modified } pub(crate) fn set_scheduling_policy(&mut self, p: ShardSchedulingPolicy) { diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 5ca31644a910..463e4a3b012e 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2287,6 +2287,14 @@ def node_register(self, node: NeonPageserver): headers=self.headers(TokenScope.ADMIN), ) + def node_delete(self, node_id): + log.info(f"node_delete({node_id})") + self.request( + "DELETE", + f"{self.env.storage_controller_api}/control/v1/node/{node_id}", + headers=self.headers(TokenScope.ADMIN), + ) + def node_drain(self, node_id): log.info(f"node_drain({node_id})") self.request( diff --git a/test_runner/regress/test_compatibility.py b/test_runner/regress/test_compatibility.py index 65649e0c0a84..1e5e320e0eff 100644 --- a/test_runner/regress/test_compatibility.py +++ b/test_runner/regress/test_compatibility.py @@ -93,6 +93,29 @@ ) +def fixup_storage_controller(env: NeonEnv): + """ + After importing a repo_dir, we need to massage the storage controller's state a bit: it will have + initially started up with no nodes, but some tenants, and thereby those tenants won't be scheduled + anywhere. + + After NeonEnv.start() is done (i.e. nodes are started + registered), call this function to get + the storage controller into a good state. + + This function should go away once compat tests carry the controller database in their snapshots, so + that the controller properly remembers nodes between creating + restoring the snapshot. + """ + env.storage_controller.allowed_errors.extend( + [ + ".*Tenant shard .+ references non-existent node.*", + ".*Failed to schedule tenant .+ at startup.*", + ] + ) + env.storage_controller.stop() + env.storage_controller.start() + env.storage_controller.reconcile_until_idle() + + @pytest.mark.xdist_group("compatibility") @pytest.mark.order(before="test_forward_compatibility") def test_create_snapshot( @@ -175,6 +198,7 @@ def test_backward_compatibility( neon_env_builder.num_safekeepers = 3 env = neon_env_builder.from_repo_dir(compatibility_snapshot_dir / "repo") neon_env_builder.start() + fixup_storage_controller(env) check_neon_works( env, @@ -263,6 +287,7 @@ def test_forward_compatibility( assert not env.pageserver.log_contains("git-env:" + prev_pageserver_version) neon_env_builder.start() + fixup_storage_controller(env) # ensure the specified pageserver is running assert env.pageserver.log_contains("git-env:" + prev_pageserver_version) diff --git a/test_runner/regress/test_storage_controller.py b/test_runner/regress/test_storage_controller.py index d37f7aae3dfd..741f16685e68 100644 --- a/test_runner/regress/test_storage_controller.py +++ b/test_runner/regress/test_storage_controller.py @@ -1611,3 +1611,91 @@ def test_background_operation_cancellation(neon_env_builder: NeonEnvBuilder): env.storage_controller.cancel_node_drain(ps_id_to_drain) env.storage_controller.poll_node_status(ps_id_to_drain, "Active", max_attempts=6, backoff=2) + + +@pytest.mark.parametrize("while_offline", [True, False]) +def test_storage_controller_node_deletion( + neon_env_builder: NeonEnvBuilder, + compute_reconfigure_listener: ComputeReconfigure, + while_offline: bool, +): + """ + Test that deleting a node works & properly reschedules everything that was on the node. + """ + neon_env_builder.num_pageservers = 3 + env = neon_env_builder.init_configs() + env.start() + + tenant_count = 10 + shard_count_per_tenant = 8 + tenant_ids = [] + for _ in range(0, tenant_count): + tid = TenantId.generate() + tenant_ids.append(tid) + env.neon_cli.create_tenant( + tid, placement_policy='{"Attached":1}', shard_count=shard_count_per_tenant + ) + + victim = env.pageservers[-1] + + # The procedure a human would follow is: + # 1. Mark pageserver scheduling=pause + # 2. Mark pageserver availability=offline to trigger migrations away from it + # 3. Wait for attachments to all move elsewhere + # 4. Call deletion API + # 5. Stop the node. + + env.storage_controller.node_configure(victim.id, {"scheduling": "Pause"}) + + if while_offline: + victim.stop(immediate=True) + env.storage_controller.node_configure(victim.id, {"availability": "Offline"}) + + def assert_shards_migrated(): + counts = get_node_shard_counts(env, tenant_ids) + elsewhere = sum(v for (k, v) in counts.items() if k != victim.id) + log.info(f"Shards on nodes other than on victim: {elsewhere}") + assert elsewhere == tenant_count * shard_count_per_tenant + + wait_until(30, 1, assert_shards_migrated) + + log.info(f"Deleting pageserver {victim.id}") + env.storage_controller.node_delete(victim.id) + + if not while_offline: + + def assert_victim_evacuated(): + counts = get_node_shard_counts(env, tenant_ids) + count = counts[victim.id] + log.info(f"Shards on node {victim.id}: {count}") + assert count == 0 + + wait_until(30, 1, assert_victim_evacuated) + + # The node should be gone from the list API + assert victim.id not in [n["id"] for n in env.storage_controller.node_list()] + + # No tenants should refer to the node in their intent + for tenant_id in tenant_ids: + describe = env.storage_controller.tenant_describe(tenant_id) + for shard in describe["shards"]: + assert shard["node_attached"] != victim.id + assert victim.id not in shard["node_secondary"] + + # Reconciles running during deletion should all complete + # FIXME: this currently doesn't work because the deletion schedules shards without a proper ScheduleContext, resulting + # in states that background_reconcile wants to optimize, but can't proceed with migrations yet because this is a short3 + # test that hasn't uploaded any heatmaps for secondaries. + # In the interim, just do a reconcile_all to enable the consistency check. + # env.storage_controller.reconcile_until_idle() + env.storage_controller.reconcile_all() + + # Controller should pass its own consistency checks + env.storage_controller.consistency_check() + + # The node should stay gone across a restart + env.storage_controller.stop() + env.storage_controller.start() + assert victim.id not in [n["id"] for n in env.storage_controller.node_list()] + env.storage_controller.reconcile_all() # FIXME: workaround for optimizations happening on startup, see FIXME above. + env.storage_controller.consistency_check() From cd29156927474219b92d2e5d8fda5f045a58d7af Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Thu, 11 Jul 2024 19:14:49 +0300 Subject: [PATCH 093/194] Fix memory context of NeonWALReader allocation. Allocating it in short living context is wrong because it is reused during backend lifetime. --- pgxn/neon/neon_walreader.c | 9 +++++---- test_runner/regress/test_logical_replication.py | 6 ++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pgxn/neon/neon_walreader.c b/pgxn/neon/neon_walreader.c index 60eb8e1fc985..0f76514b866d 100644 --- a/pgxn/neon/neon_walreader.c +++ b/pgxn/neon/neon_walreader.c @@ -109,11 +109,12 @@ NeonWALReaderAllocate(int wal_segment_size, XLogRecPtr available_lsn, char *log_ { NeonWALReader *reader; + /* + * Note: we allocate in TopMemoryContext, reusing the reader for all process + * reads. + */ reader = (NeonWALReader *) - palloc_extended(sizeof(NeonWALReader), - MCXT_ALLOC_NO_OOM | MCXT_ALLOC_ZERO); - if (!reader) - return NULL; + MemoryContextAllocZero(TopMemoryContext, sizeof(NeonWALReader)); reader->available_lsn = available_lsn; reader->seg.ws_file = -1; diff --git a/test_runner/regress/test_logical_replication.py b/test_runner/regress/test_logical_replication.py index 41283e4d2ca0..66afe9ddfdde 100644 --- a/test_runner/regress/test_logical_replication.py +++ b/test_runner/regress/test_logical_replication.py @@ -247,6 +247,12 @@ def test_ondemand_wal_download_in_replication_slot_funcs(neon_env_builder: NeonE cur.execute( "SELECT * FROM pg_logical_slot_peek_binary_changes('slotty_mcslotface', NULL, NULL, 'include-xids', '0')" ) + # do the peek second time: we've had a bug using wrong memory context + # for NeonWALReader leading to the crash in this case. + log.info("peek_changes again") + cur.execute( + "SELECT * FROM pg_logical_slot_peek_binary_changes('slotty_mcslotface', NULL, NULL, 'include-xids', '0')" + ) # Tests that walsender correctly blocks until WAL is downloaded from safekeepers From 38b4ed297eeb50ad2d97e58b8328d7f8c46fdb6f Mon Sep 17 00:00:00 2001 From: Alex Chi Z Date: Thu, 11 Jul 2024 14:28:16 -0400 Subject: [PATCH 094/194] feat(pageserver): rewrite streaming vectored read planner (#8242) Rewrite streaming vectored read planner to be a separate struct. The API is designed to produce batches around `max_read_size` instead of exactly less than that so that `handle_XX` returns one batch a time. --------- Signed-off-by: Alex Chi Z --- .../src/tenant/storage_layer/delta_layer.rs | 10 +- .../src/tenant/storage_layer/image_layer.rs | 7 +- pageserver/src/tenant/vectored_blob_io.rs | 269 +++++++++++++----- 3 files changed, 216 insertions(+), 70 deletions(-) diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index dfd0196c87e9..2d36ac744277 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -1321,7 +1321,7 @@ impl DeltaLayerInner { offsets.start.pos(), offsets.end.pos(), meta, - Some(max_read_size), + max_read_size, )) } } else { @@ -1615,13 +1615,17 @@ impl<'a> DeltaLayerIterator<'a> { let lsn = DeltaKey::extract_lsn_from_buf(&raw_key); let blob_ref = BlobRef(value); let offset = blob_ref.pos(); - if let Some(batch_plan) = self.planner.handle(key, lsn, offset, BlobFlag::None) { + if let Some(batch_plan) = self.planner.handle(key, lsn, offset) { break batch_plan; } } else { self.is_end = true; let data_end_offset = self.delta_layer.index_start_offset(); - break self.planner.handle_range_end(data_end_offset); + if let Some(item) = self.planner.handle_range_end(data_end_offset) { + break item; + } else { + return Ok(()); // TODO: test empty iterator + } } }; let vectored_blob_reader = VectoredBlobReader::new(&self.delta_layer.file); diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index 1e03e1a58c92..1440c0db84a4 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -994,14 +994,17 @@ impl<'a> ImageLayerIterator<'a> { Key::from_slice(&raw_key[..KEY_SIZE]), self.image_layer.lsn, offset, - BlobFlag::None, ) { break batch_plan; } } else { self.is_end = true; let payload_end = self.image_layer.index_start_blk as u64 * PAGE_SZ as u64; - break self.planner.handle_range_end(payload_end); + if let Some(item) = self.planner.handle_range_end(payload_end) { + break item; + } else { + return Ok(()); // TODO: a test case on empty iterator + } } }; let vectored_blob_reader = VectoredBlobReader::new(&self.image_layer.file); diff --git a/pageserver/src/tenant/vectored_blob_io.rs b/pageserver/src/tenant/vectored_blob_io.rs index 7ad8446e0411..1b470034db21 100644 --- a/pageserver/src/tenant/vectored_blob_io.rs +++ b/pageserver/src/tenant/vectored_blob_io.rs @@ -68,7 +68,7 @@ impl VectoredRead { } } -#[derive(Eq, PartialEq)] +#[derive(Eq, PartialEq, Debug)] pub(crate) enum VectoredReadExtended { Yes, No, @@ -91,7 +91,7 @@ impl VectoredReadBuilder { start_offset: u64, end_offset: u64, meta: BlobMeta, - max_read_size: Option, + max_read_size: usize, ) -> Self { let mut blobs_at = VecMap::default(); blobs_at @@ -102,10 +102,9 @@ impl VectoredReadBuilder { start: start_offset, end: end_offset, blobs_at, - max_read_size, + max_read_size: Some(max_read_size), } } - /// Attempt to extend the current read with a new blob if the start /// offset matches with the current end of the vectored read /// and the resuting size is below the max read size @@ -164,7 +163,7 @@ pub struct VectoredReadPlanner { // Arguments for previous blob passed into [`VectoredReadPlanner::handle`] prev: Option<(Key, Lsn, u64, BlobFlag)>, - max_read_size: Option, + max_read_size: usize, } impl VectoredReadPlanner { @@ -172,20 +171,7 @@ impl VectoredReadPlanner { Self { blobs: BTreeMap::new(), prev: None, - max_read_size: Some(max_read_size), - } - } - - /// This function should *only* be used if the caller has a way to control the limit. e.g., in [`StreamingVectoredReadPlanner`], - /// it uses the vectored read planner to avoid duplicated logic on handling blob start/end, while expecting the vectored - /// read planner to give a single read to a continuous range of bytes in the image layer. Therefore, it does not need the - /// code path to split reads into chunks of `max_read_size`, and controls the read size itself. - #[cfg(test)] - pub(crate) fn new_caller_controlled_max_limit() -> Self { - Self { - blobs: BTreeMap::new(), - prev: None, - max_read_size: None, + max_read_size, } } @@ -376,17 +362,18 @@ impl<'a> VectoredBlobReader<'a> { } /// Read planner used in [`crate::tenant::storage_layer::image_layer::ImageLayerIterator`]. It provides a streaming API for -/// getting read blobs. It returns a batch when `handle` gets called and when the current key would exceed the read_size and -/// max_cnt constraints. Underlying it uses [`VectoredReadPlanner`]. +/// getting read blobs. It returns a batch when `handle` gets called and when the current key would just exceed the read_size and +/// max_cnt constraints. #[cfg(test)] pub struct StreamingVectoredReadPlanner { - planner: VectoredReadPlanner, - /// Max read size per batch + read_builder: Option, + // Arguments for previous blob passed into [`StreamingVectoredReadPlanner::handle`] + prev: Option<(Key, Lsn, u64)>, + /// Max read size per batch. This is not a strict limit. If there are [0, 100) and [100, 200), while the `max_read_size` is 150, + /// we will produce a single batch instead of split them. max_read_size: u64, /// Max item count per batch max_cnt: usize, - /// The first offset of this batch - this_batch_first_offset: Option, /// Size of the current batch cnt: usize, } @@ -397,63 +384,89 @@ impl StreamingVectoredReadPlanner { assert!(max_cnt > 0); assert!(max_read_size > 0); Self { - // We want to have exactly one read syscall (plus several others for index lookup) for each `next_batch` call. - // Therefore, we enforce `self.max_read_size` by ourselves instead of using the VectoredReadPlanner's capability, - // to avoid splitting into two I/Os. - planner: VectoredReadPlanner::new_caller_controlled_max_limit(), + read_builder: None, + prev: None, max_cnt, max_read_size, - this_batch_first_offset: None, cnt: 0, } } - fn emit(&mut self, this_batch_first_offset: u64) -> VectoredRead { - let planner = std::mem::replace( - &mut self.planner, - VectoredReadPlanner::new_caller_controlled_max_limit(), - ); - self.this_batch_first_offset = Some(this_batch_first_offset); - self.cnt = 1; - let mut batch = planner.finish(); - assert_eq!(batch.len(), 1, "should have exactly one read batch"); - batch.pop().unwrap() + pub fn handle(&mut self, key: Key, lsn: Lsn, offset: u64) -> Option { + // Implementation note: internally lag behind by one blob such that + // we have a start and end offset when initialising [`VectoredRead`] + let (prev_key, prev_lsn, prev_offset) = match self.prev { + None => { + self.prev = Some((key, lsn, offset)); + return None; + } + Some(prev) => prev, + }; + + let res = self.add_blob(prev_key, prev_lsn, prev_offset, offset, false); + + self.prev = Some((key, lsn, offset)); + + res + } + + pub fn handle_range_end(&mut self, offset: u64) -> Option { + let res = if let Some((prev_key, prev_lsn, prev_offset)) = self.prev { + self.add_blob(prev_key, prev_lsn, prev_offset, offset, true) + } else { + None + }; + + self.prev = None; + + res } - pub fn handle( + fn add_blob( &mut self, key: Key, lsn: Lsn, - offset: u64, - flag: BlobFlag, + start_offset: u64, + end_offset: u64, + is_last_blob_in_read: bool, ) -> Option { - if let Some(begin_offset) = self.this_batch_first_offset { - // Each batch will have at least one item b/c `self.this_batch_first_offset` is set - // after one item gets processed - if offset - begin_offset > self.max_read_size { - self.planner.handle_range_end(offset); // End the current batch with the offset - let batch = self.emit(offset); // Produce a batch - self.planner.handle(key, lsn, offset, flag); // Add this key to the next batch - return Some(batch); + match &mut self.read_builder { + Some(read_builder) => { + let extended = read_builder.extend(start_offset, end_offset, BlobMeta { key, lsn }); + assert_eq!(extended, VectoredReadExtended::Yes); + } + None => { + self.read_builder = { + let mut blobs_at = VecMap::default(); + blobs_at + .append(start_offset, BlobMeta { key, lsn }) + .expect("First insertion always succeeds"); + + Some(VectoredReadBuilder { + start: start_offset, + end: end_offset, + blobs_at, + max_read_size: None, + }) + }; } - } else { - self.this_batch_first_offset = Some(offset) - } - if self.cnt >= self.max_cnt { - self.planner.handle_range_end(offset); // End the current batch with the offset - let batch = self.emit(offset); // Produce a batch - self.planner.handle(key, lsn, offset, flag); // Add this key to the next batch - return Some(batch); } - self.planner.handle(key, lsn, offset, flag); // Add this key to the current batch + let read_builder = self.read_builder.as_mut().unwrap(); self.cnt += 1; + if is_last_blob_in_read + || read_builder.size() >= self.max_read_size as usize + || self.cnt >= self.max_cnt + { + let prev_read_builder = self.read_builder.take(); + self.cnt = 0; + + // `current_read_builder` is None in the first iteration + if let Some(read_builder) = prev_read_builder { + return Some(read_builder.build()); + } + } None } - - pub fn handle_range_end(&mut self, offset: u64) -> VectoredRead { - self.planner.handle_range_end(offset); - self.emit(offset) - } } #[cfg(test)] @@ -509,8 +522,11 @@ mod tests { planner.handle_range_end(652 * 1024); let reads = planner.finish(); + assert_eq!(reads.len(), 6); + // TODO: could remove zero reads to produce 5 reads here + for (idx, read) in reads.iter().enumerate() { validate_read(read, ranges[idx]); } @@ -548,4 +564,127 @@ mod tests { validate_read(read, ranges[idx]); } } + + #[test] + fn streaming_planner_max_read_size_test() { + let max_read_size = 128 * 1024; + let key = Key::MIN; + let lsn = Lsn(0); + + let blob_descriptions = vec![ + (key, lsn, 0, BlobFlag::None), + (key, lsn, 32 * 1024, BlobFlag::None), + (key, lsn, 96 * 1024, BlobFlag::None), + (key, lsn, 128 * 1024, BlobFlag::None), + (key, lsn, 198 * 1024, BlobFlag::None), + (key, lsn, 268 * 1024, BlobFlag::None), + (key, lsn, 396 * 1024, BlobFlag::None), + (key, lsn, 652 * 1024, BlobFlag::None), + ]; + + let ranges = [ + &blob_descriptions[0..3], + &blob_descriptions[3..5], + &blob_descriptions[5..6], + &blob_descriptions[6..7], + &blob_descriptions[7..], + ]; + + let mut planner = StreamingVectoredReadPlanner::new(max_read_size, 1000); + let mut reads = Vec::new(); + for (key, lsn, offset, _) in blob_descriptions.clone() { + reads.extend(planner.handle(key, lsn, offset)); + } + reads.extend(planner.handle_range_end(652 * 1024)); + + assert_eq!(reads.len(), ranges.len()); + + for (idx, read) in reads.iter().enumerate() { + validate_read(read, ranges[idx]); + } + } + + #[test] + fn streaming_planner_max_cnt_test() { + let max_read_size = 1024 * 1024; + let key = Key::MIN; + let lsn = Lsn(0); + + let blob_descriptions = vec![ + (key, lsn, 0, BlobFlag::None), + (key, lsn, 32 * 1024, BlobFlag::None), + (key, lsn, 96 * 1024, BlobFlag::None), + (key, lsn, 128 * 1024, BlobFlag::None), + (key, lsn, 198 * 1024, BlobFlag::None), + (key, lsn, 268 * 1024, BlobFlag::None), + (key, lsn, 396 * 1024, BlobFlag::None), + (key, lsn, 652 * 1024, BlobFlag::None), + ]; + + let ranges = [ + &blob_descriptions[0..2], + &blob_descriptions[2..4], + &blob_descriptions[4..6], + &blob_descriptions[6..], + ]; + + let mut planner = StreamingVectoredReadPlanner::new(max_read_size, 2); + let mut reads = Vec::new(); + for (key, lsn, offset, _) in blob_descriptions.clone() { + reads.extend(planner.handle(key, lsn, offset)); + } + reads.extend(planner.handle_range_end(652 * 1024)); + + assert_eq!(reads.len(), ranges.len()); + + for (idx, read) in reads.iter().enumerate() { + validate_read(read, ranges[idx]); + } + } + + #[test] + fn streaming_planner_edge_test() { + let max_read_size = 1024 * 1024; + let key = Key::MIN; + let lsn = Lsn(0); + { + let mut planner = StreamingVectoredReadPlanner::new(max_read_size, 1); + let mut reads = Vec::new(); + reads.extend(planner.handle_range_end(652 * 1024)); + assert!(reads.is_empty()); + } + { + let mut planner = StreamingVectoredReadPlanner::new(max_read_size, 1); + let mut reads = Vec::new(); + reads.extend(planner.handle(key, lsn, 0)); + reads.extend(planner.handle_range_end(652 * 1024)); + assert_eq!(reads.len(), 1); + validate_read(&reads[0], &[(key, lsn, 0, BlobFlag::None)]); + } + { + let mut planner = StreamingVectoredReadPlanner::new(max_read_size, 1); + let mut reads = Vec::new(); + reads.extend(planner.handle(key, lsn, 0)); + reads.extend(planner.handle(key, lsn, 128 * 1024)); + reads.extend(planner.handle_range_end(652 * 1024)); + assert_eq!(reads.len(), 2); + validate_read(&reads[0], &[(key, lsn, 0, BlobFlag::None)]); + validate_read(&reads[1], &[(key, lsn, 128 * 1024, BlobFlag::None)]); + } + { + let mut planner = StreamingVectoredReadPlanner::new(max_read_size, 2); + let mut reads = Vec::new(); + reads.extend(planner.handle(key, lsn, 0)); + reads.extend(planner.handle(key, lsn, 128 * 1024)); + reads.extend(planner.handle_range_end(652 * 1024)); + assert_eq!(reads.len(), 1); + validate_read( + &reads[0], + &[ + (key, lsn, 0, BlobFlag::None), + (key, lsn, 128 * 1024, BlobFlag::None), + ], + ); + } + } } From 4a87bac036f7d21545183dd1894df00e960179ad Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 11 Jul 2024 22:03:35 +0300 Subject: [PATCH 095/194] test: limit `test_layer_download_timeouted` to MOCK_S3 (#8331) Requests against REAL_S3 on CI can consistently take longer than 1s; testing the short timeouts against it made no sense in hindsight, as MOCK_S3 works just as well. evidence: https://neon-github-public-dev.s3.amazonaws.com/reports/pr-8229/9857994025/index.html#suites/b97efae3a617afb71cb8142f5afa5224/6828a50921660a32 --- test_runner/regress/test_ondemand_download.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test_runner/regress/test_ondemand_download.py b/test_runner/regress/test_ondemand_download.py index 4a25dfd874b4..c8249bb2cec6 100644 --- a/test_runner/regress/test_ondemand_download.py +++ b/test_runner/regress/test_ondemand_download.py @@ -764,7 +764,9 @@ def test_layer_download_timeouted(neon_env_builder: NeonEnvBuilder): """ Pause using a pausable_failpoint longer than the client timeout to simulate the timeout happening. """ - neon_env_builder.enable_pageserver_remote_storage(s3_storage()) + # running this test is not reliable against REAL_S3, because operations can + # take longer than 1s we want to use as a timeout + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.MOCK_S3) assert isinstance(neon_env_builder.pageserver_remote_storage, S3Storage) neon_env_builder.pageserver_remote_storage.custom_timeout = "1s" From 82b9a44ab48a1658fce7942e60dc61f68bd29945 Mon Sep 17 00:00:00 2001 From: Sasha Krassovsky Date: Thu, 11 Jul 2024 13:29:35 -0700 Subject: [PATCH 096/194] Grant execute on snapshot functions to neon_superuser (#8346) ## Problem I need `neon_superuser` to be allowed to create snapshots for replication tests ## Summary of changes Adds a migration that grants these functions to neon_superuser --- ...nt_snapshot_synchronization_funcs_to_neon_superuser.sql | 7 +++++++ compute_tools/src/spec.rs | 3 +++ test_runner/regress/test_migrations.py | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 compute_tools/src/migrations/0009-grant_snapshot_synchronization_funcs_to_neon_superuser.sql diff --git a/compute_tools/src/migrations/0009-grant_snapshot_synchronization_funcs_to_neon_superuser.sql b/compute_tools/src/migrations/0009-grant_snapshot_synchronization_funcs_to_neon_superuser.sql new file mode 100644 index 000000000000..28750e00dd49 --- /dev/null +++ b/compute_tools/src/migrations/0009-grant_snapshot_synchronization_funcs_to_neon_superuser.sql @@ -0,0 +1,7 @@ +DO $$ +BEGIN + IF (SELECT setting::numeric >= 160000 FROM pg_settings WHERE name = 'server_version_num') THEN + EXECUTE 'GRANT EXECUTE ON FUNCTION pg_export_snapshot TO neon_superuser'; + EXECUTE 'GRANT EXECUTE ON FUNCTION pg_log_standby_snapshot TO neon_superuser'; + END IF; +END $$; diff --git a/compute_tools/src/spec.rs b/compute_tools/src/spec.rs index 37090b08fd37..1d12b88c7ce3 100644 --- a/compute_tools/src/spec.rs +++ b/compute_tools/src/spec.rs @@ -790,6 +790,9 @@ pub fn handle_migrations(client: &mut Client) -> Result<()> { "./migrations/0007-grant_all_on_sequences_to_neon_superuser_with_grant_option.sql" ), include_str!("./migrations/0008-revoke_replication_for_previously_allowed_roles.sql"), + include_str!( + "./migrations/0009-grant_snapshot_synchronization_funcs_to_neon_superuser.sql" + ), ]; MigrationRunner::new(client, &migrations).run_migrations()?; diff --git a/test_runner/regress/test_migrations.py b/test_runner/regress/test_migrations.py index 5637f160cfcb..91bd3ea50caf 100644 --- a/test_runner/regress/test_migrations.py +++ b/test_runner/regress/test_migrations.py @@ -13,7 +13,7 @@ def test_migrations(neon_simple_env: NeonEnv): endpoint.wait_for_migrations() - num_migrations = 9 + num_migrations = 10 with endpoint.cursor() as cur: cur.execute("SELECT id FROM neon_migration.migration_id") From 30bbfde50d2e2e224cb8e4d9c0113b000111657b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Fri, 12 Jul 2024 01:43:44 +0200 Subject: [PATCH 097/194] Pass configured compression param to image generation (#8363) We need to pass on the configured compression param during image layer generation. This was an oversight of #8106, and the likely cause why #8288 didn't bring any interesting regressions. Part of https://github.com/neondatabase/neon/issues/5431 --- pageserver/src/tenant/storage_layer/image_layer.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index 1440c0db84a4..a88a1e642958 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -809,7 +809,11 @@ impl ImageLayerWriterInner { ctx: &RequestContext, ) -> anyhow::Result<()> { ensure!(self.key_range.contains(&key)); - let (_img, res) = self.blob_writer.write_blob(img, ctx).await; + let compression = self.conf.image_compression; + let (_img, res) = self + .blob_writer + .write_blob_maybe_compressed(img, ctx, compression) + .await; // TODO: re-use the buffer for `img` further upstack let off = res?; From 2e37aa3fe80bc9b60f90e12365da348ed108e4fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Fri, 12 Jul 2024 04:32:34 +0200 Subject: [PATCH 098/194] Implement decompression for vectored reads (#8302) Implement decompression of images for vectored reads. This doesn't implement support for still treating blobs as uncompressed with the bits we reserved for compression, as we have removed that functionality in #8300 anyways. Part of #5431 --- pageserver/src/tenant/blob_io.rs | 40 ++++--- pageserver/src/tenant/vectored_blob_io.rs | 127 +++++++++++++++++++--- 2 files changed, 139 insertions(+), 28 deletions(-) diff --git a/pageserver/src/tenant/blob_io.rs b/pageserver/src/tenant/blob_io.rs index e98ed66ef998..791eefebe989 100644 --- a/pageserver/src/tenant/blob_io.rs +++ b/pageserver/src/tenant/blob_io.rs @@ -137,14 +137,14 @@ impl<'a> BlockCursor<'a> { } /// Reserved bits for length and compression -const LEN_COMPRESSION_BIT_MASK: u8 = 0xf0; +pub(super) const LEN_COMPRESSION_BIT_MASK: u8 = 0xf0; /// The maximum size of blobs we support. The highest few bits /// are reserved for compression and other further uses. const MAX_SUPPORTED_LEN: usize = 0x0fff_ffff; -const BYTE_UNCOMPRESSED: u8 = 0x80; -const BYTE_ZSTD: u8 = BYTE_UNCOMPRESSED | 0x10; +pub(super) const BYTE_UNCOMPRESSED: u8 = 0x80; +pub(super) const BYTE_ZSTD: u8 = BYTE_UNCOMPRESSED | 0x10; /// A wrapper of `VirtualFile` that allows users to write blobs. /// @@ -390,51 +390,63 @@ impl BlobWriter { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; use crate::{context::DownloadBehavior, task_mgr::TaskKind, tenant::block_io::BlockReaderRef}; + use camino::Utf8PathBuf; + use camino_tempfile::Utf8TempDir; use rand::{Rng, SeedableRng}; async fn round_trip_test(blobs: &[Vec]) -> Result<(), Error> { round_trip_test_compressed::(blobs, false).await } - async fn round_trip_test_compressed( + pub(crate) async fn write_maybe_compressed( blobs: &[Vec], compression: bool, - ) -> Result<(), Error> { + ctx: &RequestContext, + ) -> Result<(Utf8TempDir, Utf8PathBuf, Vec), Error> { let temp_dir = camino_tempfile::tempdir()?; let pathbuf = temp_dir.path().join("file"); - let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error); // Write part (in block to drop the file) let mut offsets = Vec::new(); { - let file = VirtualFile::create(pathbuf.as_path(), &ctx).await?; + let file = VirtualFile::create(pathbuf.as_path(), ctx).await?; let mut wtr = BlobWriter::::new(file, 0); for blob in blobs.iter() { let (_, res) = if compression { wtr.write_blob_maybe_compressed( blob.clone(), - &ctx, + ctx, ImageCompressionAlgorithm::Zstd { level: Some(1) }, ) .await } else { - wtr.write_blob(blob.clone(), &ctx).await + wtr.write_blob(blob.clone(), ctx).await }; let offs = res?; offsets.push(offs); } // Write out one page worth of zeros so that we can // read again with read_blk - let (_, res) = wtr.write_blob(vec![0; PAGE_SZ], &ctx).await; + let (_, res) = wtr.write_blob(vec![0; PAGE_SZ], ctx).await; let offs = res?; println!("Writing final blob at offs={offs}"); - wtr.flush_buffer(&ctx).await?; + wtr.flush_buffer(ctx).await?; } + Ok((temp_dir, pathbuf, offsets)) + } + + async fn round_trip_test_compressed( + blobs: &[Vec], + compression: bool, + ) -> Result<(), Error> { + let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error); + let (_temp_dir, pathbuf, offsets) = + write_maybe_compressed::(blobs, compression, &ctx).await?; - let file = VirtualFile::open(pathbuf.as_path(), &ctx).await?; + let file = VirtualFile::open(pathbuf, &ctx).await?; let rdr = BlockReaderRef::VirtualFile(&file); let rdr = BlockCursor::new_with_compression(rdr, compression); for (idx, (blob, offset)) in blobs.iter().zip(offsets.iter()).enumerate() { @@ -447,7 +459,7 @@ mod tests { Ok(()) } - fn random_array(len: usize) -> Vec { + pub(crate) fn random_array(len: usize) -> Vec { let mut rng = rand::thread_rng(); (0..len).map(|_| rng.gen()).collect::<_>() } diff --git a/pageserver/src/tenant/vectored_blob_io.rs b/pageserver/src/tenant/vectored_blob_io.rs index 1b470034db21..cb81f1d76d6a 100644 --- a/pageserver/src/tenant/vectored_blob_io.rs +++ b/pageserver/src/tenant/vectored_blob_io.rs @@ -20,11 +20,13 @@ use std::num::NonZeroUsize; use bytes::BytesMut; use pageserver_api::key::Key; +use tokio::io::AsyncWriteExt; use tokio_epoll_uring::BoundedBuf; use utils::lsn::Lsn; use utils::vec_map::VecMap; use crate::context::RequestContext; +use crate::tenant::blob_io::{BYTE_UNCOMPRESSED, BYTE_ZSTD, LEN_COMPRESSION_BIT_MASK}; use crate::virtual_file::VirtualFile; #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -301,7 +303,7 @@ impl<'a> VectoredBlobReader<'a> { read.size(), buf.capacity() ); - let buf = self + let mut buf = self .file .read_exact_at(buf.slice(0..read.size()), read.start, ctx) .await? @@ -323,38 +325,68 @@ impl<'a> VectoredBlobReader<'a> { .chain(std::iter::once(None)), ); + // Some scratch space, put here for reusing the allocation + let mut decompressed_vec = Vec::new(); + for ((offset, meta), next) in pairs { let offset_in_buf = offset - start_offset; let first_len_byte = buf[offset_in_buf as usize]; - // Each blob is prefixed by a header containing it's size. + // Each blob is prefixed by a header containing its size and compression information. // Extract the size and skip that header to find the start of the data. // The size can be 1 or 4 bytes. The most significant bit is 0 in the // 1 byte case and 1 in the 4 byte case. - let (size_length, blob_size) = if first_len_byte < 0x80 { - (1, first_len_byte as u64) + let (size_length, blob_size, compression_bits) = if first_len_byte < 0x80 { + (1, first_len_byte as u64, BYTE_UNCOMPRESSED) } else { let mut blob_size_buf = [0u8; 4]; let offset_in_buf = offset_in_buf as usize; blob_size_buf.copy_from_slice(&buf[offset_in_buf..offset_in_buf + 4]); - blob_size_buf[0] &= 0x7f; - (4, u32::from_be_bytes(blob_size_buf) as u64) + blob_size_buf[0] &= !LEN_COMPRESSION_BIT_MASK; + + let compression_bits = first_len_byte & LEN_COMPRESSION_BIT_MASK; + ( + 4, + u32::from_be_bytes(blob_size_buf) as u64, + compression_bits, + ) }; - let start = offset_in_buf + size_length; - let end = match next { + let start_raw = offset_in_buf + size_length; + let end_raw = match next { Some((next_blob_start_offset, _)) => next_blob_start_offset - start_offset, - None => start + blob_size, + None => start_raw + blob_size, }; - - assert_eq!(end - start, blob_size); + assert_eq!(end_raw - start_raw, blob_size); + let (start, end); + if compression_bits == BYTE_UNCOMPRESSED { + start = start_raw as usize; + end = end_raw as usize; + } else if compression_bits == BYTE_ZSTD { + let mut decoder = + async_compression::tokio::write::ZstdDecoder::new(&mut decompressed_vec); + decoder + .write_all(&buf[start_raw as usize..end_raw as usize]) + .await?; + decoder.flush().await?; + start = buf.len(); + buf.extend_from_slice(&decompressed_vec); + end = buf.len(); + decompressed_vec.clear(); + } else { + let error = std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("invalid compression byte {compression_bits:x}"), + ); + return Err(error); + } metas.push(VectoredBlob { - start: start as usize, - end: end as usize, + start, + end, meta: *meta, - }) + }); } Ok(VectoredBlobsBuf { buf, blobs: metas }) @@ -471,6 +503,13 @@ impl StreamingVectoredReadPlanner { #[cfg(test)] mod tests { + use anyhow::Error; + + use crate::context::DownloadBehavior; + use crate::page_cache::PAGE_SZ; + use crate::task_mgr::TaskKind; + + use super::super::blob_io::tests::{random_array, write_maybe_compressed}; use super::*; fn validate_read(read: &VectoredRead, offset_range: &[(Key, Lsn, u64, BlobFlag)]) { @@ -687,4 +726,64 @@ mod tests { ); } } + + async fn round_trip_test_compressed(blobs: &[Vec], compression: bool) -> Result<(), Error> { + let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error); + let (_temp_dir, pathbuf, offsets) = + write_maybe_compressed::(blobs, compression, &ctx).await?; + + let file = VirtualFile::open(&pathbuf, &ctx).await?; + let file_len = std::fs::metadata(&pathbuf)?.len(); + + // Multiply by two (compressed data might need more space), and add a few bytes for the header + let reserved_bytes = blobs.iter().map(|bl| bl.len()).max().unwrap() * 2 + 16; + let mut buf = BytesMut::with_capacity(reserved_bytes); + + let vectored_blob_reader = VectoredBlobReader::new(&file); + let meta = BlobMeta { + key: Key::MIN, + lsn: Lsn(0), + }; + + for (idx, (blob, offset)) in blobs.iter().zip(offsets.iter()).enumerate() { + let end = offsets.get(idx + 1).unwrap_or(&file_len); + if idx + 1 == offsets.len() { + continue; + } + let read_builder = VectoredReadBuilder::new(*offset, *end, meta, 16 * 4096); + let read = read_builder.build(); + let result = vectored_blob_reader.read_blobs(&read, buf, &ctx).await?; + assert_eq!(result.blobs.len(), 1); + let read_blob = &result.blobs[0]; + let read_buf = &result.buf[read_blob.start..read_blob.end]; + assert_eq!(blob, read_buf, "mismatch for idx={idx} at offset={offset}"); + buf = result.buf; + } + Ok(()) + } + + #[tokio::test] + async fn test_really_big_array() -> Result<(), Error> { + let blobs = &[ + b"test".to_vec(), + random_array(10 * PAGE_SZ), + b"hello".to_vec(), + random_array(66 * PAGE_SZ), + vec![0xf3; 24 * PAGE_SZ], + b"foobar".to_vec(), + ]; + round_trip_test_compressed(blobs, false).await?; + round_trip_test_compressed(blobs, true).await?; + Ok(()) + } + + #[tokio::test] + async fn test_arrays_inc() -> Result<(), Error> { + let blobs = (0..PAGE_SZ / 8) + .map(|v| random_array(v * 16)) + .collect::>(); + round_trip_test_compressed(&blobs, false).await?; + round_trip_test_compressed(&blobs, true).await?; + Ok(()) + } } From 86d6ef305a6cfe5ab91febb876f6a9bae0dd982f Mon Sep 17 00:00:00 2001 From: Japin Li Date: Fri, 12 Jul 2024 17:56:06 +0800 Subject: [PATCH 099/194] Remove fs2 dependency (#8350) The fs2 dependency is not needed anymore after commit d42700280. --- Cargo.lock | 11 ----------- Cargo.toml | 1 - safekeeper/Cargo.toml | 1 - 3 files changed, 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b31ac69e6c50..bab0b4dd1f7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2028,16 +2028,6 @@ dependencies = [ "tokio-util", ] -[[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "fsevent-sys" version = "4.1.0" @@ -5179,7 +5169,6 @@ dependencies = [ "crc32c", "desim", "fail", - "fs2", "futures", "git-version", "hex", diff --git a/Cargo.toml b/Cargo.toml index 6bad8e3b20ce..670e3241d51d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,7 +83,6 @@ enumset = "1.0.12" fail = "0.5.0" fallible-iterator = "0.2" framed-websockets = { version = "0.1.0", git = "https://github.com/neondatabase/framed-websockets" } -fs2 = "0.4.3" futures = "0.3" futures-core = "0.3" futures-util = "0.3" diff --git a/safekeeper/Cargo.toml b/safekeeper/Cargo.toml index 9f32016fd97b..0fdb3147bfc3 100644 --- a/safekeeper/Cargo.toml +++ b/safekeeper/Cargo.toml @@ -23,7 +23,6 @@ clap = { workspace = true, features = ["derive"] } const_format.workspace = true crc32c.workspace = true fail.workspace = true -fs2.workspace = true git-version.workspace = true hex.workspace = true humantime.workspace = true From 0645ae318e49115055b5903791dcd9294ce67521 Mon Sep 17 00:00:00 2001 From: John Spray Date: Fri, 12 Jul 2024 12:04:02 +0100 Subject: [PATCH 100/194] pageserver: circuit breaker on compaction (#8359) ## Problem We already back off on compaction retries, but the impact of a failing compaction can be so great that backing off up to 300s isn't enough. The impact is consuming a lot of I/O+CPU in the case of image layer generation for large tenants, and potentially also leaking disk space. Compaction failures are extremely rare and almost always indicate a bug, frequently a bug that will not let compaction to proceed until it is fixed. Related: https://github.com/neondatabase/neon/issues/6738 ## Summary of changes - Introduce a CircuitBreaker type - Add a circuit breaker for compaction, with a policy that after 5 failures, compaction will not be attempted again for 24 hours. - Add metrics that we can alert on: any >0 value for `pageserver_circuit_breaker_broken_total` should generate an alert. - Add a test that checks this works as intended. Couple notes to reviewers: - Circuit breakers are intrinsically a defense-in-depth measure: this is not the solution to any underlying issues, it is just a general mitigation for "unknown unknowns" that might be encountered in future. - This PR isn't primarily about writing a perfect CircuitBreaker type: the one in this PR is meant to be just enough to mitigate issues in compaction, and make it easy to monitor/alert on these failures. We can refine this type in future as/when we want to use it elsewhere. --- libs/utils/src/circuit_breaker.rs | 114 +++++++++++++++++++++++++ libs/utils/src/lib.rs | 2 + pageserver/src/metrics.rs | 16 ++++ pageserver/src/tenant.rs | 36 +++++++- test_runner/regress/test_compaction.py | 63 ++++++++++++++ 5 files changed, 229 insertions(+), 2 deletions(-) create mode 100644 libs/utils/src/circuit_breaker.rs diff --git a/libs/utils/src/circuit_breaker.rs b/libs/utils/src/circuit_breaker.rs new file mode 100644 index 000000000000..720ea39d4f77 --- /dev/null +++ b/libs/utils/src/circuit_breaker.rs @@ -0,0 +1,114 @@ +use std::{ + fmt::Display, + time::{Duration, Instant}, +}; + +use metrics::IntCounter; + +/// Circuit breakers are for operations that are expensive and fallible: if they fail repeatedly, +/// we will stop attempting them for some period of time, to avoid denial-of-service from retries, and +/// to mitigate the log spam from repeated failures. +pub struct CircuitBreaker { + /// An identifier that enables us to log useful errors when a circuit is broken + name: String, + + /// Consecutive failures since last success + fail_count: usize, + + /// How many consecutive failures before we break the circuit + fail_threshold: usize, + + /// If circuit is broken, when was it broken? + broken_at: Option, + + /// If set, we will auto-reset the circuit this long after it was broken. If None, broken + /// circuits stay broken forever, or until success() is called. + reset_period: Option, + + /// If this is true, no actual circuit-breaking happens. This is for overriding a circuit breaker + /// to permit something to keep running even if it would otherwise have tripped it. + short_circuit: bool, +} + +impl CircuitBreaker { + pub fn new(name: String, fail_threshold: usize, reset_period: Option) -> Self { + Self { + name, + fail_count: 0, + fail_threshold, + broken_at: None, + reset_period, + short_circuit: false, + } + } + + /// Construct an unbreakable circuit breaker, for use in unit tests etc. + pub fn short_circuit() -> Self { + Self { + name: String::new(), + fail_threshold: 0, + fail_count: 0, + broken_at: None, + reset_period: None, + short_circuit: true, + } + } + + pub fn fail(&mut self, metric: &IntCounter, error: E) + where + E: Display, + { + if self.short_circuit { + return; + } + + self.fail_count += 1; + if self.broken_at.is_none() && self.fail_count >= self.fail_threshold { + self.break_circuit(metric, error); + } + } + + /// Call this after successfully executing an operation + pub fn success(&mut self, metric: &IntCounter) { + self.fail_count = 0; + if let Some(broken_at) = &self.broken_at { + tracing::info!(breaker=%self.name, "Circuit breaker failure ended (was broken for {})", + humantime::format_duration(broken_at.elapsed())); + self.broken_at = None; + metric.inc(); + } + } + + /// Call this before attempting an operation, and skip the operation if we are currently broken. + pub fn is_broken(&mut self) -> bool { + if self.short_circuit { + return false; + } + + if let Some(broken_at) = self.broken_at { + match self.reset_period { + Some(reset_period) if broken_at.elapsed() > reset_period => { + self.reset_circuit(); + false + } + _ => true, + } + } else { + false + } + } + + fn break_circuit(&mut self, metric: &IntCounter, error: E) + where + E: Display, + { + self.broken_at = Some(Instant::now()); + tracing::error!(breaker=%self.name, "Circuit breaker broken! Last error: {error}"); + metric.inc(); + } + + fn reset_circuit(&mut self) { + self.broken_at = None; + self.fail_count = 0; + } +} diff --git a/libs/utils/src/lib.rs b/libs/utils/src/lib.rs index 711e617801ea..9ad1752fb724 100644 --- a/libs/utils/src/lib.rs +++ b/libs/utils/src/lib.rs @@ -98,6 +98,8 @@ pub mod poison; pub mod toml_edit_ext; +pub mod circuit_breaker; + /// This is a shortcut to embed git sha into binaries and avoid copying the same build script to all packages /// /// we have several cases: diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index e67fa656d02e..9b3bb481b9ae 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -569,6 +569,22 @@ static VALID_LSN_LEASE_COUNT: Lazy = Lazy::new(|| { .expect("failed to define a metric") }); +pub(crate) static CIRCUIT_BREAKERS_BROKEN: Lazy = Lazy::new(|| { + register_int_counter!( + "pageserver_circuit_breaker_broken", + "How many times a circuit breaker has broken" + ) + .expect("failed to define a metric") +}); + +pub(crate) static CIRCUIT_BREAKERS_UNBROKEN: Lazy = Lazy::new(|| { + register_int_counter!( + "pageserver_circuit_breaker_unbroken", + "How many times a circuit breaker has been un-broken (recovered)" + ) + .expect("failed to define a metric") +}); + pub(crate) mod initial_logical_size { use metrics::{register_int_counter, register_int_counter_vec, IntCounter, IntCounterVec}; use once_cell::sync::Lazy; diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index bf23513527fb..6333fd3b6341 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -39,6 +39,7 @@ use tokio::task::JoinSet; use tokio_util::sync::CancellationToken; use tracing::*; use utils::backoff; +use utils::circuit_breaker::CircuitBreaker; use utils::completion; use utils::crashsafe::path_with_suffix_extension; use utils::failpoint_support; @@ -76,7 +77,8 @@ use crate::is_uninit_mark; use crate::l0_flush::L0FlushGlobalState; use crate::metrics::TENANT; use crate::metrics::{ - remove_tenant_metrics, BROKEN_TENANTS_SET, TENANT_STATE_METRIC, TENANT_SYNTHETIC_SIZE_METRIC, + remove_tenant_metrics, BROKEN_TENANTS_SET, CIRCUIT_BREAKERS_BROKEN, CIRCUIT_BREAKERS_UNBROKEN, + TENANT_STATE_METRIC, TENANT_SYNTHETIC_SIZE_METRIC, }; use crate::repository::GcResult; use crate::task_mgr; @@ -276,6 +278,10 @@ pub struct Tenant { eviction_task_tenant_state: tokio::sync::Mutex, + /// Track repeated failures to compact, so that we can back off. + /// Overhead of mutex is acceptable because compaction is done with a multi-second period. + compaction_circuit_breaker: std::sync::Mutex, + /// If the tenant is in Activating state, notify this to encourage it /// to proceed to Active as soon as possible, rather than waiting for lazy /// background warmup. @@ -1641,13 +1647,31 @@ impl Tenant { timelines_to_compact }; + // Before doing any I/O work, check our circuit breaker + if self.compaction_circuit_breaker.lock().unwrap().is_broken() { + info!("Skipping compaction due to previous failures"); + return Ok(()); + } + for (timeline_id, timeline) in &timelines_to_compact { timeline .compact(cancel, EnumSet::empty(), ctx) .instrument(info_span!("compact_timeline", %timeline_id)) - .await?; + .await + .map_err(|e| { + self.compaction_circuit_breaker + .lock() + .unwrap() + .fail(&CIRCUIT_BREAKERS_BROKEN, &e); + e + })?; } + self.compaction_circuit_breaker + .lock() + .unwrap() + .success(&CIRCUIT_BREAKERS_UNBROKEN); + Ok(()) } @@ -2563,6 +2587,14 @@ impl Tenant { cached_logical_sizes: tokio::sync::Mutex::new(HashMap::new()), cached_synthetic_tenant_size: Arc::new(AtomicU64::new(0)), eviction_task_tenant_state: tokio::sync::Mutex::new(EvictionTaskTenantState::default()), + compaction_circuit_breaker: std::sync::Mutex::new(CircuitBreaker::new( + format!("compaction-{tenant_shard_id}"), + 5, + // Compaction can be a very expensive operation, and might leak disk space. It also ought + // to be infallible, as long as remote storage is available. So if it repeatedly fails, + // use an extremely long backoff. + Some(Duration::from_secs(3600 * 24)), + )), activate_now_sem: tokio::sync::Semaphore::new(0), cancel: CancellationToken::default(), gate: Gate::default(), diff --git a/test_runner/regress/test_compaction.py b/test_runner/regress/test_compaction.py index 49dcb9b86a9d..f321c09b2729 100644 --- a/test_runner/regress/test_compaction.py +++ b/test_runner/regress/test_compaction.py @@ -1,12 +1,14 @@ import enum import json import os +import time from typing import Optional import pytest from fixtures.log_helper import log from fixtures.neon_fixtures import NeonEnvBuilder, generate_uploads_and_deletions from fixtures.pageserver.http import PageserverApiException +from fixtures.utils import wait_until from fixtures.workload import Workload AGGRESIVE_COMPACTION_TENANT_CONF = { @@ -257,3 +259,64 @@ def test_uploads_and_deletions( found_allowed_error = any(env.pageserver.log_contains(e) for e in allowed_errors) if not found_allowed_error: raise Exception("None of the allowed_errors occured in the log") + + +def test_pageserver_compaction_circuit_breaker(neon_env_builder: NeonEnvBuilder): + """ + Check that repeated failures in compaction result in a circuit breaker breaking + """ + TENANT_CONF = { + # Very frequent runs to rack up failures quickly + "compaction_period": "100ms", + # Small checkpoint distance to create many layers + "checkpoint_distance": 1024 * 128, + # Compact small layers + "compaction_target_size": 1024 * 128, + "image_creation_threshold": 1, + } + + FAILPOINT = "delta-layer-writer-fail-before-finish" + BROKEN_LOG = ".*Circuit breaker broken!.*" + + env = neon_env_builder.init_start(initial_tenant_conf=TENANT_CONF) + + workload = Workload(env, env.initial_tenant, env.initial_timeline) + workload.init() + + # Set a failpoint that will prevent compaction succeeding + env.pageserver.http_client().configure_failpoints((FAILPOINT, "return")) + + # Write some data to trigger compaction + workload.write_rows(1024, upload=False) + workload.write_rows(1024, upload=False) + workload.write_rows(1024, upload=False) + + def assert_broken(): + env.pageserver.assert_log_contains(BROKEN_LOG) + assert ( + env.pageserver.http_client().get_metric_value("pageserver_circuit_breaker_broken_total") + or 0 + ) == 1 + assert ( + env.pageserver.http_client().get_metric_value( + "pageserver_circuit_breaker_unbroken_total" + ) + or 0 + ) == 0 + + # Wait for enough failures to break the circuit breaker + # This wait is fairly long because we back off on compaction failures, so 5 retries takes ~30s + wait_until(60, 1, assert_broken) + + # Sleep for a while, during which time we expect that compaction will _not_ be retried + time.sleep(10) + + assert ( + env.pageserver.http_client().get_metric_value("pageserver_circuit_breaker_broken_total") + or 0 + ) == 1 + assert ( + env.pageserver.http_client().get_metric_value("pageserver_circuit_breaker_unbroken_total") + or 0 + ) == 0 + assert not env.pageserver.log_contains(".*Circuit breaker failure ended.*") From 411a130675363bd2e06be937926803390d748319 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Fri, 12 Jul 2024 13:58:04 +0100 Subject: [PATCH 101/194] Fix nightly warnings 2024 june (#8151) ## Problem new clippy warnings on nightly. ## Summary of changes broken up each commit by warning type. 1. Remove some unnecessary refs. 2. In edition 2024, inference will default to `!` and not `()`. 3. Clippy complains about doc comment indentation 4. Fix `Trait + ?Sized` where `Trait: Sized`. 5. diesel_derives triggering `non_local_defintions` --- compute_tools/src/bin/compute_ctl.rs | 3 +-- compute_tools/src/compute.rs | 1 + control_plane/storcon_cli/src/main.rs | 2 +- libs/utils/src/http/endpoint.rs | 14 +++++------ pageserver/compaction/src/interface.rs | 2 +- pageserver/src/context.rs | 1 + pageserver/src/pgdatadir_mapping.rs | 2 +- pageserver/src/tenant/disk_btree.rs | 6 ++--- .../src/tenant/storage_layer/layer_desc.rs | 2 +- pageserver/src/tenant/timeline.rs | 11 +++++---- pageserver/src/tenant/timeline/delete.rs | 6 +++-- .../src/tenant/timeline/logical_size.rs | 24 +++++++++---------- pageserver/src/tenant/timeline/walreceiver.rs | 10 ++++---- pageserver/src/tenant/vectored_blob_io.rs | 4 ++-- .../virtual_file/owned_buffers_io/write.rs | 1 + proxy/src/compute.rs | 2 +- proxy/src/redis/cancellation_publisher.rs | 2 +- .../connection_with_credentials_provider.rs | 2 +- proxy/src/redis/notifications.rs | 2 +- proxy/src/serverless/backend.rs | 2 +- proxy/src/serverless/conn_pool.rs | 2 +- proxy/src/waiters.rs | 2 +- safekeeper/src/wal_backup.rs | 1 + storage_controller/src/persistence.rs | 1 + storage_controller/src/service.rs | 14 +++++------ storage_controller/src/tenant_shard.rs | 1 + 26 files changed, 64 insertions(+), 56 deletions(-) diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index f4c396a85d84..0ba2c1aeb497 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -6,7 +6,7 @@ //! - Every start is a fresh start, so the data directory is removed and //! initialized again on each run. //! - If remote_extension_config is provided, it will be used to fetch extensions list -//! and download `shared_preload_libraries` from the remote storage. +//! and download `shared_preload_libraries` from the remote storage. //! - Next it will put configuration files into the `PGDATA` directory. //! - Sync safekeepers and get commit LSN. //! - Get `basebackup` from pageserver using the returned on the previous step LSN. @@ -33,7 +33,6 @@ //! -b /usr/local/bin/postgres \ //! -r http://pg-ext-s3-gateway \ //! ``` -//! use std::collections::HashMap; use std::fs::File; use std::path::Path; diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index 1112795d3034..91855d954d05 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -56,6 +56,7 @@ pub struct ComputeNode { /// - we push new spec and it does reconfiguration /// - but then something happens and compute pod / VM is destroyed, /// so k8s controller starts it again with the **old** spec + /// /// and the same for empty computes: /// - we started compute without any spec /// - we push spec and it does configuration diff --git a/control_plane/storcon_cli/src/main.rs b/control_plane/storcon_cli/src/main.rs index 815f5c940f4c..777a717a7378 100644 --- a/control_plane/storcon_cli/src/main.rs +++ b/control_plane/storcon_cli/src/main.rs @@ -341,7 +341,7 @@ async fn main() -> anyhow::Result<()> { } Command::TenantCreate { tenant_id } => { storcon_client - .dispatch( + .dispatch::<_, ()>( Method::POST, "v1/tenant".to_string(), Some(TenantCreateRequest { diff --git a/libs/utils/src/http/endpoint.rs b/libs/utils/src/http/endpoint.rs index f8a5f681315a..8ee5abd434d4 100644 --- a/libs/utils/src/http/endpoint.rs +++ b/libs/utils/src/http/endpoint.rs @@ -52,17 +52,17 @@ struct RequestId(String); /// There could be other ways to implement similar functionality: /// /// * procmacros placed on top of all handler methods -/// With all the drawbacks of procmacros, brings no difference implementation-wise, -/// and little code reduction compared to the existing approach. +/// With all the drawbacks of procmacros, brings no difference implementation-wise, +/// and little code reduction compared to the existing approach. /// /// * Another `TraitExt` with e.g. the `get_with_span`, `post_with_span` methods to do similar logic, -/// implemented for [`RouterBuilder`]. -/// Could be simpler, but we don't want to depend on [`routerify`] more, targeting to use other library later. +/// implemented for [`RouterBuilder`]. +/// Could be simpler, but we don't want to depend on [`routerify`] more, targeting to use other library later. /// /// * In theory, a span guard could've been created in a pre-request middleware and placed into a global collection, to be dropped -/// later, in a post-response middleware. -/// Due to suspendable nature of the futures, would give contradictive results which is exactly the opposite of what `tracing-futures` -/// tries to achive with its `.instrument` used in the current approach. +/// later, in a post-response middleware. +/// Due to suspendable nature of the futures, would give contradictive results which is exactly the opposite of what `tracing-futures` +/// tries to achive with its `.instrument` used in the current approach. /// /// If needed, a declarative macro to substitute the |r| ... closure boilerplate could be introduced. pub async fn request_span(request: Request, handler: H) -> R::Output diff --git a/pageserver/compaction/src/interface.rs b/pageserver/compaction/src/interface.rs index 35519b5d0aff..5bc9b5ca1de9 100644 --- a/pageserver/compaction/src/interface.rs +++ b/pageserver/compaction/src/interface.rs @@ -131,7 +131,7 @@ impl CompactionKey for Key { pub type CompactionKeySpace = Vec>; /// Functions needed from all layers. -pub trait CompactionLayer { +pub trait CompactionLayer { fn key_range(&self) -> &Range; fn lsn_range(&self) -> &Range; diff --git a/pageserver/src/context.rs b/pageserver/src/context.rs index 86d0390c30b1..0b07e0752465 100644 --- a/pageserver/src/context.rs +++ b/pageserver/src/context.rs @@ -59,6 +59,7 @@ //! 1. It should be easy to forward the context to callees. //! 2. To propagate more data from high-level to low-level code, the functions in //! the middle should not need to be modified. +//! //! The solution is to have a container structure ([`RequestContext`]) that //! carries the information. Functions that don't care about what's in it //! pass it along to callees. diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index 8a6cfea92b3b..a821b824d0c3 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -522,7 +522,7 @@ impl Timeline { ctx: &RequestContext, ) -> Result, PageReconstructError> { let mut max: Option = None; - self.map_all_timestamps(probe_lsn, ctx, |timestamp| { + self.map_all_timestamps::<()>(probe_lsn, ctx, |timestamp| { if let Some(max_prev) = max { max = Some(max_prev.max(timestamp)); } else { diff --git a/pageserver/src/tenant/disk_btree.rs b/pageserver/src/tenant/disk_btree.rs index b76498b60859..251d2ab4aded 100644 --- a/pageserver/src/tenant/disk_btree.rs +++ b/pageserver/src/tenant/disk_btree.rs @@ -550,10 +550,10 @@ where /// We maintain the length of the stack to be always greater than zero. /// Two exceptions are: /// 1. `Self::flush_node`. The method will push the new node if it extracted the last one. - /// So because other methods cannot see the intermediate state invariant still holds. + /// So because other methods cannot see the intermediate state invariant still holds. /// 2. `Self::finish`. It consumes self and does not return it back, - /// which means that this is where the structure is destroyed. - /// Thus stack of zero length cannot be observed by other methods. + /// which means that this is where the structure is destroyed. + /// Thus stack of zero length cannot be observed by other methods. stack: Vec>, /// Last key that was appended to the tree. Used to sanity check that append diff --git a/pageserver/src/tenant/storage_layer/layer_desc.rs b/pageserver/src/tenant/storage_layer/layer_desc.rs index a89b66e4a1b7..bd765560e4e3 100644 --- a/pageserver/src/tenant/storage_layer/layer_desc.rs +++ b/pageserver/src/tenant/storage_layer/layer_desc.rs @@ -25,7 +25,7 @@ pub struct PersistentLayerDesc { /// /// - For an open in-memory layer, the end bound is MAX_LSN /// - For a frozen in-memory layer or a delta layer, the end bound is a valid lsn after the - /// range start + /// range start /// - An image layer represents snapshot at one LSN, so end_lsn is always the snapshot LSN + 1 pub lsn_range: Range, /// Whether this is a delta layer, and also, is this incremental. diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 762e903bf85d..a3ddb3a1d190 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -3408,6 +3408,7 @@ impl Timeline { } } + #[allow(clippy::doc_lazy_continuation)] /// Get the data needed to reconstruct all keys in the provided keyspace /// /// The algorithm is as follows: @@ -4474,10 +4475,10 @@ impl Timeline { /// are required. Since checking if new image layers are required is expensive in /// terms of CPU, we only do it in the following cases: /// 1. If the timeline has ingested sufficient WAL to justify the cost - /// 2. If enough time has passed since the last check - /// 2.1. For large tenants, we wish to perform the check more often since they - /// suffer from the lack of image layers - /// 2.2. For small tenants (that can mostly fit in RAM), we use a much longer interval + /// 2. If enough time has passed since the last check: + /// 1. For large tenants, we wish to perform the check more often since they + /// suffer from the lack of image layers + /// 2. For small tenants (that can mostly fit in RAM), we use a much longer interval fn should_check_if_image_layers_required(self: &Arc, lsn: Lsn) -> bool { const LARGE_TENANT_THRESHOLD: u64 = 2 * 1024 * 1024 * 1024; @@ -4719,7 +4720,7 @@ impl Timeline { /// Requires a timeline that: /// - has an ancestor to detach from /// - the ancestor does not have an ancestor -- follows from the original RFC limitations, not - /// a technical requirement + /// a technical requirement /// /// After the operation has been started, it cannot be canceled. Upon restart it needs to be /// polled again until completion. diff --git a/pageserver/src/tenant/timeline/delete.rs b/pageserver/src/tenant/timeline/delete.rs index b0088f4ea228..d32945d9e416 100644 --- a/pageserver/src/tenant/timeline/delete.rs +++ b/pageserver/src/tenant/timeline/delete.rs @@ -182,13 +182,15 @@ async fn remove_timeline_from_tenant( /// 5. Delete index part /// 6. Delete meta, timeline directory /// 7. Delete mark file +/// /// It is resumable from any step in case a crash/restart occurs. /// There are three entrypoints to the process: /// 1. [`DeleteTimelineFlow::run`] this is the main one called by a management api handler. /// 2. [`DeleteTimelineFlow::resume_deletion`] is called during restarts when local metadata is still present -/// and we possibly neeed to continue deletion of remote files. +/// and we possibly neeed to continue deletion of remote files. /// 3. [`DeleteTimelineFlow::cleanup_remaining_timeline_fs_traces`] is used when we deleted remote -/// index but still have local metadata, timeline directory and delete mark. +/// index but still have local metadata, timeline directory and delete mark. +/// /// Note the only other place that messes around timeline delete mark is the logic that scans directory with timelines during tenant load. #[derive(Default)] pub enum DeleteTimelineFlow { diff --git a/pageserver/src/tenant/timeline/logical_size.rs b/pageserver/src/tenant/timeline/logical_size.rs index 8f9ca0e29f5e..b0d6c4a27a0c 100644 --- a/pageserver/src/tenant/timeline/logical_size.rs +++ b/pageserver/src/tenant/timeline/logical_size.rs @@ -11,11 +11,11 @@ use std::sync::atomic::{AtomicBool, AtomicI64, Ordering as AtomicOrdering}; /// Calculation consists of two stages: /// /// 1. Initial size calculation. That might take a long time, because it requires -/// reading all layers containing relation sizes at `initial_part_end`. +/// reading all layers containing relation sizes at `initial_part_end`. /// /// 2. Collecting an incremental part and adding that to the initial size. -/// Increments are appended on walreceiver writing new timeline data, -/// which result in increase or decrease of the logical size. +/// Increments are appended on walreceiver writing new timeline data, +/// which result in increase or decrease of the logical size. pub(super) struct LogicalSize { /// Size, potentially slow to compute. Calculating this might require reading multiple /// layers, and even ancestor's layers. @@ -45,17 +45,17 @@ pub(super) struct LogicalSize { /// Size shouldn't ever be negative, but this is signed for two reasons: /// /// 1. If we initialized the "baseline" size lazily, while we already - /// process incoming WAL, the incoming WAL records could decrement the - /// variable and temporarily make it negative. (This is just future-proofing; - /// the initialization is currently not done lazily.) + /// process incoming WAL, the incoming WAL records could decrement the + /// variable and temporarily make it negative. (This is just future-proofing; + /// the initialization is currently not done lazily.) /// /// 2. If there is a bug and we e.g. forget to increment it in some cases - /// when size grows, but remember to decrement it when it shrinks again, the - /// variable could go negative. In that case, it seems better to at least - /// try to keep tracking it, rather than clamp or overflow it. Note that - /// get_current_logical_size() will clamp the returned value to zero if it's - /// negative, and log an error. Could set it permanently to zero or some - /// special value to indicate "broken" instead, but this will do for now. + /// when size grows, but remember to decrement it when it shrinks again, the + /// variable could go negative. In that case, it seems better to at least + /// try to keep tracking it, rather than clamp or overflow it. Note that + /// get_current_logical_size() will clamp the returned value to zero if it's + /// negative, and log an error. Could set it permanently to zero or some + /// special value to indicate "broken" instead, but this will do for now. /// /// Note that we also expose a copy of this value as a prometheus metric, /// see `current_logical_size_gauge`. Use the `update_current_logical_size` diff --git a/pageserver/src/tenant/timeline/walreceiver.rs b/pageserver/src/tenant/timeline/walreceiver.rs index a085154a5a9f..4a3a5c621b1f 100644 --- a/pageserver/src/tenant/timeline/walreceiver.rs +++ b/pageserver/src/tenant/timeline/walreceiver.rs @@ -2,13 +2,13 @@ //! To do so, a current implementation needs to do the following: //! //! * acknowledge the timelines that it needs to stream WAL into. -//! Pageserver is able to dynamically (un)load tenants on attach and detach, -//! hence WAL receiver needs to react on such events. +//! Pageserver is able to dynamically (un)load tenants on attach and detach, +//! hence WAL receiver needs to react on such events. //! //! * get a broker subscription, stream data from it to determine that a timeline needs WAL streaming. -//! For that, it watches specific keys in storage_broker and pulls the relevant data periodically. -//! The data is produced by safekeepers, that push it periodically and pull it to synchronize between each other. -//! Without this data, no WAL streaming is possible currently. +//! For that, it watches specific keys in storage_broker and pulls the relevant data periodically. +//! The data is produced by safekeepers, that push it periodically and pull it to synchronize between each other. +//! Without this data, no WAL streaming is possible currently. //! //! Only one active WAL streaming connection is allowed at a time. //! The connection is supposed to be updated periodically, based on safekeeper timeline data. diff --git a/pageserver/src/tenant/vectored_blob_io.rs b/pageserver/src/tenant/vectored_blob_io.rs index cb81f1d76d6a..5a0986ea12ec 100644 --- a/pageserver/src/tenant/vectored_blob_io.rs +++ b/pageserver/src/tenant/vectored_blob_io.rs @@ -191,9 +191,9 @@ impl VectoredReadPlanner { /// /// The `flag` argument has two interesting values: /// * [`BlobFlag::ReplaceAll`]: The blob for this key should replace all existing blobs. - /// This is used for WAL records that `will_init`. + /// This is used for WAL records that `will_init`. /// * [`BlobFlag::Ignore`]: This blob should not be included in the read. This happens - /// if the blob is cached. + /// if the blob is cached. pub fn handle(&mut self, key: Key, lsn: Lsn, offset: u64, flag: BlobFlag) { // Implementation note: internally lag behind by one blob such that // we have a start and end offset when initialising [`VectoredRead`] diff --git a/pageserver/src/virtual_file/owned_buffers_io/write.rs b/pageserver/src/virtual_file/owned_buffers_io/write.rs index 885a9221c5c1..8599d95cdf9f 100644 --- a/pageserver/src/virtual_file/owned_buffers_io/write.rs +++ b/pageserver/src/virtual_file/owned_buffers_io/write.rs @@ -33,6 +33,7 @@ pub struct BufferedWriter { /// invariant: always remains Some(buf) except /// - while IO is ongoing => goes back to Some() once the IO completed successfully /// - after an IO error => stays `None` forever + /// /// In these exceptional cases, it's `None`. buf: Option, } diff --git a/proxy/src/compute.rs b/proxy/src/compute.rs index a50a96e5e844..f91693c704e5 100644 --- a/proxy/src/compute.rs +++ b/proxy/src/compute.rs @@ -319,7 +319,7 @@ impl ConnCfg { let pause = ctx.latency_timer.pause(crate::metrics::Waiting::Compute); let (client, connection) = self.0.connect_raw(stream, tls).await?; drop(pause); - tracing::Span::current().record("pid", &tracing::field::display(client.get_process_id())); + tracing::Span::current().record("pid", tracing::field::display(client.get_process_id())); let stream = connection.stream.into_inner(); info!( diff --git a/proxy/src/redis/cancellation_publisher.rs b/proxy/src/redis/cancellation_publisher.rs index 7baf1043742c..c9a946fa4a7b 100644 --- a/proxy/src/redis/cancellation_publisher.rs +++ b/proxy/src/redis/cancellation_publisher.rs @@ -106,7 +106,7 @@ impl RedisPublisherClient { cancel_key_data, session_id, }))?; - self.client.publish(PROXY_CHANNEL_NAME, payload).await?; + let _: () = self.client.publish(PROXY_CHANNEL_NAME, payload).await?; Ok(()) } pub async fn try_connect(&mut self) -> anyhow::Result<()> { diff --git a/proxy/src/redis/connection_with_credentials_provider.rs b/proxy/src/redis/connection_with_credentials_provider.rs index 3a90d911c28c..b02ce472c0f2 100644 --- a/proxy/src/redis/connection_with_credentials_provider.rs +++ b/proxy/src/redis/connection_with_credentials_provider.rs @@ -178,7 +178,7 @@ impl ConnectionWithCredentialsProvider { credentials_provider: Arc, ) -> anyhow::Result<()> { let (user, password) = credentials_provider.provide_credentials().await?; - redis::cmd("AUTH") + let _: () = redis::cmd("AUTH") .arg(user) .arg(password) .query_async(con) diff --git a/proxy/src/redis/notifications.rs b/proxy/src/redis/notifications.rs index 87d723d17e30..efd7437d5d20 100644 --- a/proxy/src/redis/notifications.rs +++ b/proxy/src/redis/notifications.rs @@ -127,7 +127,7 @@ impl MessageHandler { Cancel(cancel_session) => { tracing::Span::current().record( "session_id", - &tracing::field::display(cancel_session.session_id), + tracing::field::display(cancel_session.session_id), ); Metrics::get() .proxy diff --git a/proxy/src/serverless/backend.rs b/proxy/src/serverless/backend.rs index 6c34d48338b3..3b86c1838cde 100644 --- a/proxy/src/serverless/backend.rs +++ b/proxy/src/serverless/backend.rs @@ -245,7 +245,7 @@ impl ConnectMechanism for TokioMechanism { drop(pause); let (client, connection) = permit.release_result(res)?; - tracing::Span::current().record("pid", &tracing::field::display(client.get_process_id())); + tracing::Span::current().record("pid", tracing::field::display(client.get_process_id())); Ok(poll_client( self.pool.clone(), ctx, diff --git a/proxy/src/serverless/conn_pool.rs b/proxy/src/serverless/conn_pool.rs index 170bda062e51..dbc58d48ec6c 100644 --- a/proxy/src/serverless/conn_pool.rs +++ b/proxy/src/serverless/conn_pool.rs @@ -403,7 +403,7 @@ impl GlobalConnPool { tracing::Span::current().record("conn_id", tracing::field::display(client.conn_id)); tracing::Span::current().record( "pid", - &tracing::field::display(client.inner.get_process_id()), + tracing::field::display(client.inner.get_process_id()), ); info!( cold_start_info = ColdStartInfo::HttpPoolHit.as_str(), diff --git a/proxy/src/waiters.rs b/proxy/src/waiters.rs index bba5494cfe53..888ad380489e 100644 --- a/proxy/src/waiters.rs +++ b/proxy/src/waiters.rs @@ -111,7 +111,7 @@ mod tests { let waiters = Arc::clone(&waiters); let notifier = tokio::spawn(async move { - waiters.notify(key, Default::default())?; + waiters.notify(key, ())?; Ok(()) }); diff --git a/safekeeper/src/wal_backup.rs b/safekeeper/src/wal_backup.rs index 9ea048a3c76a..5a590689c374 100644 --- a/safekeeper/src/wal_backup.rs +++ b/safekeeper/src/wal_backup.rs @@ -119,6 +119,7 @@ async fn shut_down_task(entry: &mut Option) { /// time we have several ones as they PUT the same files. Also, /// - frequently changing the offloader would be bad; /// - electing seriously lagging safekeeper is undesirable; +/// /// So we deterministically choose among the reasonably caught up candidates. /// TODO: take into account failed attempts to deal with hypothetical situation /// where s3 is unreachable only for some sks. diff --git a/storage_controller/src/persistence.rs b/storage_controller/src/persistence.rs index 47caf7ae81ab..9f7b2f775e97 100644 --- a/storage_controller/src/persistence.rs +++ b/storage_controller/src/persistence.rs @@ -542,6 +542,7 @@ impl Persistence { Ok(Generation::new(g as u32)) } + #[allow(non_local_definitions)] /// For use when updating a persistent property of a tenant, such as its config or placement_policy. /// /// Do not use this for settting generation, unless in the special onboarding code path (/location_config) diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index b6e2b5319132..deaac83ea526 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -5070,7 +5070,7 @@ impl Service { /// we did the split, but are probably better placed elsewhere. /// - Creating new secondary locations if it improves the spreading of a sharded tenant /// * e.g. after a shard split, some locations will be on the same node (where the split - /// happened), and will probably be better placed elsewhere. + /// happened), and will probably be better placed elsewhere. /// /// To put it more briefly: whereas the scheduler respects soft constraints in a ScheduleContext at /// the time of scheduling, this function looks for cases where a better-scoring location is available @@ -5633,14 +5633,14 @@ impl Service { /// Create a node fill plan (pick secondaries to promote) that meets the following requirements: /// 1. The node should be filled until it reaches the expected cluster average of - /// attached shards. If there are not enough secondaries on the node, the plan stops early. + /// attached shards. If there are not enough secondaries on the node, the plan stops early. /// 2. Select tenant shards to promote such that the number of attached shards is balanced - /// throughout the cluster. We achieve this by picking tenant shards from each node, - /// starting from the ones with the largest number of attached shards, until the node - /// reaches the expected cluster average. + /// throughout the cluster. We achieve this by picking tenant shards from each node, + /// starting from the ones with the largest number of attached shards, until the node + /// reaches the expected cluster average. /// 3. Avoid promoting more shards of the same tenant than required. The upper bound - /// for the number of tenants from the same shard promoted to the node being filled is: - /// shard count for the tenant divided by the number of nodes in the cluster. + /// for the number of tenants from the same shard promoted to the node being filled is: + /// shard count for the tenant divided by the number of nodes in the cluster. fn fill_node_plan(&self, node_id: NodeId) -> Vec { let mut locked = self.inner.write().unwrap(); let fill_requirement = locked.scheduler.compute_fill_requirement(node_id); diff --git a/storage_controller/src/tenant_shard.rs b/storage_controller/src/tenant_shard.rs index 2574dc297ae9..ee2ba6c4eecb 100644 --- a/storage_controller/src/tenant_shard.rs +++ b/storage_controller/src/tenant_shard.rs @@ -124,6 +124,7 @@ pub(crate) struct TenantShard { /// - ReconcileWaiters need to Arc-clone the overall object to read it later /// - ReconcileWaitError needs to use an `Arc` because we can construct /// many waiters for one shard, and the underlying error types are not Clone. + /// /// TODO: generalize to an array of recent events /// TOOD: use a ArcSwap instead of mutex for faster reads? #[serde(serialize_with = "read_last_error")] From 4184685721f5bd0e70ee9587d569b09bce0f306c Mon Sep 17 00:00:00 2001 From: Alex Chi Z Date: Fri, 12 Jul 2024 09:28:13 -0400 Subject: [PATCH 102/194] fix(pageserver): unique test harness name for merge_in_between (#8366) As title, there should be a way to detect duplicated harness names in the future :( Signed-off-by: Alex Chi Z --- pageserver/src/tenant/storage_layer/merge_iterator.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pageserver/src/tenant/storage_layer/merge_iterator.rs b/pageserver/src/tenant/storage_layer/merge_iterator.rs index 36386c87c999..68759f758576 100644 --- a/pageserver/src/tenant/storage_layer/merge_iterator.rs +++ b/pageserver/src/tenant/storage_layer/merge_iterator.rs @@ -275,7 +275,7 @@ mod tests { use crate::repository::Value; use bytes::Bytes; - let harness = TenantHarness::create("merge_iterator_delta_merge").unwrap(); + let harness = TenantHarness::create("merge_iterator_merge_in_between").unwrap(); let (tenant, ctx) = harness.load().await; let tline = tenant From b329b1c610e7166fc28a1499375666da7723ae24 Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Fri, 12 Jul 2024 17:31:17 +0100 Subject: [PATCH 103/194] tests: allow list breaching min resident size in statvfs test (#8358) ## Problem This test would sometimes violate the min resident size during disk eviction and fail due to the generate warning log. Disk usage candidate collection only takes into account active tenants. However, the statvfs call takes into account the entire tenants directory, which includes tenants which haven't become active yet. After re-starting the pageserver, disk usage eviction may kick in *before* both tenants have become active. Hence, the logic will try to satisfy thedisk usage requirements by evicting everything belonging to the active tenant, and hence violating the tenant minimum resident size. ## Summary of changes Allow the warning --- test_runner/regress/test_disk_usage_eviction.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test_runner/regress/test_disk_usage_eviction.py b/test_runner/regress/test_disk_usage_eviction.py index 7722828c79dd..fb8b7b22fa71 100644 --- a/test_runner/regress/test_disk_usage_eviction.py +++ b/test_runner/regress/test_disk_usage_eviction.py @@ -794,6 +794,16 @@ def less_than_max_usage_pct(): wait_until(2, 2, less_than_max_usage_pct) + # Disk usage candidate collection only takes into account active tenants. + # However, the statvfs call takes into account the entire tenants directory, + # which includes tenants which haven't become active yet. + # + # After re-starting the pageserver, disk usage eviction may kick in *before* + # both tenants have become active. Hence, the logic will try to satisfy the + # disk usage requirements by evicting everything belonging to the active tenant, + # and hence violating the tenant minimum resident size. + env.neon_env.pageserver.allowed_errors.append(".*" + GLOBAL_LRU_LOG_LINE) + def test_statvfs_pressure_min_avail_bytes(eviction_env: EvictionEnv): """ From 8da3b547f834497377b7b5db727c10bd88697cde Mon Sep 17 00:00:00 2001 From: Luca Bruno Date: Mon, 15 Jul 2024 13:38:52 +0200 Subject: [PATCH 104/194] proxy/http: switch to typed_json (#8377) ## Summary of changes This switches JSON rendering logic to `typed_json` in order to reduce the number of allocations in the HTTP responder path. Followup from https://github.com/neondatabase/neon/pull/8319#issuecomment-2216991760. --------- Co-authored-by: Conrad Ludgate --- Cargo.lock | 11 +++ Cargo.toml | 1 + proxy/Cargo.toml | 1 + proxy/src/serverless/sql_over_http.rs | 97 +++++++++++++-------------- 4 files changed, 59 insertions(+), 51 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bab0b4dd1f7a..88973647017a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4404,6 +4404,7 @@ dependencies = [ "tracing-opentelemetry", "tracing-subscriber", "tracing-utils", + "typed-json", "url", "urlencoding", "utils", @@ -6665,6 +6666,16 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "typed-json" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6024a8d0025400b3f6b189366e9aa92012cf9c4fe1cd2620848dd61425c49eed" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "typenum" version = "1.16.0" diff --git a/Cargo.toml b/Cargo.toml index 670e3241d51d..4f42203683d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -184,6 +184,7 @@ tracing-error = "0.2.0" tracing-opentelemetry = "0.21.0" tracing-subscriber = { version = "0.3", default-features = false, features = ["smallvec", "fmt", "tracing-log", "std", "env-filter", "json", "ansi"] } twox-hash = { version = "1.6.3", default-features = false } +typed-json = "0.1" url = "2.2" urlencoding = "2.1" uuid = { version = "1.6.1", features = ["v4", "v7", "serde"] } diff --git a/proxy/Cargo.toml b/proxy/Cargo.toml index 288f7769fef3..2f18b5fbc6cf 100644 --- a/proxy/Cargo.toml +++ b/proxy/Cargo.toml @@ -92,6 +92,7 @@ tracing-opentelemetry.workspace = true tracing-subscriber.workspace = true tracing-utils.workspace = true tracing.workspace = true +typed-json.workspace = true url.workspace = true urlencoding.workspace = true utils.workspace = true diff --git a/proxy/src/serverless/sql_over_http.rs b/proxy/src/serverless/sql_over_http.rs index 8118ae5ea89d..6400e4ac7b3a 100644 --- a/proxy/src/serverless/sql_over_http.rs +++ b/proxy/src/serverless/sql_over_http.rs @@ -18,7 +18,7 @@ use hyper1::Response; use hyper1::StatusCode; use hyper1::{HeaderMap, Request}; use pq_proto::StartupMessageParamsBuilder; -use serde_json::json; +use serde::Serialize; use serde_json::Value; use tokio::time; use tokio_postgres::error::DbError; @@ -32,6 +32,7 @@ use tokio_postgres::Transaction; use tokio_util::sync::CancellationToken; use tracing::error; use tracing::info; +use typed_json::json; use url::Url; use utils::http::error::ApiError; @@ -263,13 +264,8 @@ pub async fn handle( | SqlOverHttpError::Postgres(e) => e.as_db_error(), _ => None, }; - fn get<'a, T: serde::Serialize>( - db: Option<&'a DbError>, - x: impl FnOnce(&'a DbError) -> T, - ) -> Value { - db.map(x) - .and_then(|t| serde_json::to_value(t).ok()) - .unwrap_or_default() + fn get<'a, T: Default>(db: Option<&'a DbError>, x: impl FnOnce(&'a DbError) -> T) -> T { + db.map(x).unwrap_or_default() } if let Some(db_error) = db_error { @@ -278,17 +274,11 @@ pub async fn handle( let position = db_error.and_then(|db| db.position()); let (position, internal_position, internal_query) = match position { - Some(ErrorPosition::Original(position)) => ( - Value::String(position.to_string()), - Value::Null, - Value::Null, - ), - Some(ErrorPosition::Internal { position, query }) => ( - Value::Null, - Value::String(position.to_string()), - Value::String(query.clone()), - ), - None => (Value::Null, Value::Null, Value::Null), + Some(ErrorPosition::Original(position)) => (Some(position.to_string()), None, None), + Some(ErrorPosition::Internal { position, query }) => { + (None, Some(position.to_string()), Some(query.clone())) + } + None => (None, None, None), }; let code = get(db_error, |db| db.code().code()); @@ -578,10 +568,8 @@ async fn handle_inner( .status(StatusCode::OK) .header(header::CONTENT_TYPE, "application/json"); - // - // Now execute the query and return the result - // - let result = match payload { + // Now execute the query and return the result. + let json_output = match payload { Payload::Single(stmt) => stmt.process(cancel, &mut client, parsed_headers).await?, Payload::Batch(statements) => { if parsed_headers.txn_read_only { @@ -605,11 +593,9 @@ async fn handle_inner( let metrics = client.metrics(); - // how could this possibly fail - let body = serde_json::to_string(&result).expect("json serialization should not fail"); - let len = body.len(); + let len = json_output.len(); let response = response - .body(Full::new(Bytes::from(body))) + .body(Full::new(Bytes::from(json_output))) // only fails if invalid status code or invalid header/values are given. // these are not user configurable so it cannot fail dynamically .expect("building response payload should not fail"); @@ -631,7 +617,7 @@ impl QueryData { cancel: CancellationToken, client: &mut Client, parsed_headers: HttpHeaders, - ) -> Result { + ) -> Result { let (inner, mut discard) = client.inner(); let cancel_token = inner.cancel_token(); @@ -644,7 +630,10 @@ impl QueryData { // The query successfully completed. Either::Left((Ok((status, results)), __not_yet_cancelled)) => { discard.check_idle(status); - Ok(results) + + let json_output = + serde_json::to_string(&results).expect("json serialization should not fail"); + Ok(json_output) } // The query failed with an error Either::Left((Err(e), __not_yet_cancelled)) => { @@ -662,7 +651,10 @@ impl QueryData { // query successed before it was cancelled. Ok(Ok((status, results))) => { discard.check_idle(status); - Ok(results) + + let json_output = serde_json::to_string(&results) + .expect("json serialization should not fail"); + Ok(json_output) } // query failed or was cancelled. Ok(Err(error)) => { @@ -696,7 +688,7 @@ impl BatchQueryData { cancel: CancellationToken, client: &mut Client, parsed_headers: HttpHeaders, - ) -> Result { + ) -> Result { info!("starting transaction"); let (inner, mut discard) = client.inner(); let cancel_token = inner.cancel_token(); @@ -718,9 +710,9 @@ impl BatchQueryData { e })?; - let results = + let json_output = match query_batch(cancel.child_token(), &transaction, self, parsed_headers).await { - Ok(results) => { + Ok(json_output) => { info!("commit"); let status = transaction.commit().await.map_err(|e| { // if we cannot commit - for now don't return connection to pool @@ -729,7 +721,7 @@ impl BatchQueryData { e })?; discard.check_idle(status); - results + json_output } Err(SqlOverHttpError::Cancelled(_)) => { if let Err(err) = cancel_token.cancel_query(NoTls).await { @@ -753,7 +745,7 @@ impl BatchQueryData { } }; - Ok(json!({ "results": results })) + Ok(json_output) } } @@ -762,7 +754,7 @@ async fn query_batch( transaction: &Transaction<'_>, queries: BatchQueryData, parsed_headers: HttpHeaders, -) -> Result, SqlOverHttpError> { +) -> Result { let mut results = Vec::with_capacity(queries.queries.len()); let mut current_size = 0; for stmt in queries.queries { @@ -787,7 +779,11 @@ async fn query_batch( } } } - Ok(results) + + let results = json!({ "results": results }); + let json_output = serde_json::to_string(&results).expect("json serialization should not fail"); + + Ok(json_output) } async fn query_to_json( @@ -795,7 +791,7 @@ async fn query_to_json( data: QueryData, current_size: &mut usize, parsed_headers: HttpHeaders, -) -> Result<(ReadyForQueryStatus, Value), SqlOverHttpError> { +) -> Result<(ReadyForQueryStatus, impl Serialize), SqlOverHttpError> { info!("executing query"); let query_params = data.params; let mut row_stream = std::pin::pin!(client.query_raw_txt(&data.query, query_params).await?); @@ -844,8 +840,8 @@ async fn query_to_json( for c in row_stream.columns() { fields.push(json!({ - "name": Value::String(c.name().to_owned()), - "dataTypeID": Value::Number(c.type_().oid().into()), + "name": c.name().to_owned(), + "dataTypeID": c.type_().oid(), "tableID": c.table_oid(), "columnID": c.column_id(), "dataTypeSize": c.type_size(), @@ -863,15 +859,14 @@ async fn query_to_json( .map(|row| pg_text_row_to_json(row, &columns, parsed_headers.raw_output, array_mode)) .collect::, _>>()?; - // resulting JSON format is based on the format of node-postgres result - Ok(( - ready, - json!({ - "command": command_tag_name, - "rowCount": command_tag_count, - "rows": rows, - "fields": fields, - "rowAsArray": array_mode, - }), - )) + // Resulting JSON format is based on the format of node-postgres result. + let results = json!({ + "command": command_tag_name.to_string(), + "rowCount": command_tag_count, + "rows": rows, + "fields": fields, + "rowAsArray": array_mode, + }); + + Ok((ready, results)) } From 4bdfb96078951e3eb471d0ebd668777db048fb67 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Thu, 11 Jul 2024 16:35:31 +0300 Subject: [PATCH 105/194] Fix test_timeline_copy flakiness. fixes https://github.com/neondatabase/neon/issues/8355 --- safekeeper/src/copy_timeline.rs | 10 ++++++++-- test_runner/regress/test_wal_acceptor.py | 5 +++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/safekeeper/src/copy_timeline.rs b/safekeeper/src/copy_timeline.rs index 14bd3c03b810..220988c3ce14 100644 --- a/safekeeper/src/copy_timeline.rs +++ b/safekeeper/src/copy_timeline.rs @@ -74,10 +74,16 @@ pub async fn handle_request(request: Request) -> Result<()> { assert!(flush_lsn >= start_lsn); if request.until_lsn > flush_lsn { - bail!("requested LSN is beyond the end of the timeline"); + bail!(format!( + "requested LSN {} is beyond the end of the timeline {}", + request.until_lsn, flush_lsn + )); } if request.until_lsn < start_lsn { - bail!("requested LSN is before the start of the timeline"); + bail!(format!( + "requested LSN {} is before the start of the timeline {}", + request.until_lsn, start_lsn + )); } if request.until_lsn > commit_lsn { diff --git a/test_runner/regress/test_wal_acceptor.py b/test_runner/regress/test_wal_acceptor.py index e0ad4fdd5cf9..2e906e616051 100644 --- a/test_runner/regress/test_wal_acceptor.py +++ b/test_runner/regress/test_wal_acceptor.py @@ -2065,6 +2065,11 @@ def remember_lsn(): log.info(f"Original digest: {orig_digest}") for sk in env.safekeepers: + wait( + partial(is_flush_lsn_caught_up, sk, tenant_id, timeline_id, lsn), + f"sk_id={sk.id} to flush {lsn}", + ) + sk.http_client().copy_timeline( tenant_id, timeline_id, From 8a8b83df27383a07bb7dbba519325c15d2f46357 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 15 Jul 2024 15:52:00 +0300 Subject: [PATCH 106/194] Add neon.running_xacts_overflow_policy to make it possible for RO replica to startup without primary even in case running xacts overflow (#8323) ## Problem Right now if there are too many running xacts to be restored from CLOG at replica startup, then replica is not trying to restore them and wait for non-overflown running-xacs WAL record from primary. But if primary is not active, then replica will not start at all. Too many running xacts can be caused by transactions with large number of subtractions. But right now it can be also cause by two reasons: - Lack of shutdown checkpoint which updates `oldestRunningXid` (because of immediate shutdown) - nextXid alignment on 1024 boundary (which cause loosing ~1k XIDs on each restart) Both problems are somehow addressed now. But we have existed customers with "sparse" CLOG and lack of checkpoints. To be able to start RO replicas for such customers I suggest to add GUC which allows replica to start even in case of subxacts overflow. ## Summary of changes Add `neon.running_xacts_overflow_policy` with the following values: - ignore: restore from CLOG last N XIDs and accept connections - skip: do not restore any XIDs from CXLOGbut still accept connections - wait: wait non-overflown running xacts record from primary node ## Checklist before requesting a review - [ ] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. ## Checklist before merging - [ ] Do not forget to reformat commit message to not include the above checklist --------- Co-authored-by: Konstantin Knizhnik --- pgxn/neon/neon.c | 47 +++++++++++++++++++++-- test_runner/regress/test_replica_start.py | 46 +++++++++++++++++++++- 2 files changed, 88 insertions(+), 5 deletions(-) diff --git a/pgxn/neon/neon.c b/pgxn/neon/neon.c index e4968bdf8991..3197a7e715a1 100644 --- a/pgxn/neon/neon.c +++ b/pgxn/neon/neon.c @@ -46,6 +46,21 @@ void _PG_init(void); static int logical_replication_max_snap_files = 300; +static int running_xacts_overflow_policy; + +enum RunningXactsOverflowPolicies { + OP_IGNORE, + OP_SKIP, + OP_WAIT +}; + +static const struct config_enum_entry running_xacts_overflow_policies[] = { + {"ignore", OP_IGNORE, false}, + {"skip", OP_SKIP, false}, + {"wait", OP_WAIT, false}, + {NULL, 0, false} +}; + static void InitLogicalReplicationMonitor(void) { @@ -414,6 +429,7 @@ RestoreRunningXactsFromClog(CheckPoint *checkpoint, TransactionId **xids, int *n restored_xids = (TransactionId *) palloc(max_xcnt * sizeof(TransactionId)); n_restored_xids = 0; next_prepared_idx = 0; + for (TransactionId xid = from; xid != till;) { XLogRecPtr xidlsn; @@ -424,7 +440,7 @@ RestoreRunningXactsFromClog(CheckPoint *checkpoint, TransactionId **xids, int *n /* * "Merge" the prepared transactions into the restored_xids array as * we go. The prepared transactions array is sorted. This is mostly - * a sanity check to ensure that all the prpeared transactions are + * a sanity check to ensure that all the prepared transactions are * seen as in-progress. (There is a check after the loop that we didn't * miss any.) */ @@ -522,14 +538,23 @@ RestoreRunningXactsFromClog(CheckPoint *checkpoint, TransactionId **xids, int *n elog(LOG, "too many running xacts to restore from the CLOG; oldestXid=%u oldestActiveXid=%u nextXid %u", checkpoint->oldestXid, checkpoint->oldestActiveXid, XidFromFullTransactionId(checkpoint->nextXid)); - goto fail; + + switch (running_xacts_overflow_policy) + { + case OP_WAIT: + goto fail; + case OP_IGNORE: + goto success; + case OP_SKIP: + n_restored_xids = 0; + goto success; + } } restored_xids[n_restored_xids++] = xid; skip: TransactionIdAdvance(xid); - continue; } /* sanity check */ @@ -540,11 +565,13 @@ RestoreRunningXactsFromClog(CheckPoint *checkpoint, TransactionId **xids, int *n Assert(false); goto fail; } - + success: elog(LOG, "restored %d running xacts by scanning the CLOG; oldestXid=%u oldestActiveXid=%u nextXid %u", n_restored_xids, checkpoint->oldestXid, checkpoint->oldestActiveXid, XidFromFullTransactionId(checkpoint->nextXid)); *nxids = n_restored_xids; *xids = restored_xids; + if (prepared_xids) + pfree(prepared_xids); return true; fail: @@ -581,6 +608,18 @@ _PG_init(void) restore_running_xacts_callback = RestoreRunningXactsFromClog; + + DefineCustomEnumVariable( + "neon.running_xacts_overflow_policy", + "Action performed on snapshot overflow when restoring runnings xacts from CLOG", + NULL, + &running_xacts_overflow_policy, + OP_IGNORE, + running_xacts_overflow_policies, + PGC_POSTMASTER, + 0, + NULL, NULL, NULL); + /* * Important: This must happen after other parts of the extension are * loaded, otherwise any settings to GUCs that were set before the diff --git a/test_runner/regress/test_replica_start.py b/test_runner/regress/test_replica_start.py index 17d476a8a690..0d95109d6b18 100644 --- a/test_runner/regress/test_replica_start.py +++ b/test_runner/regress/test_replica_start.py @@ -210,7 +210,11 @@ def test_replica_start_wait_subxids_finish(neon_simple_env: NeonEnv): # Start it in a separate thread, so that we can do other stuff while it's # blocked waiting for the startup to finish. wait_for_last_flush_lsn(env, primary, env.initial_tenant, env.initial_timeline) - secondary = env.endpoints.new_replica(origin=primary, endpoint_id="secondary") + secondary = env.endpoints.new_replica( + origin=primary, + endpoint_id="secondary", + config_lines=["neon.running_xacts_overflow_policy='wait'"], + ) start_secondary_thread = threading.Thread(target=secondary.start) start_secondary_thread.start() @@ -644,3 +648,43 @@ def test_replica_start_with_prepared_xacts_with_many_subxacts(neon_simple_env: N wait_replica_caughtup(primary, secondary) secondary_cur.execute("select count(*) from t") assert secondary_cur.fetchone() == (200001,) + + +def test_replica_start_with_too_many_unused_xids(neon_simple_env: NeonEnv): + """ + Test the CLOG-scanning mechanism at hot standby startup in the presence of + large number of unsued XIDs, caused by XID alignment and frequent primary restarts + """ + n_restarts = 50 + + # Initialize the primary and a test table + env = neon_simple_env + primary = env.endpoints.create_start(branch_name="main", endpoint_id="primary") + with primary.cursor() as primary_cur: + primary_cur.execute("create table t(pk serial primary key, payload integer)") + + for _ in range(n_restarts): + with primary.cursor() as primary_cur: + primary_cur.execute("insert into t (payload) values (0)") + # restart primary + primary.stop("immediate") + primary.start() + + # Wait for the WAL to be flushed + wait_for_last_flush_lsn(env, primary, env.initial_tenant, env.initial_timeline) + + # stop primary to check that we can start replica without it + primary.stop(mode="immediate") + + # Create a replica. It should start up normally, because of ignore policy + # mechanism. + secondary = env.endpoints.new_replica_start( + origin=primary, + endpoint_id="secondary", + config_lines=["neon.running_xacts_overflow_policy='ignore'"], + ) + + # Check that replica see all changes + with secondary.cursor() as secondary_cur: + secondary_cur.execute("select count(*) from t") + assert secondary_cur.fetchone() == (n_restarts,) From b49b450dc4b607bf6d1aa267a16d8ff8180c998f Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Mon, 15 Jul 2024 16:33:56 +0200 Subject: [PATCH 107/194] remove page_service `show ` (#8372) This operation isn't used in practice, so let's remove it. Context: in https://github.com/neondatabase/neon/pull/8339 --- pageserver/src/metrics.rs | 1 - pageserver/src/page_service.rs | 60 ---------------- test_runner/regress/test_auth.py | 2 +- test_runner/regress/test_tenant_conf.py | 96 ++----------------------- 4 files changed, 5 insertions(+), 154 deletions(-) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 9b3bb481b9ae..abad4b44b802 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -1490,7 +1490,6 @@ pub(crate) enum ComputeCommandKind { Basebackup, Fullbackup, LeaseLsn, - Show, } pub(crate) struct ComputeCommandCounters { diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index f94b0d335e8e..00147a8ca6cd 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -1479,66 +1479,6 @@ where ))? } }; - } else if let Some(params) = parts.strip_prefix(&["show"]) { - // show - if params.len() != 1 { - return Err(QueryError::Other(anyhow::anyhow!( - "invalid param number for config command" - ))); - } - let tenant_id = TenantId::from_str(params[0]) - .with_context(|| format!("Failed to parse tenant id from {}", params[0]))?; - - tracing::Span::current().record("tenant_id", field::display(tenant_id)); - - self.check_permission(Some(tenant_id))?; - - COMPUTE_COMMANDS_COUNTERS - .for_command(ComputeCommandKind::Show) - .inc(); - - let tenant = self - .get_active_tenant_with_timeout( - tenant_id, - ShardSelector::Zero, - ACTIVE_TENANT_TIMEOUT, - ) - .await?; - pgb.write_message_noflush(&BeMessage::RowDescription(&[ - RowDescriptor::int8_col(b"checkpoint_distance"), - RowDescriptor::int8_col(b"checkpoint_timeout"), - RowDescriptor::int8_col(b"compaction_target_size"), - RowDescriptor::int8_col(b"compaction_period"), - RowDescriptor::int8_col(b"compaction_threshold"), - RowDescriptor::int8_col(b"gc_horizon"), - RowDescriptor::int8_col(b"gc_period"), - RowDescriptor::int8_col(b"image_creation_threshold"), - RowDescriptor::int8_col(b"pitr_interval"), - ]))? - .write_message_noflush(&BeMessage::DataRow(&[ - Some(tenant.get_checkpoint_distance().to_string().as_bytes()), - Some( - tenant - .get_checkpoint_timeout() - .as_secs() - .to_string() - .as_bytes(), - ), - Some(tenant.get_compaction_target_size().to_string().as_bytes()), - Some( - tenant - .get_compaction_period() - .as_secs() - .to_string() - .as_bytes(), - ), - Some(tenant.get_compaction_threshold().to_string().as_bytes()), - Some(tenant.get_gc_horizon().to_string().as_bytes()), - Some(tenant.get_gc_period().as_secs().to_string().as_bytes()), - Some(tenant.get_image_creation_threshold().to_string().as_bytes()), - Some(tenant.get_pitr_interval().as_secs().to_string().as_bytes()), - ]))? - .write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?; } else { return Err(QueryError::Other(anyhow::anyhow!( "unknown command {query_string}" diff --git a/test_runner/regress/test_auth.py b/test_runner/regress/test_auth.py index 922a21a99929..7cb85e3dd1b2 100644 --- a/test_runner/regress/test_auth.py +++ b/test_runner/regress/test_auth.py @@ -211,7 +211,7 @@ def op(): def check_pageserver(expect_success: bool, **conn_kwargs): check_connection( env.pageserver, - f"show {env.initial_tenant}", + f"pagestream {env.initial_tenant} {env.initial_timeline}", expect_success, **conn_kwargs, ) diff --git a/test_runner/regress/test_tenant_conf.py b/test_runner/regress/test_tenant_conf.py index 80fb2b55b8b2..1a8bc3b98363 100644 --- a/test_runner/regress/test_tenant_conf.py +++ b/test_runner/regress/test_tenant_conf.py @@ -1,10 +1,7 @@ import json -from contextlib import closing from typing import Any, Dict -import psycopg2.extras from fixtures.common_types import Lsn -from fixtures.log_helper import log from fixtures.neon_fixtures import ( NeonEnvBuilder, ) @@ -63,25 +60,6 @@ def set_some_nondefault_global_config(ps_cfg: Dict[str, Any]): # check the configuration of the default tenant # it should match global configuration - with closing(env.pageserver.connect()) as psconn: - with psconn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as pscur: - log.info(f"show {env.initial_tenant}") - pscur.execute(f"show {env.initial_tenant}") - res = pscur.fetchone() - assert res is not None - assert all( - i in res.items() - for i in { - "checkpoint_distance": 10000, - "compaction_target_size": 1048576, - "compaction_period": 20, - "compaction_threshold": 10, - "gc_horizon": 67108864, - "gc_period": 60 * 60, - "image_creation_threshold": 3, - "pitr_interval": 604800, # 7 days - }.items() - ), f"Unexpected res: {res}" default_tenant_config = http_client.tenant_config(tenant_id=env.initial_tenant) assert ( not default_tenant_config.tenant_specific_overrides @@ -103,25 +81,6 @@ def set_some_nondefault_global_config(ps_cfg: Dict[str, Any]): } # check the configuration of the new tenant - with closing(env.pageserver.connect()) as psconn: - with psconn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as pscur: - pscur.execute(f"show {tenant}") - res = pscur.fetchone() - log.info(f"res: {res}") - assert res is not None - assert all( - i in res.items() - for i in { - "checkpoint_distance": 20000, - "compaction_target_size": 1048576, - "compaction_period": 20, - "compaction_threshold": 10, - "gc_horizon": 67108864, - "gc_period": 30, - "image_creation_threshold": 3, - "pitr_interval": 604800, - }.items() - ), f"Unexpected res: {res}" new_tenant_config = http_client.tenant_config(tenant_id=tenant) new_specific_config = new_tenant_config.tenant_specific_overrides assert new_specific_config["checkpoint_distance"] == 20000 @@ -166,25 +125,6 @@ def set_some_nondefault_global_config(ps_cfg: Dict[str, Any]): conf=conf_update, ) - with closing(env.pageserver.connect()) as psconn: - with psconn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as pscur: - pscur.execute(f"show {tenant}") - res = pscur.fetchone() - log.info(f"after config res: {res}") - assert res is not None - assert all( - i in res.items() - for i in { - "checkpoint_distance": 15000, - "compaction_target_size": 1048576, - "compaction_period": 80, - "compaction_threshold": 10, - "gc_horizon": 67108864, - "gc_period": 80, - "image_creation_threshold": 2, - "pitr_interval": 604800, - }.items() - ), f"Unexpected res: {res}" updated_tenant_config = http_client.tenant_config(tenant_id=tenant) updated_specific_config = updated_tenant_config.tenant_specific_overrides assert updated_specific_config["checkpoint_distance"] == 15000 @@ -222,25 +162,6 @@ def set_some_nondefault_global_config(ps_cfg: Dict[str, Any]): env.pageserver.stop() env.pageserver.start() - with closing(env.pageserver.connect()) as psconn: - with psconn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as pscur: - pscur.execute(f"show {tenant}") - res = pscur.fetchone() - log.info(f"after restart res: {res}") - assert res is not None - assert all( - i in res.items() - for i in { - "checkpoint_distance": 15000, - "compaction_target_size": 1048576, - "compaction_period": 80, - "compaction_threshold": 10, - "gc_horizon": 67108864, - "gc_period": 80, - "image_creation_threshold": 2, - "pitr_interval": 604800, - }.items() - ), f"Unexpected res: {res}" restarted_tenant_config = http_client.tenant_config(tenant_id=tenant) assert ( restarted_tenant_config == updated_tenant_config @@ -283,19 +204,10 @@ def set_some_nondefault_global_config(ps_cfg: Dict[str, Any]): env.pageserver.stop() env.pageserver.start() - with closing(env.pageserver.connect()) as psconn: - with psconn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as pscur: - pscur.execute(f"show {tenant}") - res = pscur.fetchone() - log.info(f"after restart res: {res}") - assert res is not None - assert all( - i in res.items() - for i in { - "compaction_period": 20, - "pitr_interval": 60, - }.items() - ), f"Unexpected res: {res}" + restarted_final_tenant_config = http_client.tenant_config(tenant_id=tenant) + assert ( + restarted_final_tenant_config == final_tenant_config + ), "Updated config should not change after the restart" def test_creating_tenant_conf_after_attach(neon_env_builder: NeonEnvBuilder): From 324e4e008fe4994ec84f96312aead9430afa178c Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Mon, 15 Jul 2024 18:08:24 +0300 Subject: [PATCH 108/194] feat(storcon): timeline detach ancestor passthrough (#8353) Currently storage controller does not support forwarding timeline detach ancestor requests to pageservers. Add support for forwarding `PUT .../:tenant_id/timelines/:timeline_id/detach_ancestor`. Implement the support mostly as is, because the timeline detach ancestor will be made (mostly) idempotent in future PR. Cc: #6994 --- .../src/models/detach_ancestor.rs | 2 +- pageserver/client/src/mgmt_api.rs | 18 +++ storage_controller/src/http.rs | 26 ++++ storage_controller/src/pageserver_client.rs | 22 ++- storage_controller/src/service.rs | 140 ++++++++++++++++-- test_runner/fixtures/neon_fixtures.py | 2 +- .../regress/test_timeline_detach_ancestor.py | 97 +++++++++++- 7 files changed, 281 insertions(+), 26 deletions(-) diff --git a/libs/pageserver_api/src/models/detach_ancestor.rs b/libs/pageserver_api/src/models/detach_ancestor.rs index fc1f10e7345f..ae5a21bab91c 100644 --- a/libs/pageserver_api/src/models/detach_ancestor.rs +++ b/libs/pageserver_api/src/models/detach_ancestor.rs @@ -1,6 +1,6 @@ use utils::id::TimelineId; -#[derive(Default, serde::Serialize)] +#[derive(Debug, Default, PartialEq, serde::Serialize, serde::Deserialize)] pub struct AncestorDetached { pub reparented_timelines: Vec, } diff --git a/pageserver/client/src/mgmt_api.rs b/pageserver/client/src/mgmt_api.rs index e3ddb446fa2c..ac3ff1bb896a 100644 --- a/pageserver/client/src/mgmt_api.rs +++ b/pageserver/client/src/mgmt_api.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use bytes::Bytes; +use detach_ancestor::AncestorDetached; use pageserver_api::{models::*, shard::TenantShardId}; use reqwest::{IntoUrl, Method, StatusCode}; use utils::{ @@ -418,6 +419,23 @@ impl Client { } } + pub async fn timeline_detach_ancestor( + &self, + tenant_shard_id: TenantShardId, + timeline_id: TimelineId, + ) -> Result { + let uri = format!( + "{}/v1/tenant/{tenant_shard_id}/timeline/{timeline_id}/detach_ancestor", + self.mgmt_api_endpoint + ); + + self.request(Method::PUT, &uri, ()) + .await? + .json() + .await + .map_err(Error::ReceiveBody) + } + pub async fn tenant_reset(&self, tenant_shard_id: TenantShardId) -> Result<()> { let uri = format!( "{}/v1/tenant/{}/reset", diff --git a/storage_controller/src/http.rs b/storage_controller/src/http.rs index 3a62c0dd4ffb..9ddf98eb3bb6 100644 --- a/storage_controller/src/http.rs +++ b/storage_controller/src/http.rs @@ -330,6 +330,22 @@ async fn handle_tenant_timeline_delete( .await } +async fn handle_tenant_timeline_detach_ancestor( + service: Arc, + req: Request, +) -> Result, ApiError> { + let tenant_id: TenantId = parse_request_param(&req, "tenant_id")?; + check_permissions(&req, Scope::PageServerApi)?; + + let timeline_id: TimelineId = parse_request_param(&req, "timeline_id")?; + + let res = service + .tenant_timeline_detach_ancestor(tenant_id, timeline_id) + .await?; + + json_response(StatusCode::OK, res) +} + async fn handle_tenant_timeline_passthrough( service: Arc, req: Request, @@ -1006,6 +1022,16 @@ pub fn make_router( RequestName("v1_tenant_timeline"), ) }) + .put( + "/v1/tenant/:tenant_id/timeline/:timeline_id/detach_ancestor", + |r| { + tenant_service_handler( + r, + handle_tenant_timeline_detach_ancestor, + RequestName("v1_tenant_timeline_detach_ancestor"), + ) + }, + ) // Tenant detail GET passthrough to shard zero: .get("/v1/tenant/:tenant_id", |r| { tenant_service_handler( diff --git a/storage_controller/src/pageserver_client.rs b/storage_controller/src/pageserver_client.rs index 769aba80cad1..8d64201cd939 100644 --- a/storage_controller/src/pageserver_client.rs +++ b/storage_controller/src/pageserver_client.rs @@ -1,8 +1,9 @@ use pageserver_api::{ models::{ - LocationConfig, LocationConfigListResponse, PageserverUtilization, SecondaryProgress, - TenantScanRemoteStorageResponse, TenantShardSplitRequest, TenantShardSplitResponse, - TimelineCreateRequest, TimelineInfo, TopTenantShardsRequest, TopTenantShardsResponse, + detach_ancestor::AncestorDetached, LocationConfig, LocationConfigListResponse, + PageserverUtilization, SecondaryProgress, TenantScanRemoteStorageResponse, + TenantShardSplitRequest, TenantShardSplitResponse, TimelineCreateRequest, TimelineInfo, + TopTenantShardsRequest, TopTenantShardsResponse, }, shard::TenantShardId, }; @@ -226,6 +227,21 @@ impl PageserverClient { ) } + pub(crate) async fn timeline_detach_ancestor( + &self, + tenant_shard_id: TenantShardId, + timeline_id: TimelineId, + ) -> Result { + measured_request!( + "timeline_detach_ancestor", + crate::metrics::Method::Put, + &self.node_id_label, + self.inner + .timeline_detach_ancestor(tenant_shard_id, timeline_id) + .await + ) + } + pub(crate) async fn get_utilization(&self) -> Result { measured_request!( "utilization", diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index deaac83ea526..95522525cb6e 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -117,6 +117,7 @@ enum TenantOperations { TimelineCreate, TimelineDelete, AttachHook, + TimelineDetachAncestor, } #[derive(Clone, strum_macros::Display)] @@ -2376,18 +2377,18 @@ impl Service { tracing::info!("Doing time travel recovery for shard {tenant_shard_id}",); client - .tenant_time_travel_remote_storage( - tenant_shard_id, - ×tamp, - &done_if_after, - ) - .await - .map_err(|e| { - ApiError::InternalServerError(anyhow::anyhow!( - "Error doing time travel recovery for shard {tenant_shard_id} on node {}: {e}", - node - )) - })?; + .tenant_time_travel_remote_storage( + tenant_shard_id, + ×tamp, + &done_if_after, + ) + .await + .map_err(|e| { + ApiError::InternalServerError(anyhow::anyhow!( + "Error doing time travel recovery for shard {tenant_shard_id} on node {}: {e}", + node + )) + })?; } } Ok(()) @@ -2757,7 +2758,7 @@ impl Service { // Create timeline on remaining shards with number >0 if !targets.is_empty() { // If we had multiple shards, issue requests for the remainder now. - let jwt = self.config.jwt_token.clone(); + let jwt = &self.config.jwt_token; self.tenant_for_shards(targets, |tenant_shard_id: TenantShardId, node: Node| { let create_req = create_req.clone(); Box::pin(create_one(tenant_shard_id, node, jwt.clone(), create_req)) @@ -2768,6 +2769,115 @@ impl Service { Ok(timeline_info) } + pub(crate) async fn tenant_timeline_detach_ancestor( + &self, + tenant_id: TenantId, + timeline_id: TimelineId, + ) -> Result { + tracing::info!("Detaching timeline {tenant_id}/{timeline_id}",); + + let _tenant_lock = trace_shared_lock( + &self.tenant_op_locks, + tenant_id, + TenantOperations::TimelineDetachAncestor, + ) + .await; + + self.ensure_attached_wait(tenant_id).await?; + + let targets = { + let locked = self.inner.read().unwrap(); + let mut targets = Vec::new(); + + for (tenant_shard_id, shard) in + locked.tenants.range(TenantShardId::tenant_range(tenant_id)) + { + let node_id = shard.intent.get_attached().ok_or_else(|| { + ApiError::InternalServerError(anyhow::anyhow!("Shard not scheduled")) + })?; + let node = locked + .nodes + .get(&node_id) + .expect("Pageservers may not be deleted while referenced"); + + targets.push((*tenant_shard_id, node.clone())); + } + targets + }; + + if targets.is_empty() { + return Err(ApiError::NotFound( + anyhow::anyhow!("Tenant not found").into(), + )); + } + + async fn detach_one( + tenant_shard_id: TenantShardId, + timeline_id: TimelineId, + node: Node, + jwt: Option, + ) -> Result<(ShardNumber, models::detach_ancestor::AncestorDetached), ApiError> { + tracing::info!( + "Detaching timeline on shard {tenant_shard_id}/{timeline_id}, attached to node {node}", + ); + + let client = PageserverClient::new(node.get_id(), node.base_url(), jwt.as_deref()); + client + .timeline_detach_ancestor(tenant_shard_id, timeline_id) + .await + .map_err(|e| { + use mgmt_api::Error; + + match e { + // no ancestor (ever) + Error::ApiError(StatusCode::CONFLICT, msg) => { + ApiError::Conflict(format!("{node}: {msg}")) + } + // too many ancestors + Error::ApiError(StatusCode::BAD_REQUEST, msg) => { + ApiError::BadRequest(anyhow::anyhow!("{node}: {msg}")) + } + // rest can be mapped + other => passthrough_api_error(&node, other), + } + }) + .map(|res| (tenant_shard_id.shard_number, res)) + } + + // no shard needs to go first/last; the operation should be idempotent + // TODO: it would be great to ensure that all shards return the same error + let mut results = self + .tenant_for_shards(targets, |tenant_shard_id, node| { + futures::FutureExt::boxed(detach_one( + tenant_shard_id, + timeline_id, + node, + self.config.jwt_token.clone(), + )) + }) + .await?; + + let any = results.pop().expect("we must have at least one response"); + + // FIXME: the ordering is not stable yet on pageserver, should be (ancestor_lsn, + // TimelineId) + let mismatching = results + .iter() + .filter(|(_, res)| res != &any.1) + .collect::>(); + if !mismatching.is_empty() { + let matching = results.len() - mismatching.len(); + tracing::error!( + matching, + compared_against=?any, + ?mismatching, + "shards returned different results" + ); + } + + Ok(any.1) + } + /// Helper for concurrently calling a pageserver API on a number of shards, such as timeline creation. /// /// On success, the returned vector contains exactly the same number of elements as the input `locations`. @@ -2894,8 +3004,8 @@ impl Service { .await .map_err(|e| { ApiError::InternalServerError(anyhow::anyhow!( - "Error deleting timeline {timeline_id} on {tenant_shard_id} on node {node}: {e}", - )) + "Error deleting timeline {timeline_id} on {tenant_shard_id} on node {node}: {e}", + )) }) } diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 463e4a3b012e..90ed838e1db3 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2400,7 +2400,7 @@ def tenant_create( def locate(self, tenant_id: TenantId) -> list[dict[str, Any]]: """ - :return: list of {"shard_id": "", "node_id": int, "listen_pg_addr": str, "listen_pg_port": int, "listen_http_addr: str, "listen_http_port: int} + :return: list of {"shard_id": "", "node_id": int, "listen_pg_addr": str, "listen_pg_port": int, "listen_http_addr": str, "listen_http_port": int} """ response = self.request( "GET", diff --git a/test_runner/regress/test_timeline_detach_ancestor.py b/test_runner/regress/test_timeline_detach_ancestor.py index 606ce203cdc6..803fcac58357 100644 --- a/test_runner/regress/test_timeline_detach_ancestor.py +++ b/test_runner/regress/test_timeline_detach_ancestor.py @@ -11,11 +11,12 @@ from fixtures.neon_fixtures import ( NeonEnvBuilder, PgBin, + flush_ep_to_pageserver, wait_for_last_flush_lsn, ) from fixtures.pageserver.http import HistoricLayerInfo, PageserverApiException -from fixtures.pageserver.utils import wait_timeline_detail_404 -from fixtures.remote_storage import LocalFsStorage +from fixtures.pageserver.utils import wait_for_last_record_lsn, wait_timeline_detail_404 +from fixtures.remote_storage import LocalFsStorage, RemoteStorageKind from fixtures.utils import assert_pageserver_backups_equal @@ -559,11 +560,24 @@ def delta_layers(timeline_id: TimelineId): assert_pageserver_backups_equal(fullbackup_before, fullbackup_after, set()) -def test_timeline_ancestor_errors(neon_env_builder: NeonEnvBuilder): - env = neon_env_builder.init_start() - env.pageserver.allowed_errors.extend(SHUTDOWN_ALLOWED_ERRORS) +@pytest.mark.parametrize("sharded", [True, False]) +def test_timeline_ancestor_detach_errors(neon_env_builder: NeonEnvBuilder, sharded: bool): + shards = 2 if sharded else 1 - client = env.pageserver.http_client() + neon_env_builder.num_pageservers = shards + env = neon_env_builder.init_start(initial_tenant_shard_count=shards if sharded else None) + + pageservers = dict((int(p.id), p) for p in env.pageservers) + + for ps in pageservers.values(): + ps.allowed_errors.extend(SHUTDOWN_ALLOWED_ERRORS) + + if sharded: + # FIXME: should this be in the neon_env_builder.init_start? + env.storage_controller.reconcile_until_idle() + client = env.storage_controller.pageserver_api() + else: + client = env.pageserver.http_client() with pytest.raises(PageserverApiException, match=".* no ancestors") as info: client.detach_ancestor(env.initial_tenant, env.initial_timeline) @@ -577,6 +591,17 @@ def test_timeline_ancestor_errors(neon_env_builder: NeonEnvBuilder): client.detach_ancestor(env.initial_tenant, second_branch) assert info.value.status_code == 400 + client.detach_ancestor(env.initial_tenant, first_branch) + + # FIXME: this should be done by the http req handler + for ps in pageservers.values(): + ps.quiesce_tenants() + + with pytest.raises(PageserverApiException, match=".* no ancestors") as info: + client.detach_ancestor(env.initial_tenant, first_branch) + # FIXME: this should be 200 OK because we've already completed it + assert info.value.status_code == 409 + client.tenant_delete(env.initial_tenant) with pytest.raises(PageserverApiException) as e: @@ -584,6 +609,58 @@ def test_timeline_ancestor_errors(neon_env_builder: NeonEnvBuilder): assert e.value.status_code == 404 +def test_sharded_timeline_detach_ancestor(neon_env_builder: NeonEnvBuilder): + branch_name = "soon_detached" + shard_count = 4 + neon_env_builder.num_pageservers = shard_count + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.MOCK_S3) + + env = neon_env_builder.init_start(initial_tenant_shard_count=shard_count) + for ps in env.pageservers: + ps.allowed_errors.extend(SHUTDOWN_ALLOWED_ERRORS) + + # FIXME: should this be in the neon_env_builder.init_start? + env.storage_controller.reconcile_until_idle() + shards = env.storage_controller.locate(env.initial_tenant) + + branch_timeline_id = env.neon_cli.create_branch(branch_name, tenant_id=env.initial_tenant) + + with env.endpoints.create_start(branch_name, tenant_id=env.initial_tenant) as ep: + ep.safe_psql( + "create table foo as select 1::bigint, i::bigint from generate_series(1, 10000) v(i)" + ) + lsn = flush_ep_to_pageserver(env, ep, env.initial_tenant, branch_timeline_id) + + pageservers = dict((int(p.id), p) for p in env.pageservers) + + for shard_info in shards: + node_id = int(shard_info["node_id"]) + shard_id = shard_info["shard_id"] + detail = pageservers[node_id].http_client().timeline_detail(shard_id, branch_timeline_id) + + assert Lsn(detail["last_record_lsn"]) >= lsn + assert Lsn(detail["initdb_lsn"]) < lsn + assert TimelineId(detail["ancestor_timeline_id"]) == env.initial_timeline + + env.storage_controller.pageserver_api().detach_ancestor(env.initial_tenant, branch_timeline_id) + + for shard_info in shards: + node_id = int(shard_info["node_id"]) + shard_id = shard_info["shard_id"] + + # TODO: ensure quescing is done on pageserver? + pageservers[node_id].quiesce_tenants() + detail = pageservers[node_id].http_client().timeline_detail(shard_id, branch_timeline_id) + wait_for_last_record_lsn( + pageservers[node_id].http_client(), shard_id, branch_timeline_id, lsn + ) + assert detail.get("ancestor_timeline_id") is None + + with env.endpoints.create_start(branch_name, tenant_id=env.initial_tenant) as ep: + count = int(ep.safe_psql("select count(*) from foo")[0][0]) + assert count == 10000 + + # TODO: # - after starting the operation, tenant is deleted # - after starting the operation, pageserver is shutdown, restarted @@ -591,3 +668,11 @@ def test_timeline_ancestor_errors(neon_env_builder: NeonEnvBuilder): # - deletion of reparented while reparenting should fail once, then succeed (?) # - branch near existing L1 boundary, image layers? # - investigate: why are layers started at uneven lsn? not just after branching, but in general. +# +# TEST: 1. tad which partially succeeds, one returns 500 +# 2. create branch below timeline? or delete timeline below +# 3. on retry all should report the same reparented timelines +# +# TEST: 1. tad is started, one node stalls, other restarts +# 2. client timeout before stall over +# 3. on retry with stalled and other being able to proceed From 04448ac3231deaae9f418812b96c60ed2bfa5bd1 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 15 Jul 2024 17:43:05 +0100 Subject: [PATCH 109/194] pageserver: use PITR GC cutoffs as authoritative (#8365) ## Problem Pageserver GC uses a size-based condition (GC "horizon" in addition to time-based "PITR"). Eventually we plan to retire the size-based condition: https://github.com/neondatabase/neon/issues/6374 Currently, we always apply the more conservative of the two, meaning that tenants always retain at least 64MB of history (default horizon), even after a very long time has passed. This is particularly acute in cases where someone has dropped tables/databases, and then leaves a database idle: the horizon can prevent GCing very large quantities of historical data (we already account for this in synthetic size by ignoring gc horizon). We're not entirely removing GC horizon right now because we don't want to 100% rely on standby_horizon for robustness of physical replication, but we can tweak our logic to avoid retaining that 64MB LSN length indefinitely. ## Summary of changes - Rework `Timeline::find_gc_cutoffs`, with new logic: - If there is no PITR set, then use `DEFAULT_PITR_INTERVAL` (1 week) to calculate a time threshold. Retain either the horizon or up to that thresholds, whichever requires less data. - When there is a PITR set, and we have unambiguously resolved the timestamp to an LSN, then ignore the GC horizon entirely. For typical PITRs (1 day, 1 week), this will still easily retain enough data to avoid stressing read only replicas. The key property we end up with, whether a PITR is set or not, is that after enough time has passed, our GC cutoff on an idle timeline will catch up with the last_record_lsn. Using `DEFAULT_PITR_INTERVAL` is a bit of an arbitrary hack, but this feels like it isn't really worth the noise of exposing in TenantConfig. We could just make it a different named constant though. The end-end state will be that there is no gc_horizon at all, and that tenants with pitr_interval=0 would truly retain no history, so this constant would go away. --- pageserver/src/tenant/timeline.rs | 144 +++++++++++++--------- test_runner/regress/test_branch_and_gc.py | 4 +- 2 files changed, 88 insertions(+), 60 deletions(-) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index a3ddb3a1d190..0996616a670e 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -69,6 +69,7 @@ use std::{ use crate::{ aux_file::AuxFileSizeEstimator, tenant::{ + config::defaults::DEFAULT_PITR_INTERVAL, layer_map::{LayerMap, SearchResult}, metadata::TimelineMetadata, storage_layer::PersistentLayerDesc, @@ -4945,20 +4946,17 @@ impl Timeline { } /// Find the Lsns above which layer files need to be retained on - /// garbage collection. This is separate from actually performing the GC, - /// and is updated more frequently, so that compaction can remove obsolete - /// page versions more aggressively. + /// garbage collection. /// - /// TODO: that's wishful thinking, compaction doesn't actually do that - /// currently. + /// We calculate two cutoffs, one based on time and one based on WAL size. `pitr` + /// controls the time cutoff (or ZERO to disable time-based retention), and `cutoff_horizon` controls + /// the space-based retention. /// - /// The 'cutoff_horizon' point is used to retain recent versions that might still be - /// needed by read-only nodes. (As of this writing, the caller just passes - /// the latest LSN subtracted by a constant, and doesn't do anything smart - /// to figure out what read-only nodes might actually need.) - /// - /// The 'pitr' duration is used to calculate a 'pitr_cutoff', which can be used to determine - /// whether a record is needed for PITR. + /// This function doesn't simply to calculate time & space based retention: it treats time-based + /// retention as authoritative if enabled, and falls back to space-based retention if calculating + /// the LSN for a time point isn't possible. Therefore the GcCutoffs::horizon in the response might + /// be different to the `cutoff_horizon` input. Callers should treat the min() of the two cutoffs + /// in the response as the GC cutoff point for the timeline. #[instrument(skip_all, fields(timeline_id=%self.timeline_id))] pub(super) async fn find_gc_cutoffs( &self, @@ -4975,58 +4973,88 @@ impl Timeline { pausable_failpoint!("Timeline::find_gc_cutoffs-pausable"); - // First, calculate pitr_cutoff_timestamp and then convert it to LSN. - // - // Some unit tests depend on garbage-collection working even when - // CLOG data is missing, so that find_lsn_for_timestamp() doesn't - // work, so avoid calling it altogether if time-based retention is not - // configured. It would be pointless anyway. - let pitr_cutoff = if pitr != Duration::ZERO { + if cfg!(test) { + // Unit tests which specify zero PITR interval expect to avoid doing any I/O for timestamp lookup + if pitr == Duration::ZERO { + return Ok(GcCutoffs { + pitr: self.get_last_record_lsn(), + horizon: cutoff_horizon, + }); + } + } + + // Calculate a time-based limit on how much to retain: + // - if PITR interval is set, then this is our cutoff. + // - if PITR interval is not set, then we do a lookup + // based on DEFAULT_PITR_INTERVAL, so that size-based retention (horizon) + // does not result in keeping history around permanently on idle databases. + let time_cutoff = { let now = SystemTime::now(); - if let Some(pitr_cutoff_timestamp) = now.checked_sub(pitr) { - let pitr_timestamp = to_pg_timestamp(pitr_cutoff_timestamp); + let time_range = if pitr == Duration::ZERO { + humantime::parse_duration(DEFAULT_PITR_INTERVAL).expect("constant is invalid") + } else { + pitr + }; - match self - .find_lsn_for_timestamp(pitr_timestamp, cancel, ctx) - .await? - { - LsnForTimestamp::Present(lsn) => lsn, - LsnForTimestamp::Future(lsn) => { - // The timestamp is in the future. That sounds impossible, - // but what it really means is that there hasn't been - // any commits since the cutoff timestamp. - // - // In this case we should use the LSN of the most recent commit, - // which is implicitly the last LSN in the log. - debug!("future({})", lsn); - self.get_last_record_lsn() - } - LsnForTimestamp::Past(lsn) => { - debug!("past({})", lsn); - // conservative, safe default is to remove nothing, when we - // have no commit timestamp data available - *self.get_latest_gc_cutoff_lsn() - } - LsnForTimestamp::NoData(lsn) => { - debug!("nodata({})", lsn); - // conservative, safe default is to remove nothing, when we - // have no commit timestamp data available - *self.get_latest_gc_cutoff_lsn() - } + // If PITR is so large or `now` is so small that this underflows, we will retain no history (highly unexpected case) + let time_cutoff = now.checked_sub(time_range).unwrap_or(now); + let timestamp = to_pg_timestamp(time_cutoff); + + match self.find_lsn_for_timestamp(timestamp, cancel, ctx).await? { + LsnForTimestamp::Present(lsn) => Some(lsn), + LsnForTimestamp::Future(lsn) => { + // The timestamp is in the future. That sounds impossible, + // but what it really means is that there hasn't been + // any commits since the cutoff timestamp. + // + // In this case we should use the LSN of the most recent commit, + // which is implicitly the last LSN in the log. + debug!("future({})", lsn); + Some(self.get_last_record_lsn()) + } + LsnForTimestamp::Past(lsn) => { + debug!("past({})", lsn); + None + } + LsnForTimestamp::NoData(lsn) => { + debug!("nodata({})", lsn); + None } - } else { - // If we don't have enough data to convert to LSN, - // play safe and don't remove any layers. - *self.get_latest_gc_cutoff_lsn() } - } else { - // No time-based retention was configured. Interpret this as "keep no history". - self.get_last_record_lsn() }; - Ok(GcCutoffs { - horizon: cutoff_horizon, - pitr: pitr_cutoff, + Ok(match (pitr, time_cutoff) { + (Duration::ZERO, Some(time_cutoff)) => { + // PITR is not set. Retain the size-based limit, or the default time retention, + // whichever requires less data. + GcCutoffs { + pitr: std::cmp::max(time_cutoff, cutoff_horizon), + horizon: std::cmp::max(time_cutoff, cutoff_horizon), + } + } + (Duration::ZERO, None) => { + // PITR is not set, and time lookup failed + GcCutoffs { + pitr: self.get_last_record_lsn(), + horizon: cutoff_horizon, + } + } + (_, None) => { + // PITR interval is set & we didn't look up a timestamp successfully. Conservatively assume PITR + // cannot advance beyond what was already GC'd, and respect space-based retention + GcCutoffs { + pitr: *self.get_latest_gc_cutoff_lsn(), + horizon: cutoff_horizon, + } + } + (_, Some(time_cutoff)) => { + // PITR interval is set and we looked up timestamp successfully. Ignore + // size based retention and make time cutoff authoritative + GcCutoffs { + pitr: time_cutoff, + horizon: time_cutoff, + } + } }) } diff --git a/test_runner/regress/test_branch_and_gc.py b/test_runner/regress/test_branch_and_gc.py index eb503ddbfa0d..f2e3855c123e 100644 --- a/test_runner/regress/test_branch_and_gc.py +++ b/test_runner/regress/test_branch_and_gc.py @@ -65,8 +65,8 @@ def test_branch_and_gc(neon_simple_env: NeonEnv, build_type: str): "compaction_period": "1 s", "compaction_threshold": "2", "image_creation_threshold": "1", - # set PITR interval to be small, so we can do GC - "pitr_interval": "1 s", + # Disable PITR, this test will set an explicit space-based GC limit + "pitr_interval": "0 s", } ) From 730db859c741f6e782f721de12e8ec776c4ceb0a Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Mon, 15 Jul 2024 20:47:53 +0300 Subject: [PATCH 110/194] feat(timeline_detach_ancestor): success idempotency (#8354) Right now timeline detach ancestor reports an error (409, "no ancestor") on a new attempt after successful completion. This makes it troublesome for storage controller retries. Fix it to respond with `200 OK` as if the operation had just completed quickly. Additionally, the returned timeline identifiers in the 200 OK response are now ordered so that responses between different nodes for error comparison are done by the storage controller added in #8353. Design-wise, this PR introduces a new strategy for accessing the latest uploaded IndexPart: `RemoteTimelineClient::initialized_upload_queue(&self) -> Result, NotInitialized>`. It should be a more scalable way to query the latest uploaded `IndexPart` than to add a query method for each question directly on `RemoteTimelineClient`. GC blocking will need to be introduced to make the operation fully idempotent. However, it is idempotent for the cases demonstrated by tests. Cc: #6994 --- pageserver/src/http/routes.rs | 47 +- .../src/tenant/remote_timeline_client.rs | 27 +- .../tenant/remote_timeline_client/index.rs | 26 ++ pageserver/src/tenant/timeline.rs | 8 +- .../src/tenant/timeline/detach_ancestor.rs | 130 +++++- pageserver/src/tenant/upload_queue.rs | 10 +- storage_controller/src/service.rs | 9 +- test_runner/fixtures/pageserver/http.py | 21 +- .../regress/test_timeline_detach_ancestor.py | 430 ++++++++++++++++-- 9 files changed, 633 insertions(+), 75 deletions(-) diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 6f8f3e6389d5..d7ef70477f45 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -1721,7 +1721,9 @@ async fn timeline_detach_ancestor_handler( request: Request, _cancel: CancellationToken, ) -> Result, ApiError> { - use crate::tenant::timeline::detach_ancestor::Options; + use crate::tenant::timeline::detach_ancestor; + use pageserver_api::models::detach_ancestor::AncestorDetached; + let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?; check_permission(&request, Some(tenant_shard_id.tenant_id))?; let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; @@ -1729,7 +1731,7 @@ async fn timeline_detach_ancestor_handler( let span = tracing::info_span!("detach_ancestor", tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), %timeline_id); async move { - let mut options = Options::default(); + let mut options = detach_ancestor::Options::default(); let rewrite_concurrency = parse_query_param::<_, std::num::NonZeroUsize>(&request, "rewrite_concurrency")?; @@ -1757,27 +1759,36 @@ async fn timeline_detach_ancestor_handler( let timeline = tenant.get_timeline(timeline_id, true)?; - let (_guard, prepared) = timeline + let progress = timeline .prepare_to_detach_from_ancestor(&tenant, options, ctx) .await?; - let res = state - .tenant_manager - .complete_detaching_timeline_ancestor(tenant_shard_id, timeline_id, prepared, ctx) - .await; - - match res { - Ok(reparented_timelines) => { - let resp = pageserver_api::models::detach_ancestor::AncestorDetached { + // uncomment to allow early as possible Tenant::drop + // drop(tenant); + + let resp = match progress { + detach_ancestor::Progress::Prepared(_guard, prepared) => { + // it would be great to tag the guard on to the tenant activation future + let reparented_timelines = state + .tenant_manager + .complete_detaching_timeline_ancestor( + tenant_shard_id, + timeline_id, + prepared, + ctx, + ) + .await + .context("timeline detach ancestor completion") + .map_err(ApiError::InternalServerError)?; + + AncestorDetached { reparented_timelines, - }; - - json_response(StatusCode::OK, resp) + } } - Err(e) => Err(ApiError::InternalServerError( - e.context("timeline detach completion"), - )), - } + detach_ancestor::Progress::Done(resp) => resp, + }; + + json_response(StatusCode::OK, resp) } .instrument(span) .await diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index bc9364de61d4..66b759c8e0d8 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -241,7 +241,7 @@ use self::index::IndexPart; use super::metadata::MetadataUpdate; use super::storage_layer::{Layer, LayerName, ResidentLayer}; -use super::upload_queue::SetDeletedFlagProgress; +use super::upload_queue::{NotInitialized, SetDeletedFlagProgress}; use super::Generation; pub(crate) use download::{ @@ -1930,6 +1930,31 @@ impl RemoteTimelineClient { } } } + + /// Returns an accessor which will hold the UploadQueue mutex for accessing the upload queue + /// externally to RemoteTimelineClient. + pub(crate) fn initialized_upload_queue( + &self, + ) -> Result, NotInitialized> { + let mut inner = self.upload_queue.lock().unwrap(); + inner.initialized_mut()?; + Ok(UploadQueueAccessor { inner }) + } +} + +pub(crate) struct UploadQueueAccessor<'a> { + inner: std::sync::MutexGuard<'a, UploadQueue>, +} + +impl<'a> UploadQueueAccessor<'a> { + pub(crate) fn latest_uploaded_index_part(&self) -> &IndexPart { + match &*self.inner { + UploadQueue::Initialized(x) => &x.clean.0, + UploadQueue::Uninitialized | UploadQueue::Stopped(_) => { + unreachable!("checked before constructing") + } + } + } } pub fn remote_tenant_path(tenant_shard_id: &TenantShardId) -> RemotePath { diff --git a/pageserver/src/tenant/remote_timeline_client/index.rs b/pageserver/src/tenant/remote_timeline_client/index.rs index 6233a3477e4d..b439df8edb10 100644 --- a/pageserver/src/tenant/remote_timeline_client/index.rs +++ b/pageserver/src/tenant/remote_timeline_client/index.rs @@ -176,6 +176,24 @@ pub(crate) struct Lineage { /// /// If you are adding support for detaching from a hierarchy, consider changing the ancestry /// into a `Vec<(TimelineId, Lsn)>` to be a path instead. + // FIXME: this is insufficient even for path of two timelines for future wal recovery + // purposes: + // + // assuming a "old main" which has received most of the WAL, and has a branch "new main", + // starting a bit before "old main" last_record_lsn. the current version works fine, + // because we will know to replay wal and branch at the recorded Lsn to do wal recovery. + // + // then assuming "new main" would similarly receive a branch right before its last_record_lsn, + // "new new main". the current implementation would just store ("new main", ancestor_lsn, _) + // here. however, we cannot recover from WAL using only that information, we would need the + // whole ancestry here: + // + // ```json + // [ + // ["old main", ancestor_lsn("new main"), _], + // ["new main", ancestor_lsn("new new main"), _] + // ] + // ``` #[serde(skip_serializing_if = "Option::is_none", default)] original_ancestor: Option<(TimelineId, Lsn, NaiveDateTime)>, } @@ -217,6 +235,14 @@ impl Lineage { self.original_ancestor .is_some_and(|(_, ancestor_lsn, _)| ancestor_lsn == lsn) } + + pub(crate) fn is_detached_from_original_ancestor(&self) -> bool { + self.original_ancestor.is_some() + } + + pub(crate) fn is_reparented(&self) -> bool { + !self.reparenting_history.is_empty() + } } #[cfg(test)] diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 0996616a670e..239dce878640 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -4733,13 +4733,7 @@ impl Timeline { tenant: &crate::tenant::Tenant, options: detach_ancestor::Options, ctx: &RequestContext, - ) -> Result< - ( - completion::Completion, - detach_ancestor::PreparedTimelineDetach, - ), - detach_ancestor::Error, - > { + ) -> Result { detach_ancestor::prepare(self, tenant, options, ctx).await } diff --git a/pageserver/src/tenant/timeline/detach_ancestor.rs b/pageserver/src/tenant/timeline/detach_ancestor.rs index 4fc89330ba42..49ce3db3e63d 100644 --- a/pageserver/src/tenant/timeline/detach_ancestor.rs +++ b/pageserver/src/tenant/timeline/detach_ancestor.rs @@ -10,6 +10,7 @@ use crate::{ }, virtual_file::{MaybeFatalIo, VirtualFile}, }; +use pageserver_api::models::detach_ancestor::AncestorDetached; use tokio_util::sync::CancellationToken; use tracing::Instrument; use utils::{completion, generation::Generation, http::error::ApiError, id::TimelineId, lsn::Lsn}; @@ -39,6 +40,9 @@ pub(crate) enum Error { #[error("unexpected error")] Unexpected(#[source] anyhow::Error), + + #[error("failpoint: {}", .0)] + Failpoint(&'static str), } impl From for ApiError { @@ -57,11 +61,41 @@ impl From for ApiError { | e @ Error::CopyDeltaPrefix(_) | e @ Error::UploadRewritten(_) | e @ Error::CopyFailed(_) - | e @ Error::Unexpected(_) => ApiError::InternalServerError(e.into()), + | e @ Error::Unexpected(_) + | e @ Error::Failpoint(_) => ApiError::InternalServerError(e.into()), + } + } +} + +impl From for Error { + fn from(_: crate::tenant::upload_queue::NotInitialized) -> Self { + // treat all as shutting down signals, even though that is not entirely correct + // (uninitialized state) + Error::ShuttingDown + } +} + +impl From for Error { + fn from(value: FlushLayerError) -> Self { + match value { + FlushLayerError::Cancelled => Error::ShuttingDown, + FlushLayerError::NotRunning(_) => { + // FIXME(#6424): technically statically unreachable right now, given how we never + // drop the sender + Error::ShuttingDown + } + FlushLayerError::CreateImageLayersError(_) | FlushLayerError::Other(_) => { + Error::FlushAncestor(value) + } } } } +pub(crate) enum Progress { + Prepared(completion::Completion, PreparedTimelineDetach), + Done(AncestorDetached), +} + pub(crate) struct PreparedTimelineDetach { layers: Vec, } @@ -88,7 +122,7 @@ pub(super) async fn prepare( tenant: &Tenant, options: Options, ctx: &RequestContext, -) -> Result<(completion::Completion, PreparedTimelineDetach), Error> { +) -> Result { use Error::*; let Some((ancestor, ancestor_lsn)) = detached @@ -96,15 +130,67 @@ pub(super) async fn prepare( .as_ref() .map(|tl| (tl.clone(), detached.ancestor_lsn)) else { - // TODO: check if we have already been detached; for this we need to read the stored data - // on remote client, for that we need a follow-up which makes uploads cheaper and maintains - // a projection of the commited data. + { + let accessor = detached.remote_client.initialized_upload_queue()?; + + // we are safe to inspect the latest uploaded, because we can only witness this after + // restart is complete and ancestor is no more. + let latest = accessor.latest_uploaded_index_part(); + if !latest.lineage.is_detached_from_original_ancestor() { + return Err(NoAncestor); + } + } + + // detached has previously been detached; let's inspect each of the current timelines and + // report back the timelines which have been reparented by our detach + let mut all_direct_children = tenant + .timelines + .lock() + .unwrap() + .values() + .filter(|tl| matches!(tl.ancestor_timeline.as_ref(), Some(ancestor) if Arc::ptr_eq(ancestor, detached))) + .map(|tl| (tl.ancestor_lsn, tl.clone())) + .collect::>(); + + let mut any_shutdown = false; + + all_direct_children.retain( + |(_, tl)| match tl.remote_client.initialized_upload_queue() { + Ok(accessor) => accessor + .latest_uploaded_index_part() + .lineage + .is_reparented(), + Err(_shutdownalike) => { + // not 100% a shutdown, but let's bail early not to give inconsistent results in + // sharded enviroment. + any_shutdown = true; + true + } + }, + ); + + if any_shutdown { + // it could be one or many being deleted; have client retry + return Err(Error::ShuttingDown); + } + + let mut reparented = all_direct_children; + // why this instead of hashset? there is a reason, but I've forgotten it many times. // - // the error is wrong per openapi - return Err(NoAncestor); + // maybe if this was a hashset we would not be able to distinguish some race condition. + reparented.sort_unstable_by_key(|(lsn, tl)| (*lsn, tl.timeline_id)); + + return Ok(Progress::Done(AncestorDetached { + reparented_timelines: reparented + .into_iter() + .map(|(_, tl)| tl.timeline_id) + .collect(), + })); }; if !ancestor_lsn.is_valid() { + // rare case, probably wouldn't even load + tracing::error!("ancestor is set, but ancestor_lsn is invalid, this timeline needs fixing"); return Err(NoAncestor); } @@ -131,6 +217,15 @@ pub(super) async fn prepare( let _gate_entered = detached.gate.enter().map_err(|_| ShuttingDown)?; + utils::pausable_failpoint!("timeline-detach-ancestor::before_starting_after_locking_pausable"); + + fail::fail_point!( + "timeline-detach-ancestor::before_starting_after_locking", + |_| Err(Error::Failpoint( + "timeline-detach-ancestor::before_starting_after_locking" + )) + ); + if ancestor_lsn >= ancestor.get_disk_consistent_lsn() { let span = tracing::info_span!("freeze_and_flush", ancestor_timeline_id=%ancestor.timeline_id); @@ -151,7 +246,7 @@ pub(super) async fn prepare( } }; - res.map_err(FlushAncestor)?; + res?; // we do not need to wait for uploads to complete but we do need `struct Layer`, // copying delta prefix is unsupported currently for `InMemoryLayer`. @@ -159,7 +254,7 @@ pub(super) async fn prepare( elapsed_ms = started_at.elapsed().as_millis(), "froze and flushed the ancestor" ); - Ok(()) + Ok::<_, Error>(()) } .instrument(span) .await?; @@ -283,7 +378,7 @@ pub(super) async fn prepare( let prepared = PreparedTimelineDetach { layers: new_layers }; - Ok((guard, prepared)) + Ok(Progress::Prepared(guard, prepared)) } fn partition_work( @@ -350,7 +445,11 @@ async fn copy_lsn_prefix( target_timeline: &Arc, ctx: &RequestContext, ) -> Result, Error> { - use Error::{CopyDeltaPrefix, RewrittenDeltaDownloadFailed}; + use Error::{CopyDeltaPrefix, RewrittenDeltaDownloadFailed, ShuttingDown}; + + if target_timeline.cancel.is_cancelled() { + return Err(ShuttingDown); + } tracing::debug!(%layer, %end_lsn, "copying lsn prefix"); @@ -529,7 +628,7 @@ pub(super) async fn complete( match res { Ok(Some(timeline)) => { tracing::info!(reparented=%timeline.timeline_id, "reparenting done"); - reparented.push(timeline.timeline_id); + reparented.push((timeline.ancestor_lsn, timeline.timeline_id)); } Ok(None) => { // lets just ignore this for now. one or all reparented timelines could had @@ -551,5 +650,12 @@ pub(super) async fn complete( tracing::info!("failed to reparent some candidates"); } + reparented.sort_unstable(); + + let reparented = reparented + .into_iter() + .map(|(_, timeline_id)| timeline_id) + .collect(); + Ok(reparented) } diff --git a/pageserver/src/tenant/upload_queue.rs b/pageserver/src/tenant/upload_queue.rs index 50c977a950fe..f7440ecdae12 100644 --- a/pageserver/src/tenant/upload_queue.rs +++ b/pageserver/src/tenant/upload_queue.rs @@ -228,18 +228,20 @@ impl UploadQueue { Ok(self.initialized_mut().expect("we just set it")) } - pub(crate) fn initialized_mut(&mut self) -> anyhow::Result<&mut UploadQueueInitialized> { + pub(crate) fn initialized_mut( + &mut self, + ) -> Result<&mut UploadQueueInitialized, NotInitialized> { use UploadQueue::*; match self { - Uninitialized => Err(NotInitialized::Uninitialized.into()), + Uninitialized => Err(NotInitialized::Uninitialized), Initialized(x) => { if x.shutting_down { - Err(NotInitialized::ShuttingDown.into()) + Err(NotInitialized::ShuttingDown) } else { Ok(x) } } - Stopped(_) => Err(NotInitialized::Stopped.into()), + Stopped(_) => Err(NotInitialized::Stopped), } } diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 95522525cb6e..3c24433c422a 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -2830,9 +2830,10 @@ impl Service { match e { // no ancestor (ever) - Error::ApiError(StatusCode::CONFLICT, msg) => { - ApiError::Conflict(format!("{node}: {msg}")) - } + Error::ApiError(StatusCode::CONFLICT, msg) => ApiError::Conflict(format!( + "{node}: {}", + msg.strip_prefix("Conflict: ").unwrap_or(&msg) + )), // too many ancestors Error::ApiError(StatusCode::BAD_REQUEST, msg) => { ApiError::BadRequest(anyhow::anyhow!("{node}: {msg}")) @@ -2859,8 +2860,6 @@ impl Service { let any = results.pop().expect("we must have at least one response"); - // FIXME: the ordering is not stable yet on pageserver, should be (ancestor_lsn, - // TimelineId) let mismatching = results .iter() .filter(|(_, res)| res != &any.1) diff --git a/test_runner/fixtures/pageserver/http.py b/test_runner/fixtures/pageserver/http.py index 03aee9e5c597..d66b94948a8e 100644 --- a/test_runner/fixtures/pageserver/http.py +++ b/test_runner/fixtures/pageserver/http.py @@ -172,6 +172,21 @@ def __init__( if auth_token is not None: self.headers["Authorization"] = f"Bearer {auth_token}" + def without_status_retrying(self) -> PageserverHttpClient: + retries = Retry( + status=0, + connect=5, + read=False, + backoff_factor=0.2, + status_forcelist=[], + allowed_methods=None, + remove_headers_on_redirect=[], + ) + + return PageserverHttpClient( + self.port, self.is_testing_enabled_or_skip, self.auth_token, retries + ) + @property def base_url(self) -> str: return f"http://localhost:{self.port}" @@ -814,17 +829,19 @@ def detach_ancestor( tenant_id: Union[TenantId, TenantShardId], timeline_id: TimelineId, batch_size: int | None = None, - ) -> Set[TimelineId]: + **kwargs, + ) -> List[TimelineId]: params = {} if batch_size is not None: params["batch_size"] = batch_size res = self.put( f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/detach_ancestor", params=params, + **kwargs, ) self.verbose_error(res) json = res.json() - return set(map(TimelineId, json["reparented_timelines"])) + return list(map(TimelineId, json["reparented_timelines"])) def evict_layer( self, tenant_id: Union[TenantId, TenantShardId], timeline_id: TimelineId, layer_name: str diff --git a/test_runner/regress/test_timeline_detach_ancestor.py b/test_runner/regress/test_timeline_detach_ancestor.py index 803fcac58357..d75ab4c0604f 100644 --- a/test_runner/regress/test_timeline_detach_ancestor.py +++ b/test_runner/regress/test_timeline_detach_ancestor.py @@ -1,5 +1,7 @@ import datetime import enum +import threading +import time from concurrent.futures import ThreadPoolExecutor from queue import Empty, Queue from threading import Barrier @@ -9,6 +11,7 @@ from fixtures.common_types import Lsn, TimelineId from fixtures.log_helper import log from fixtures.neon_fixtures import ( + LogCursor, NeonEnvBuilder, PgBin, flush_ep_to_pageserver, @@ -17,7 +20,8 @@ from fixtures.pageserver.http import HistoricLayerInfo, PageserverApiException from fixtures.pageserver.utils import wait_for_last_record_lsn, wait_timeline_detail_404 from fixtures.remote_storage import LocalFsStorage, RemoteStorageKind -from fixtures.utils import assert_pageserver_backups_equal +from fixtures.utils import assert_pageserver_backups_equal, wait_until +from requests import ReadTimeout def by_end_lsn(info: HistoricLayerInfo) -> Lsn: @@ -161,7 +165,7 @@ def test_ancestor_detach_branched_from( ) all_reparented = client.detach_ancestor(env.initial_tenant, timeline_id) - assert all_reparented == set() + assert all_reparented == [] if restart_after: env.pageserver.stop() @@ -270,7 +274,7 @@ def test_ancestor_detach_reparents_earlier(neon_env_builder: NeonEnvBuilder): after = env.neon_cli.create_branch("after", "main", env.initial_tenant, ancestor_start_lsn=None) all_reparented = client.detach_ancestor(env.initial_tenant, timeline_id) - assert all_reparented == {reparented, same_branchpoint} + assert set(all_reparented) == {reparented, same_branchpoint} env.pageserver.quiesce_tenants() @@ -530,7 +534,7 @@ def delta_layers(timeline_id: TimelineId): for _, timeline_id in skip_main: reparented = client.detach_ancestor(env.initial_tenant, timeline_id) - assert reparented == set(), "we have no earlier branches at any level" + assert reparented == [], "we have no earlier branches at any level" post_detach_l0s = list(filter(lambda x: x.l0, delta_layers(branch_timeline_id))) assert len(post_detach_l0s) == 5, "should had inherited 4 L0s, have 5 in total" @@ -561,7 +565,9 @@ def delta_layers(timeline_id: TimelineId): @pytest.mark.parametrize("sharded", [True, False]) -def test_timeline_ancestor_detach_errors(neon_env_builder: NeonEnvBuilder, sharded: bool): +def test_timeline_ancestor_detach_idempotent_success( + neon_env_builder: NeonEnvBuilder, sharded: bool +): shards = 2 if sharded else 1 neon_env_builder.num_pageservers = shards @@ -579,28 +585,28 @@ def test_timeline_ancestor_detach_errors(neon_env_builder: NeonEnvBuilder, shard else: client = env.pageserver.http_client() - with pytest.raises(PageserverApiException, match=".* no ancestors") as info: - client.detach_ancestor(env.initial_tenant, env.initial_timeline) - assert info.value.status_code == 409 - first_branch = env.neon_cli.create_branch("first_branch") - second_branch = env.neon_cli.create_branch("second_branch", ancestor_branch_name="first_branch") - # funnily enough this does not have a prefix - with pytest.raises(PageserverApiException, match="too many ancestors") as info: - client.detach_ancestor(env.initial_tenant, second_branch) - assert info.value.status_code == 400 + _ = env.neon_cli.create_branch("second_branch", ancestor_branch_name="first_branch") + + # these two will be reparented, and they should be returned in stable order + # from pageservers OR otherwise there will be an `error!` logging from + # storage controller + reparented1 = env.neon_cli.create_branch("first_reparented", ancestor_branch_name="main") + reparented2 = env.neon_cli.create_branch("second_reparented", ancestor_branch_name="main") - client.detach_ancestor(env.initial_tenant, first_branch) + first_reparenting_response = client.detach_ancestor(env.initial_tenant, first_branch) + assert set(first_reparenting_response) == {reparented1, reparented2} # FIXME: this should be done by the http req handler for ps in pageservers.values(): ps.quiesce_tenants() - with pytest.raises(PageserverApiException, match=".* no ancestors") as info: - client.detach_ancestor(env.initial_tenant, first_branch) - # FIXME: this should be 200 OK because we've already completed it - assert info.value.status_code == 409 + for _ in range(5): + # once completed, we can retry this how many times + assert ( + client.detach_ancestor(env.initial_tenant, first_branch) == first_reparenting_response + ) client.tenant_delete(env.initial_tenant) @@ -609,7 +615,50 @@ def test_timeline_ancestor_detach_errors(neon_env_builder: NeonEnvBuilder, shard assert e.value.status_code == 404 +@pytest.mark.parametrize("sharded", [True, False]) +def test_timeline_ancestor_detach_errors(neon_env_builder: NeonEnvBuilder, sharded: bool): + # the test is split from test_timeline_ancestor_detach_idempotent_success as only these error cases should create "request was dropped before completing", + # given the current first error handling + shards = 2 if sharded else 1 + + neon_env_builder.num_pageservers = shards + env = neon_env_builder.init_start(initial_tenant_shard_count=shards if sharded else None) + + pageservers = dict((int(p.id), p) for p in env.pageservers) + + for ps in pageservers.values(): + ps.allowed_errors.extend(SHUTDOWN_ALLOWED_ERRORS) + ps.allowed_errors.append( + ".* WARN .* path=/v1/tenant/.*/timeline/.*/detach_ancestor request_id=.*: request was dropped before completing" + ) + + client = ( + env.pageserver.http_client() if not sharded else env.storage_controller.pageserver_api() + ) + + with pytest.raises(PageserverApiException, match=".* no ancestors") as info: + client.detach_ancestor(env.initial_tenant, env.initial_timeline) + assert info.value.status_code == 409 + + _ = env.neon_cli.create_branch("first_branch") + + second_branch = env.neon_cli.create_branch("second_branch", ancestor_branch_name="first_branch") + + # funnily enough this does not have a prefix + with pytest.raises(PageserverApiException, match="too many ancestors") as info: + client.detach_ancestor(env.initial_tenant, second_branch) + assert info.value.status_code == 400 + + def test_sharded_timeline_detach_ancestor(neon_env_builder: NeonEnvBuilder): + """ + Sharded timeline detach ancestor; 4 nodes: 1 stuck, 1 restarted, 2 normal. + + Stuck node gets stuck on a pause failpoint for first storage controller request. + Restarted node remains stuck until explicit restart from test code. + + We retry the request until storage controller gets 200 OK from all nodes. + """ branch_name = "soon_detached" shard_count = 4 neon_env_builder.num_pageservers = shard_count @@ -621,8 +670,15 @@ def test_sharded_timeline_detach_ancestor(neon_env_builder: NeonEnvBuilder): # FIXME: should this be in the neon_env_builder.init_start? env.storage_controller.reconcile_until_idle() + # as we will stop a node, make sure there is no clever rebalancing + env.storage_controller.tenant_policy_update(env.initial_tenant, body={"scheduling": "Stop"}) + env.storage_controller.allowed_errors.append(".*: Scheduling is disabled by policy Stop .*") + shards = env.storage_controller.locate(env.initial_tenant) + utilized_pageservers = {x["node_id"] for x in shards} + assert len(utilized_pageservers) > 1, "all shards got placed on single pageserver?" + branch_timeline_id = env.neon_cli.create_branch(branch_name, tenant_id=env.initial_tenant) with env.endpoints.create_start(branch_name, tenant_id=env.initial_tenant) as ep: @@ -642,7 +698,79 @@ def test_sharded_timeline_detach_ancestor(neon_env_builder: NeonEnvBuilder): assert Lsn(detail["initdb_lsn"]) < lsn assert TimelineId(detail["ancestor_timeline_id"]) == env.initial_timeline - env.storage_controller.pageserver_api().detach_ancestor(env.initial_tenant, branch_timeline_id) + # make one of the nodes get stuck, but continue the initial operation + # make another of the nodes get stuck, then restart + + stuck = pageservers[int(shards[0]["node_id"])] + stuck.allowed_errors.append(".*: request was dropped before completing") + env.storage_controller.allowed_errors.append(".*: request was dropped before completing") + stuck_http = stuck.http_client() + stuck_http.configure_failpoints( + ("timeline-detach-ancestor::before_starting_after_locking_pausable", "pause") + ) + + restarted = pageservers[int(shards[1]["node_id"])] + restarted.allowed_errors.extend( + [ + ".*: request was dropped before completing", + ".*: Cancelled request finished with an error: ShuttingDown", + ] + ) + assert restarted.id != stuck.id + restarted_http = restarted.http_client() + restarted_http.configure_failpoints( + [ + ("timeline-detach-ancestor::before_starting_after_locking_pausable", "pause"), + ] + ) + + target = env.storage_controller.pageserver_api() + + with pytest.raises(ReadTimeout): + target.detach_ancestor(env.initial_tenant, branch_timeline_id, timeout=1) + + stuck_http.configure_failpoints( + ("timeline-detach-ancestor::before_starting_after_locking_pausable", "off") + ) + + barrier = threading.Barrier(2) + + def restart_restarted(): + barrier.wait() + # graceful shutdown should just work, because simultaneously unpaused + restarted.stop() + # this does not happen always, depends how fast we exit after unpausing + # restarted.assert_log_contains("Cancelled request finished with an error: ShuttingDown") + restarted.start() + + with ThreadPoolExecutor(max_workers=1) as pool: + fut = pool.submit(restart_restarted) + barrier.wait() + # we have 10s, lets use 1/2 of that to help the shutdown start + time.sleep(5) + restarted_http.configure_failpoints( + ("timeline-detach-ancestor::before_starting_after_locking_pausable", "off") + ) + fut.result() + + # detach ancestor request handling is not sensitive to http cancellation. + # this means that the "stuck" is on its way to complete the detach, but the restarted is off + # now it can either be complete on all nodes, or still in progress with + # one. + without_retrying = target.without_status_retrying() + + # this retry loop will be long enough that the tenant can always activate + reparented = None + for _ in range(10): + try: + reparented = without_retrying.detach_ancestor(env.initial_tenant, branch_timeline_id) + except PageserverApiException as info: + assert info.status_code == 503 + time.sleep(2) + else: + break + + assert reparented == [], "too many retries (None) or unexpected reparentings" for shard_info in shards: node_id = int(shard_info["node_id"]) @@ -661,8 +789,262 @@ def test_sharded_timeline_detach_ancestor(neon_env_builder: NeonEnvBuilder): assert count == 10000 +@pytest.mark.parametrize("mode", ["delete_timeline", "delete_tenant"]) +@pytest.mark.parametrize("sharded", [False, True]) +def test_timeline_detach_ancestor_interrupted_by_deletion( + neon_env_builder: NeonEnvBuilder, mode: str, sharded: bool +): + """ + Timeline ancestor detach interrupted by deleting either: + - the detached timeline + - the whole tenant + + after starting the detach. + + What remains not tested by this: + - shutdown winning over complete + + Shutdown winning over complete needs gc blocking and reparenting any left-overs on retry. + """ + + if sharded and mode == "delete_tenant": + # the shared/exclusive lock for tenant is blocking this: + # timeline detach ancestor takes shared, delete tenant takes exclusive + pytest.skip( + "tenant deletion while timeline ancestor detach is underway is not supported yet" + ) + + shard_count = 2 if sharded else 1 + + neon_env_builder.num_pageservers = shard_count + + env = neon_env_builder.init_start(initial_tenant_shard_count=shard_count if sharded else None) + + for ps in env.pageservers: + ps.allowed_errors.extend(SHUTDOWN_ALLOWED_ERRORS) + + pageservers = dict((int(p.id), p) for p in env.pageservers) + + detached_timeline = env.neon_cli.create_branch("detached soon", "main") + + failpoint = "timeline-detach-ancestor::before_starting_after_locking_pausable" + + env.storage_controller.reconcile_until_idle() + shards = env.storage_controller.locate(env.initial_tenant) + + assert len(set(info["node_id"] for info in shards)) == shard_count + + target = env.storage_controller.pageserver_api() if sharded else env.pageserver.http_client() + target = target.without_status_retrying() + + victim = pageservers[int(shards[-1]["node_id"])] + victim_http = victim.http_client() + victim_http.configure_failpoints((failpoint, "pause")) + + def detach_ancestor(): + target.detach_ancestor(env.initial_tenant, detached_timeline) + + def at_failpoint() -> Tuple[str, LogCursor]: + return victim.assert_log_contains(f"at failpoint {failpoint}") + + def start_delete(): + if mode == "delete_timeline": + target.timeline_delete(env.initial_tenant, detached_timeline) + elif mode == "delete_tenant": + target.tenant_delete(env.initial_tenant) + else: + raise RuntimeError(f"unimplemented mode {mode}") + + def at_waiting_on_gate_close(start_offset: LogCursor) -> LogCursor: + _, offset = victim.assert_log_contains( + "closing is taking longer than expected", offset=start_offset + ) + return offset + + def is_deleted(): + try: + if mode == "delete_timeline": + target.timeline_detail(env.initial_tenant, detached_timeline) + elif mode == "delete_tenant": + target.tenant_status(env.initial_tenant) + else: + return False + except PageserverApiException as e: + assert e.status_code == 404 + return True + else: + raise RuntimeError("waiting for 404") + + with ThreadPoolExecutor(max_workers=2) as pool: + try: + fut = pool.submit(detach_ancestor) + _, offset = wait_until(10, 1.0, at_failpoint) + + delete = pool.submit(start_delete) + + wait_until(10, 1.0, lambda: at_waiting_on_gate_close(offset)) + + victim_http.configure_failpoints((failpoint, "off")) + + delete.result() + + assert wait_until(10, 1.0, is_deleted), f"unimplemented mode {mode}" + + with pytest.raises(PageserverApiException) as exc: + fut.result() + assert exc.value.status_code == 503 + finally: + victim_http.configure_failpoints((failpoint, "off")) + + +@pytest.mark.parametrize("mode", ["delete_reparentable_timeline"]) +def test_sharded_tad_interleaved_after_partial_success(neon_env_builder: NeonEnvBuilder, mode: str): + """ + Technically possible storage controller concurrent interleaving timeline + deletion with timeline detach. + + Deletion is fine, as any sharded pageservers reach the same end state, but + creating reparentable timeline would create an issue as the two nodes would + never agree. There is a solution though: the created reparentable timeline + must be detached. + """ + + assert ( + mode == "delete_reparentable_timeline" + ), "only one now, but we could have the create just as well, need gc blocking" + + shard_count = 2 + neon_env_builder.num_pageservers = shard_count + env = neon_env_builder.init_start(initial_tenant_shard_count=shard_count) + + for ps in env.pageservers: + ps.allowed_errors.extend(SHUTDOWN_ALLOWED_ERRORS) + + pageservers = dict((int(p.id), p) for p in env.pageservers) + + env.storage_controller.reconcile_until_idle() + shards = env.storage_controller.locate(env.initial_tenant) + assert len(set(x["node_id"] for x in shards)) == shard_count + + with env.endpoints.create_start("main") as ep: + ep.safe_psql("create table foo as select i::bigint from generate_series(1, 1000) t(i)") + + # as the interleaved operation, we will delete this timeline, which was reparenting candidate + first_branch_lsn = wait_for_last_flush_lsn( + env, ep, env.initial_tenant, env.initial_timeline + ) + for ps, shard_id in [(pageservers[int(x["node_id"])], x["shard_id"]) for x in shards]: + ps.http_client().timeline_checkpoint(shard_id, env.initial_timeline) + + ep.safe_psql("create table bar as select i::bigint from generate_series(1, 2000) t(i)") + detached_branch_lsn = flush_ep_to_pageserver( + env, ep, env.initial_tenant, env.initial_timeline + ) + + for ps, shard_id in [(pageservers[int(x["node_id"])], x["shard_id"]) for x in shards]: + ps.http_client().timeline_checkpoint(shard_id, env.initial_timeline) + + first_branch = env.neon_cli.create_branch( + "first_branch", ancestor_branch_name="main", ancestor_start_lsn=first_branch_lsn + ) + detached_branch = env.neon_cli.create_branch( + "detached_branch", ancestor_branch_name="main", ancestor_start_lsn=detached_branch_lsn + ) + + pausepoint = "timeline-detach-ancestor::before_starting_after_locking_pausable" + + stuck = pageservers[int(shards[0]["node_id"])] + stuck_http = stuck.http_client().without_status_retrying() + stuck_http.configure_failpoints((pausepoint, "pause")) + + victim = pageservers[int(shards[-1]["node_id"])] + victim_http = victim.http_client().without_status_retrying() + victim_http.configure_failpoints( + (pausepoint, "pause"), + ) + + # noticed a surprising 409 if the other one would fail instead + # victim_http.configure_failpoints([ + # (pausepoint, "pause"), + # ("timeline-detach-ancestor::before_starting_after_locking", "return"), + # ]) + + # interleaving a create_timeline which could be reparented will produce two + # permanently different reparentings: one node has reparented, other has + # not + # + # with deletion there is no such problem + def detach_timeline(): + env.storage_controller.pageserver_api().detach_ancestor(env.initial_tenant, detached_branch) + + def paused_at_failpoint(): + stuck.assert_log_contains(f"at failpoint {pausepoint}") + victim.assert_log_contains(f"at failpoint {pausepoint}") + + def first_completed(): + detail = stuck_http.timeline_detail(shards[0]["shard_id"], detached_branch) + log.info(detail) + assert detail.get("ancestor_lsn") is None + + def first_branch_gone(): + try: + env.storage_controller.pageserver_api().timeline_detail( + env.initial_tenant, first_branch + ) + except PageserverApiException as e: + log.info(f"error {e}") + assert e.status_code == 404 + else: + log.info("still ok") + raise RuntimeError("not done yet") + + with ThreadPoolExecutor(max_workers=1) as pool: + try: + fut = pool.submit(detach_timeline) + wait_until(10, 1.0, paused_at_failpoint) + + # let stuck complete + stuck_http.configure_failpoints((pausepoint, "off")) + wait_until(10, 1.0, first_completed) + + # if we would let victim fail, for some reason there'd be a 409 response instead of 500 + # victim_http.configure_failpoints((pausepoint, "off")) + # with pytest.raises(PageserverApiException, match=".* 500 Internal Server Error failpoint: timeline-detach-ancestor::before_starting_after_locking") as exc: + # fut.result() + # assert exc.value.status_code == 409 + + env.storage_controller.pageserver_api().timeline_delete( + env.initial_tenant, first_branch + ) + victim_http.configure_failpoints((pausepoint, "off")) + wait_until(10, 1.0, first_branch_gone) + + # it now passes, and we should get an error messages about mixed reparenting as the stuck still had something to reparent + fut.result() + + msg, offset = env.storage_controller.assert_log_contains( + ".*/timeline/\\S+/detach_ancestor.*: shards returned different results matching=0 .*" + ) + log.info(f"expected error message: {msg}") + env.storage_controller.allowed_errors.append( + ".*: shards returned different results matching=0 .*" + ) + + detach_timeline() + + # FIXME: perhaps the above should be automatically retried, if we get mixed results? + not_found = env.storage_controller.log_contains( + ".*/timeline/\\S+/detach_ancestor.*: shards returned different results matching=0 .*", + offset=offset, + ) + + assert not_found is None + finally: + stuck_http.configure_failpoints((pausepoint, "off")) + victim_http.configure_failpoints((pausepoint, "off")) + + # TODO: -# - after starting the operation, tenant is deleted # - after starting the operation, pageserver is shutdown, restarted # - after starting the operation, bottom-most timeline is deleted, pageserver is restarted, gc is inhibited # - deletion of reparented while reparenting should fail once, then succeed (?) @@ -670,9 +1052,5 @@ def test_sharded_timeline_detach_ancestor(neon_env_builder: NeonEnvBuilder): # - investigate: why are layers started at uneven lsn? not just after branching, but in general. # # TEST: 1. tad which partially succeeds, one returns 500 -# 2. create branch below timeline? or delete timeline below +# 2. create branch below timeline? ~or delete reparented timeline~ (done) # 3. on retry all should report the same reparented timelines -# -# TEST: 1. tad is started, one node stalls, other restarts -# 2. client timeout before stall over -# 3. on retry with stalled and other being able to proceed From 7eb37fea26ab7ed3312a82617cef33af03476999 Mon Sep 17 00:00:00 2001 From: Sasha Krassovsky Date: Mon, 15 Jul 2024 14:55:57 -0700 Subject: [PATCH 111/194] Allow reusing projects between runs of logical replication benchmarks (#8393) --- test_runner/fixtures/neon_api.py | 44 +++ test_runner/fixtures/neon_fixtures.py | 14 +- .../performance/test_logical_replication.py | 341 +++++++----------- 3 files changed, 182 insertions(+), 217 deletions(-) diff --git a/test_runner/fixtures/neon_api.py b/test_runner/fixtures/neon_api.py index 39baf5fab69f..658ed119a175 100644 --- a/test_runner/fixtures/neon_api.py +++ b/test_runner/fixtures/neon_api.py @@ -261,3 +261,47 @@ def wait_for_operation_to_finish(self, project_id: str): if op["status"] in {"scheduling", "running", "cancelling"}: has_running = True time.sleep(0.5) + + +class NeonApiEndpoint: + def __init__(self, neon_api: NeonAPI, pg_version: PgVersion, project_id: Optional[str]): + self.neon_api = neon_api + if project_id is None: + project = neon_api.create_project(pg_version) + neon_api.wait_for_operation_to_finish(project["project"]["id"]) + self.project_id = project["project"]["id"] + self.endpoint_id = project["endpoints"][0]["id"] + self.connstr = project["connection_uris"][0]["connection_uri"] + self.pgbench_env = connection_parameters_to_env( + project["connection_uris"][0]["connection_parameters"] + ) + self.is_new = True + else: + project = neon_api.get_project_details(project_id) + if int(project["project"]["pg_version"]) != int(pg_version): + raise Exception( + f"A project with the provided ID exists, but it's not of the specified version (expected {pg_version}, got {project['project']['pg_version']})" + ) + self.project_id = project_id + eps = neon_api.get_endpoints(project_id)["endpoints"] + self.endpoint_id = eps[0]["id"] + self.connstr = neon_api.get_connection_uri(project_id, endpoint_id=self.endpoint_id)[ + "uri" + ] + pw = self.connstr.split("@")[0].split(":")[-1] + self.pgbench_env = { + "PGHOST": eps[0]["host"], + "PGDATABASE": "neondb", + "PGUSER": "neondb_owner", + "PGPASSWORD": pw, + } + self.is_new = False + + def restart(self): + self.neon_api.restart_endpoint(self.project_id, self.endpoint_id) + self.neon_api.wait_for_operation_to_finish(self.project_id) + + def get_synthetic_storage_size(self) -> int: + return int( + self.neon_api.get_project_details(self.project_id)["project"]["synthetic_storage_size"] + ) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 90ed838e1db3..fe4a33445834 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -87,7 +87,7 @@ ) from fixtures.utils import AuxFileStore as AuxFileStore # reexport -from .neon_api import NeonAPI +from .neon_api import NeonAPI, NeonApiEndpoint """ This file contains pytest fixtures. A fixture is a test resource that can be @@ -3158,6 +3158,18 @@ def __exit__( pass +@pytest.fixture(scope="function") +def benchmark_project_pub(neon_api: NeonAPI, pg_version: PgVersion) -> NeonApiEndpoint: + project_id = os.getenv("BENCHMARK_PROJECT_ID_PUB") + return NeonApiEndpoint(neon_api, pg_version, project_id) + + +@pytest.fixture(scope="function") +def benchmark_project_sub(neon_api: NeonAPI, pg_version: PgVersion) -> NeonApiEndpoint: + project_id = os.getenv("BENCHMARK_PROJECT_ID_SUB") + return NeonApiEndpoint(neon_api, pg_version, project_id) + + @pytest.fixture(scope="function") def remote_pg( test_output_dir: Path, pg_distrib_dir: Path, pg_version: PgVersion diff --git a/test_runner/performance/test_logical_replication.py b/test_runner/performance/test_logical_replication.py index 5ab83dd31d0b..53bb29a65908 100644 --- a/test_runner/performance/test_logical_replication.py +++ b/test_runner/performance/test_logical_replication.py @@ -1,7 +1,6 @@ from __future__ import annotations import time -import traceback from typing import TYPE_CHECKING import psycopg2 @@ -10,15 +9,12 @@ from fixtures.benchmark_fixture import MetricReport from fixtures.common_types import Lsn from fixtures.log_helper import log -from fixtures.neon_api import connection_parameters_to_env from fixtures.neon_fixtures import AuxFileStore, logical_replication_sync -from fixtures.pg_version import PgVersion if TYPE_CHECKING: from fixtures.benchmark_fixture import NeonBenchmarker - from fixtures.neon_api import NeonAPI + from fixtures.neon_api import NeonApiEndpoint from fixtures.neon_fixtures import NeonEnv, PgBin - from fixtures.pg_version import PgVersion @pytest.mark.parametrize("pageserver_aux_file_policy", [AuxFileStore.V2]) @@ -86,8 +82,8 @@ def measure_logical_replication_lag(sub_cur, pub_cur, timeout_sec=600): @pytest.mark.timeout(2 * 60 * 60) def test_subscriber_lag( pg_bin: PgBin, - neon_api: NeonAPI, - pg_version: PgVersion, + benchmark_project_pub: NeonApiEndpoint, + benchmark_project_sub: NeonApiEndpoint, zenbenchmark: NeonBenchmarker, ): """ @@ -99,125 +95,82 @@ def test_subscriber_lag( sync_interval_min = 5 pgbench_duration = f"-T{test_duration_min * 60 * 2}" - pub_project = neon_api.create_project(pg_version) - pub_project_id = pub_project["project"]["id"] - neon_api.wait_for_operation_to_finish(pub_project_id) - error_occurred = False - try: - sub_project = neon_api.create_project(pg_version) - sub_project_id = sub_project["project"]["id"] - sub_endpoint_id = sub_project["endpoints"][0]["id"] - neon_api.wait_for_operation_to_finish(sub_project_id) - try: - pub_env = connection_parameters_to_env( - pub_project["connection_uris"][0]["connection_parameters"] - ) - sub_env = connection_parameters_to_env( - sub_project["connection_uris"][0]["connection_parameters"] - ) - pub_connstr = pub_project["connection_uris"][0]["connection_uri"] - sub_connstr = sub_project["connection_uris"][0]["connection_uri"] - - pg_bin.run_capture(["pgbench", "-i", "-s100"], env=pub_env) - pg_bin.run_capture(["pgbench", "-i", "-s100"], env=sub_env) - - pub_conn = psycopg2.connect(pub_connstr) - sub_conn = psycopg2.connect(sub_connstr) - pub_conn.autocommit = True - sub_conn.autocommit = True - with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: - sub_cur.execute("truncate table pgbench_accounts") - sub_cur.execute("truncate table pgbench_history") - - pub_cur.execute( - "create publication pub1 for table pgbench_accounts, pgbench_history" - ) - sub_cur.execute( - f"create subscription sub1 connection '{pub_connstr}' publication pub1" - ) + pub_env = benchmark_project_pub.pgbench_env + sub_env = benchmark_project_sub.pgbench_env + pub_connstr = benchmark_project_pub.connstr + sub_connstr = benchmark_project_sub.connstr + + pg_bin.run_capture(["pgbench", "-i", "-s100"], env=pub_env) + pg_bin.run_capture(["pgbench", "-i", "-s100"], env=sub_env) + + pub_conn = psycopg2.connect(pub_connstr) + sub_conn = psycopg2.connect(sub_connstr) + pub_conn.autocommit = True + sub_conn.autocommit = True + with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: + if benchmark_project_pub.is_new: + pub_cur.execute("create publication pub1 for table pgbench_accounts, pgbench_history") + + if benchmark_project_sub.is_new: + sub_cur.execute("truncate table pgbench_accounts") + sub_cur.execute("truncate table pgbench_history") + + sub_cur.execute(f"create subscription sub1 connection '{pub_connstr}' publication pub1") - initial_sync_lag = measure_logical_replication_lag(sub_cur, pub_cur) - pub_conn.close() - sub_conn.close() + initial_sync_lag = measure_logical_replication_lag(sub_cur, pub_cur) + pub_conn.close() + sub_conn.close() - zenbenchmark.record( - "initial_sync_lag", initial_sync_lag, "s", MetricReport.LOWER_IS_BETTER - ) + zenbenchmark.record("initial_sync_lag", initial_sync_lag, "s", MetricReport.LOWER_IS_BETTER) + + pub_workload = pg_bin.run_nonblocking( + ["pgbench", "-c10", pgbench_duration, "-Mprepared"], env=pub_env + ) + try: + sub_workload = pg_bin.run_nonblocking( + ["pgbench", "-c10", pgbench_duration, "-S"], + env=sub_env, + ) + try: + start = time.time() + while time.time() - start < test_duration_min * 60: + time.sleep(sync_interval_min * 60) + check_pgbench_still_running(pub_workload, "pub") + check_pgbench_still_running(sub_workload, "sub") + + with psycopg2.connect(pub_connstr) as pub_conn, psycopg2.connect( + sub_connstr + ) as sub_conn: + with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: + lag = measure_logical_replication_lag(sub_cur, pub_cur) + + log.info(f"Replica lagged behind master by {lag} seconds") + zenbenchmark.record("replica_lag", lag, "s", MetricReport.LOWER_IS_BETTER) + sub_workload.terminate() + benchmark_project_sub.restart() - pub_workload = pg_bin.run_nonblocking( - ["pgbench", "-c10", pgbench_duration, "-Mprepared"], env=pub_env - ) - try: sub_workload = pg_bin.run_nonblocking( ["pgbench", "-c10", pgbench_duration, "-S"], env=sub_env, ) - try: - start = time.time() - while time.time() - start < test_duration_min * 60: - time.sleep(sync_interval_min * 60) - check_pgbench_still_running(pub_workload, "pub") - check_pgbench_still_running(sub_workload, "sub") - - with psycopg2.connect(pub_connstr) as pub_conn, psycopg2.connect( - sub_connstr - ) as sub_conn: - with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: - lag = measure_logical_replication_lag(sub_cur, pub_cur) - - log.info(f"Replica lagged behind master by {lag} seconds") - zenbenchmark.record("replica_lag", lag, "s", MetricReport.LOWER_IS_BETTER) - sub_workload.terminate() - neon_api.restart_endpoint( - sub_project_id, - sub_endpoint_id, - ) - neon_api.wait_for_operation_to_finish(sub_project_id) - sub_workload = pg_bin.run_nonblocking( - ["pgbench", "-c10", pgbench_duration, "-S"], - env=sub_env, - ) - - # Measure storage to make sure replication information isn't bloating storage - sub_storage = neon_api.get_project_details(sub_project_id)["project"][ - "synthetic_storage_size" - ] - pub_storage = neon_api.get_project_details(pub_project_id)["project"][ - "synthetic_storage_size" - ] - zenbenchmark.record( - "sub_storage", sub_storage, "B", MetricReport.LOWER_IS_BETTER - ) - zenbenchmark.record( - "pub_storage", pub_storage, "B", MetricReport.LOWER_IS_BETTER - ) - - finally: - sub_workload.terminate() - finally: - pub_workload.terminate() - except Exception as e: - error_occurred = True - log.error(f"Caught exception {e}") - log.error(traceback.format_exc()) + + # Measure storage to make sure replication information isn't bloating storage + sub_storage = benchmark_project_sub.get_synthetic_storage_size() + pub_storage = benchmark_project_pub.get_synthetic_storage_size() + zenbenchmark.record("sub_storage", sub_storage, "B", MetricReport.LOWER_IS_BETTER) + zenbenchmark.record("pub_storage", pub_storage, "B", MetricReport.LOWER_IS_BETTER) finally: - if not error_occurred: - neon_api.delete_project(sub_project_id) - except Exception as e: - error_occurred = True - log.error(f"Caught exception {e}") - log.error(traceback.format_exc()) + sub_workload.terminate() finally: - assert not error_occurred - neon_api.delete_project(pub_project_id) + pub_workload.terminate() @pytest.mark.remote_cluster @pytest.mark.timeout(2 * 60 * 60) def test_publisher_restart( pg_bin: PgBin, - neon_api: NeonAPI, - pg_version: PgVersion, + benchmark_project_pub: NeonApiEndpoint, + benchmark_project_sub: NeonApiEndpoint, zenbenchmark: NeonBenchmarker, ): """ @@ -229,114 +182,70 @@ def test_publisher_restart( sync_interval_min = 5 pgbench_duration = f"-T{test_duration_min * 60 * 2}" - pub_project = neon_api.create_project(pg_version) - pub_project_id = pub_project["project"]["id"] - pub_endpoint_id = pub_project["endpoints"][0]["id"] - neon_api.wait_for_operation_to_finish(pub_project_id) - error_occurred = False - try: - sub_project = neon_api.create_project(pg_version) - sub_project_id = sub_project["project"]["id"] - neon_api.wait_for_operation_to_finish(sub_project_id) - try: - pub_env = connection_parameters_to_env( - pub_project["connection_uris"][0]["connection_parameters"] - ) - sub_env = connection_parameters_to_env( - sub_project["connection_uris"][0]["connection_parameters"] - ) - pub_connstr = pub_project["connection_uris"][0]["connection_uri"] - sub_connstr = sub_project["connection_uris"][0]["connection_uri"] - - pg_bin.run_capture(["pgbench", "-i", "-s100"], env=pub_env) - pg_bin.run_capture(["pgbench", "-i", "-s100"], env=sub_env) - - pub_conn = psycopg2.connect(pub_connstr) - sub_conn = psycopg2.connect(sub_connstr) - pub_conn.autocommit = True - sub_conn.autocommit = True - with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: - sub_cur.execute("truncate table pgbench_accounts") - sub_cur.execute("truncate table pgbench_history") - - pub_cur.execute( - "create publication pub1 for table pgbench_accounts, pgbench_history" - ) - sub_cur.execute( - f"create subscription sub1 connection '{pub_connstr}' publication pub1" - ) + pub_env = benchmark_project_pub.pgbench_env + sub_env = benchmark_project_sub.pgbench_env + pub_connstr = benchmark_project_pub.connstr + sub_connstr = benchmark_project_sub.connstr - initial_sync_lag = measure_logical_replication_lag(sub_cur, pub_cur) - pub_conn.close() - sub_conn.close() + pg_bin.run_capture(["pgbench", "-i", "-s100"], env=pub_env) + pg_bin.run_capture(["pgbench", "-i", "-s100"], env=sub_env) - zenbenchmark.record( - "initial_sync_lag", initial_sync_lag, "s", MetricReport.LOWER_IS_BETTER - ) + pub_conn = psycopg2.connect(pub_connstr) + sub_conn = psycopg2.connect(sub_connstr) + pub_conn.autocommit = True + sub_conn.autocommit = True + with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: + if benchmark_project_pub.is_new: + pub_cur.execute("create publication pub1 for table pgbench_accounts, pgbench_history") + + if benchmark_project_sub.is_new: + sub_cur.execute("truncate table pgbench_accounts") + sub_cur.execute("truncate table pgbench_history") + + sub_cur.execute(f"create subscription sub1 connection '{pub_connstr}' publication pub1") + + initial_sync_lag = measure_logical_replication_lag(sub_cur, pub_cur) + pub_conn.close() + sub_conn.close() + + zenbenchmark.record("initial_sync_lag", initial_sync_lag, "s", MetricReport.LOWER_IS_BETTER) + + pub_workload = pg_bin.run_nonblocking( + ["pgbench", "-c10", pgbench_duration, "-Mprepared"], env=pub_env + ) + try: + sub_workload = pg_bin.run_nonblocking( + ["pgbench", "-c10", pgbench_duration, "-S"], + env=sub_env, + ) + try: + start = time.time() + while time.time() - start < test_duration_min * 60: + time.sleep(sync_interval_min * 60) + check_pgbench_still_running(pub_workload, "pub") + check_pgbench_still_running(sub_workload, "sub") - pub_workload = pg_bin.run_nonblocking( - ["pgbench", "-c10", pgbench_duration, "-Mprepared"], env=pub_env - ) - try: - sub_workload = pg_bin.run_nonblocking( - ["pgbench", "-c10", pgbench_duration, "-S"], - env=sub_env, - ) - try: - start = time.time() - while time.time() - start < test_duration_min * 60: - time.sleep(sync_interval_min * 60) - check_pgbench_still_running(pub_workload, "pub") - check_pgbench_still_running(sub_workload, "sub") - - pub_workload.terminate() - neon_api.restart_endpoint( - pub_project_id, - pub_endpoint_id, - ) - neon_api.wait_for_operation_to_finish(pub_project_id) - pub_workload = pg_bin.run_nonblocking( - ["pgbench", "-c10", pgbench_duration, "-Mprepared"], - env=pub_env, - ) - with psycopg2.connect(pub_connstr) as pub_conn, psycopg2.connect( - sub_connstr - ) as sub_conn: - with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: - lag = measure_logical_replication_lag(sub_cur, pub_cur) - - log.info(f"Replica lagged behind master by {lag} seconds") - zenbenchmark.record("replica_lag", lag, "s", MetricReport.LOWER_IS_BETTER) - - # Measure storage to make sure replication information isn't bloating storage - sub_storage = neon_api.get_project_details(sub_project_id)["project"][ - "synthetic_storage_size" - ] - pub_storage = neon_api.get_project_details(pub_project_id)["project"][ - "synthetic_storage_size" - ] - zenbenchmark.record( - "sub_storage", sub_storage, "B", MetricReport.LOWER_IS_BETTER - ) - zenbenchmark.record( - "pub_storage", pub_storage, "B", MetricReport.LOWER_IS_BETTER - ) - - finally: - sub_workload.terminate() - finally: pub_workload.terminate() - except Exception as e: - error_occurred = True - log.error(f"Caught exception {e}") - log.error(traceback.format_exc()) + benchmark_project_pub.restart() + pub_workload = pg_bin.run_nonblocking( + ["pgbench", "-c10", pgbench_duration, "-Mprepared"], + env=pub_env, + ) + with psycopg2.connect(pub_connstr) as pub_conn, psycopg2.connect( + sub_connstr + ) as sub_conn: + with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: + lag = measure_logical_replication_lag(sub_cur, pub_cur) + + log.info(f"Replica lagged behind master by {lag} seconds") + zenbenchmark.record("replica_lag", lag, "s", MetricReport.LOWER_IS_BETTER) + + # Measure storage to make sure replication information isn't bloating storage + sub_storage = benchmark_project_sub.get_synthetic_storage_size() + pub_storage = benchmark_project_pub.get_synthetic_storage_size() + zenbenchmark.record("sub_storage", sub_storage, "B", MetricReport.LOWER_IS_BETTER) + zenbenchmark.record("pub_storage", pub_storage, "B", MetricReport.LOWER_IS_BETTER) finally: - if not error_occurred: - neon_api.delete_project(sub_project_id) - except Exception as e: - error_occurred = True - log.error(f"Caught exception {e}") - log.error(traceback.format_exc()) + sub_workload.terminate() finally: - assert not error_occurred - neon_api.delete_project(pub_project_id) + pub_workload.terminate() From ee263e6a622c38369110bfa8fae1ba044c48ce0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Tue, 16 Jul 2024 02:16:18 +0200 Subject: [PATCH 112/194] Allow the new clippy::doc_lazy_continuation lint (#8388) The `doc_lazy_continuation` lint of clippy is still unknown on latest rust stable. Fixes fall-out from #8151. --- pageserver/src/tenant/timeline.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 239dce878640..58c6257c658d 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -3409,6 +3409,7 @@ impl Timeline { } } + #[allow(unknown_lints)] // doc_lazy_continuation is still a new lint #[allow(clippy::doc_lazy_continuation)] /// Get the data needed to reconstruct all keys in the provided keyspace /// From 83e07c1a5bcc8f4075474ba8b5e4731a078f6dd7 Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 16 Jul 2024 08:52:49 +0100 Subject: [PATCH 113/194] pageserver: un-Arc Timeline::layers (#8386) ## Problem This structure was in an Arc<> unnecessarily, making it harder to reason about its lifetime (i.e. it was superficially possible for LayerManager to outlive timeline, even though no code used it that way) ## Summary of changes - Remove the Arc<> --- pageserver/src/tenant/timeline.rs | 4 ++-- pageserver/src/tenant/timeline/compaction.rs | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 58c6257c658d..48a5b2d32bf7 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -198,7 +198,7 @@ impl PartialOrd for Hole { /// Temporary function for immutable storage state refactor, ensures we are dropping mutex guard instead of other things. /// Can be removed after all refactors are done. -fn drop_rlock(rlock: tokio::sync::OwnedRwLockReadGuard) { +fn drop_rlock(rlock: tokio::sync::RwLockReadGuard) { drop(rlock) } @@ -271,7 +271,7 @@ pub struct Timeline { /// /// In the future, we'll be able to split up the tuple of LayerMap and `LayerFileManager`, /// so that e.g. on-demand-download/eviction, and layer spreading, can operate just on `LayerFileManager`. - pub(crate) layers: Arc>, + pub(crate) layers: tokio::sync::RwLock, last_freeze_at: AtomicLsn, // Atomic would be more appropriate here. diff --git a/pageserver/src/tenant/timeline/compaction.rs b/pageserver/src/tenant/timeline/compaction.rs index efaa6144af95..eec5e5e53cf7 100644 --- a/pageserver/src/tenant/timeline/compaction.rs +++ b/pageserver/src/tenant/timeline/compaction.rs @@ -27,8 +27,8 @@ use utils::id::TimelineId; use crate::context::{AccessStatsBehavior, RequestContext, RequestContextBuilder}; use crate::page_cache; use crate::tenant::storage_layer::{AsLayerDesc, PersistentLayerDesc}; -use crate::tenant::timeline::{drop_rlock, Hole, ImageLayerCreationOutcome}; -use crate::tenant::timeline::{DeltaLayerWriter, ImageLayerWriter}; +use crate::tenant::timeline::{drop_rlock, DeltaLayerWriter, ImageLayerWriter}; +use crate::tenant::timeline::{Hole, ImageLayerCreationOutcome}; use crate::tenant::timeline::{Layer, ResidentLayer}; use crate::tenant::DeltaLayer; use crate::virtual_file::{MaybeFatalIo, VirtualFile}; @@ -379,7 +379,7 @@ impl Timeline { }; let begin = tokio::time::Instant::now(); - let phase1_layers_locked = Arc::clone(&self.layers).read_owned().await; + let phase1_layers_locked = self.layers.read().await; let now = tokio::time::Instant::now(); stats.read_lock_acquisition_micros = DurationRecorder::Recorded(RecordedDuration(now - begin), now); @@ -399,9 +399,9 @@ impl Timeline { } /// Level0 files first phase of compaction, explained in the [`Self::compact_legacy`] comment. - async fn compact_level0_phase1( - self: &Arc, - guard: tokio::sync::OwnedRwLockReadGuard, + async fn compact_level0_phase1<'a>( + self: &'a Arc, + guard: tokio::sync::RwLockReadGuard<'a, LayerManager>, mut stats: CompactLevel0Phase1StatsBuilder, target_file_size: u64, ctx: &RequestContext, From e6dadcd2f35ce4dd2702acef2bdebe75d583677f Mon Sep 17 00:00:00 2001 From: Anastasia Lubennikova Date: Mon, 15 Jul 2024 12:48:53 +0100 Subject: [PATCH 114/194] Compute: add compatibility patch for rum Fixes #8251 --- Dockerfile.compute-node | 3 +++ patches/rum.patch | 54 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 patches/rum.patch diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node index 7ab685625a8b..48a52bfc6d04 100644 --- a/Dockerfile.compute-node +++ b/Dockerfile.compute-node @@ -311,9 +311,12 @@ RUN wget https://github.com/iCyberon/pg_hashids/archive/refs/tags/v1.2.1.tar.gz FROM build-deps AS rum-pg-build COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ +COPY patches/rum.patch /rum.patch + RUN wget https://github.com/postgrespro/rum/archive/refs/tags/1.3.13.tar.gz -O rum.tar.gz && \ echo "6ab370532c965568df6210bd844ac6ba649f53055e48243525b0b7e5c4d69a7d rum.tar.gz" | sha256sum --check && \ mkdir rum-src && cd rum-src && tar xzf ../rum.tar.gz --strip-components=1 -C . && \ + patch -p1 < /rum.patch && \ make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config USE_PGXS=1 && \ make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config USE_PGXS=1 && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/rum.control diff --git a/patches/rum.patch b/patches/rum.patch new file mode 100644 index 000000000000..3041f8df81d6 --- /dev/null +++ b/patches/rum.patch @@ -0,0 +1,54 @@ +commit 68f3b3b0d594f08aacc4a082ee210749ed5677eb +Author: Anastasia Lubennikova +Date: Mon Jul 15 12:31:56 2024 +0100 + + Neon: fix unlogged index build patch + +diff --git a/src/ruminsert.c b/src/ruminsert.c +index e8b209d..e89bf2a 100644 +--- a/src/ruminsert.c ++++ b/src/ruminsert.c +@@ -628,6 +628,10 @@ rumbuild(Relation heap, Relation index, struct IndexInfo *indexInfo) + elog(ERROR, "index \"%s\" already contains data", + RelationGetRelationName(index)); + ++#ifdef NEON_SMGR ++ smgr_start_unlogged_build(index->rd_smgr); ++#endif ++ + initRumState(&buildstate.rumstate, index); + buildstate.rumstate.isBuild = true; + buildstate.indtuples = 0; +@@ -693,6 +697,10 @@ rumbuild(Relation heap, Relation index, struct IndexInfo *indexInfo) + buildstate.buildStats.nTotalPages = RelationGetNumberOfBlocks(index); + rumUpdateStats(index, &buildstate.buildStats, buildstate.rumstate.isBuild); + ++#ifdef NEON_SMGR ++ smgr_finish_unlogged_build_phase_1(index->rd_smgr); ++#endif ++ + /* + * Write index to xlog + */ +@@ -713,6 +721,21 @@ rumbuild(Relation heap, Relation index, struct IndexInfo *indexInfo) + UnlockReleaseBuffer(buffer); + } + ++#ifdef NEON_SMGR ++ { ++#if PG_VERSION_NUM >= 160000 ++ RelFileLocator rlocator = RelationGetSmgr(index)->smgr_rlocator.locator; ++#else ++ RelFileNode rlocator = RelationGetSmgr(index)->smgr_rnode.node; ++#endif ++ ++ SetLastWrittenLSNForBlockRange(XactLastRecEnd, rlocator, MAIN_FORKNUM, 0, RelationGetNumberOfBlocks(index)); ++ SetLastWrittenLSNForRelation(XactLastRecEnd, rlocator, MAIN_FORKNUM); ++ ++ smgr_end_unlogged_build(index->rd_smgr); ++ } ++#endif ++ + /* + * Return statistics + */ From 66337097de074de3a2e2e19bf0b1c304a21b273c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Tue, 16 Jul 2024 12:19:28 +0200 Subject: [PATCH 115/194] Avoid the storage controller in test_tenant_creation_fails (#8392) As described in #8385, the likely source for flakiness in test_tenant_creation_fails is the following sequence of events: 1. test instructs the storage controller to create the tenant 2. storage controller adds the tenant and persists it to the database. issues a creation request 3. the pageserver restarts with the failpoint disabled 4. storage controller's background reconciliation still wants to create the tenant 5. pageserver gets new request to create the tenant from background reconciliation This commit just avoids the storage controller entirely. It has its own set of issues, as the re-attach request will obviously not include the tenant, but it's still useful to test for non-existence of the tenant. The generation is also not optional any more during tenant attachment. If you omit it, the pageserver yields an error. We change the signature of `tenant_attach` to reflect that. Alternative to #8385 Fixes #8266 --- test_runner/fixtures/neon_fixtures.py | 2 +- test_runner/fixtures/pageserver/http.py | 2 +- test_runner/regress/test_tenants.py | 13 +++---------- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index fe4a33445834..625e9096f58f 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2786,8 +2786,8 @@ def tenant_attach( ) return client.tenant_attach( tenant_id, + generation, config, - generation=generation, ) def tenant_detach(self, tenant_id: TenantId): diff --git a/test_runner/fixtures/pageserver/http.py b/test_runner/fixtures/pageserver/http.py index d66b94948a8e..f1e3d1a30941 100644 --- a/test_runner/fixtures/pageserver/http.py +++ b/test_runner/fixtures/pageserver/http.py @@ -238,8 +238,8 @@ def tenant_list(self) -> List[Dict[Any, Any]]: def tenant_attach( self, tenant_id: Union[TenantId, TenantShardId], + generation: int, config: None | Dict[str, Any] = None, - generation: Optional[int] = None, ): config = config or {} diff --git a/test_runner/regress/test_tenants.py b/test_runner/regress/test_tenants.py index 04b3fdd80fa5..0ebf714de080 100644 --- a/test_runner/regress/test_tenants.py +++ b/test_runner/regress/test_tenants.py @@ -45,17 +45,10 @@ def test_tenant_creation_fails(neon_simple_env: NeonEnv): # Failure to write a config to local disk makes the pageserver assume that local disk is bad and abort the process pageserver_http.configure_failpoints(("tenant-config-before-write", "return")) - # Storage controller will see a torn TCP connection when the crash point is reached, and follow an unclean 500 error path - neon_simple_env.storage_controller.allowed_errors.extend( - [ - ".*Reconcile not done yet while creating tenant.*", - ".*Reconcile error: receive body: error sending request.*", - ".*Error processing HTTP request: InternalServerError.*", - ] - ) + tenant_id = TenantId.generate() - with pytest.raises(Exception, match="error sending request"): - _ = neon_simple_env.neon_cli.create_tenant() + with pytest.raises(requests.exceptions.ConnectionError, match="Connection aborted"): + neon_simple_env.pageserver.http_client().tenant_attach(tenant_id=tenant_id, generation=1) # Any files left behind on disk during failed creation do not prevent # a retry from succeeding. Restart pageserver with no failpoints. From d2ee760eb2ad2ad637d10e5ab1bc44e9215bc2fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jul 2024 12:20:23 +0200 Subject: [PATCH 116/194] build(deps): bump setuptools from 65.5.1 to 70.0.0 (#8387) Bumps [setuptools](https://github.com/pypa/setuptools) from 65.5.1 to 70.0.0. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: a-masterov <72613290+a-masterov@users.noreply.github.com> --- poetry.lock | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index 809114141188..5192a574ccbd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2641,19 +2641,18 @@ pbr = "*" [[package]] name = "setuptools" -version = "65.5.1" +version = "70.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "setuptools-65.5.1-py3-none-any.whl", hash = "sha256:d0b9a8433464d5800cbe05094acf5c6d52a91bfac9b52bcfc4d41382be5d5d31"}, - {file = "setuptools-65.5.1.tar.gz", hash = "sha256:e197a19aa8ec9722928f2206f8de752def0e4c9fc6953527360d1c36d94ddb2f"}, + {file = "setuptools-70.0.0-py3-none-any.whl", hash = "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4"}, + {file = "setuptools-70.0.0.tar.gz", hash = "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" From a40b402957a99de5a484284d22462cd3191b4bb1 Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 16 Jul 2024 14:54:54 +0100 Subject: [PATCH 117/194] pageserver: clean up GcCutoffs names (#8379) - `horizon` is a confusing term, it's not at all obvious that this means space-based retention limit, rather than the total GC history limit. Rename to `GcCutoffs::space`. - `pitr` is less confusing, but still an unecessary level of indirection from what we really mean: a time-based condition. The fact that we use that that time-history for Point In Time Recovery doesn't mean we have to refer to time as "pitr" everywhere. Rename to `GcCutoffs::time`. --- pageserver/src/tenant.rs | 14 +-- pageserver/src/tenant/size.rs | 61 +++++-------- pageserver/src/tenant/timeline.rs | 94 +++++++++----------- pageserver/src/tenant/timeline/compaction.rs | 4 +- 4 files changed, 75 insertions(+), 98 deletions(-) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 6333fd3b6341..dc6f42eaebaf 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -2912,7 +2912,7 @@ impl Tenant { if let Some(ancestor_id) = timeline.get_ancestor_timeline_id() { if let Some(ancestor_gc_cutoffs) = gc_cutoffs.get(&ancestor_id) { target.within_ancestor_pitr = - timeline.get_ancestor_lsn() >= ancestor_gc_cutoffs.pitr; + timeline.get_ancestor_lsn() >= ancestor_gc_cutoffs.time; } } @@ -2928,7 +2928,7 @@ impl Tenant { timeline.metrics.pitr_history_size.set( timeline .get_last_record_lsn() - .checked_sub(target.cutoffs.pitr) + .checked_sub(target.cutoffs.time) .unwrap_or(Lsn(0)) .0, ); @@ -4262,7 +4262,7 @@ mod tests { .source() .unwrap() .to_string() - .contains("is earlier than latest GC horizon")); + .contains("is earlier than latest GC cutoff")); } } @@ -6718,8 +6718,8 @@ mod tests { { // Update GC info let mut guard = tline.gc_info.write().unwrap(); - guard.cutoffs.pitr = Lsn(0x30); - guard.cutoffs.horizon = Lsn(0x30); + guard.cutoffs.time = Lsn(0x30); + guard.cutoffs.space = Lsn(0x30); } let expected_result = [ @@ -7109,8 +7109,8 @@ mod tests { *guard = GcInfo { retain_lsns: vec![], cutoffs: GcCutoffs { - pitr: Lsn(0x30), - horizon: Lsn(0x30), + time: Lsn(0x30), + space: Lsn(0x30), }, leases: Default::default(), within_ancestor_pitr: false, diff --git a/pageserver/src/tenant/size.rs b/pageserver/src/tenant/size.rs index 23354417e788..e4728ca8a8cb 100644 --- a/pageserver/src/tenant/size.rs +++ b/pageserver/src/tenant/size.rs @@ -135,11 +135,9 @@ pub struct TimelineInputs { ancestor_lsn: Lsn, last_record: Lsn, latest_gc_cutoff: Lsn, - horizon_cutoff: Lsn, - pitr_cutoff: Lsn, /// Cutoff point based on GC settings - next_gc_cutoff: Lsn, + next_pitr_cutoff: Lsn, /// Cutoff point calculated from the user-supplied 'max_retention_period' retention_param_cutoff: Option, @@ -150,7 +148,7 @@ pub struct TimelineInputs { /// Gathers the inputs for the tenant sizing model. /// -/// Tenant size does not consider the latest state, but only the state until next_gc_cutoff, which +/// Tenant size does not consider the latest state, but only the state until next_pitr_cutoff, which /// is updated on-demand, during the start of this calculation and separate from the /// [`TimelineInputs::latest_gc_cutoff`]. /// @@ -158,11 +156,8 @@ pub struct TimelineInputs { /// /// ```text /// 0-----|---------|----|------------| · · · · · |·> lsn -/// initdb_lsn branchpoints* next_gc_cutoff latest +/// initdb_lsn branchpoints* next_pitr_cutoff latest /// ``` -/// -/// Until gc_horizon_cutoff > `Timeline::last_record_lsn` for any of the tenant's timelines, the -/// tenant size will be zero. pub(super) async fn gather_inputs( tenant: &Tenant, limit: &Arc, @@ -172,7 +167,7 @@ pub(super) async fn gather_inputs( cancel: &CancellationToken, ctx: &RequestContext, ) -> Result { - // refresh is needed to update gc related pitr_cutoff and horizon_cutoff + // refresh is needed to update [`timeline::GcCutoffs`] tenant.refresh_gc_info(cancel, ctx).await?; // Collect information about all the timelines @@ -236,20 +231,18 @@ pub(super) async fn gather_inputs( // we don't consider the `Timeline::disk_consistent_lsn` at all, because we are not // actually removing files. // - // We only consider [`GcInfo::pitr_cutoff`], and not [`GcInfo::horizon_cutoff`], because from + // We only consider [`timeline::GcCutoffs::time`], and not [`timeline::GcCutoffs::space`], because from // a user's perspective they have only requested retention up to the time bound (pitr_cutoff), rather - // than a space bound (horizon cutoff). This means that if someone drops a database and waits for their + // than our internal space cutoff. This means that if someone drops a database and waits for their // PITR interval, they will see synthetic size decrease, even if we are still storing data inside - // horizon_cutoff. - let pitr_cutoff = gc_info.cutoffs.pitr; - let horizon_cutoff = gc_info.cutoffs.horizon; - let mut next_gc_cutoff = pitr_cutoff; + // the space cutoff. + let mut next_pitr_cutoff = gc_info.cutoffs.time; // If the caller provided a shorter retention period, use that instead of the GC cutoff. let retention_param_cutoff = if let Some(max_retention_period) = max_retention_period { let param_cutoff = Lsn(last_record_lsn.0.saturating_sub(max_retention_period)); - if next_gc_cutoff < param_cutoff { - next_gc_cutoff = param_cutoff; + if next_pitr_cutoff < param_cutoff { + next_pitr_cutoff = param_cutoff; } Some(param_cutoff) } else { @@ -263,7 +256,7 @@ pub(super) async fn gather_inputs( .copied() .collect::>(); - // next_gc_cutoff in parent branch are not of interest (right now at least), nor do we + // next_pitr_cutoff in parent branch are not of interest (right now at least), nor do we // want to query any logical size before initdb_lsn. let branch_start_lsn = cmp::max(ancestor_lsn, timeline.initdb_lsn); @@ -291,10 +284,10 @@ pub(super) async fn gather_inputs( ) } - // Add a point for the GC cutoff - let branch_start_needed = next_gc_cutoff <= branch_start_lsn; + // Add a point for the PITR cutoff + let branch_start_needed = next_pitr_cutoff <= branch_start_lsn; if !branch_start_needed { - lsns.push((next_gc_cutoff, LsnKind::GcCutOff)); + lsns.push((next_pitr_cutoff, LsnKind::GcCutOff)); } lsns.sort_unstable(); @@ -333,7 +326,7 @@ pub(super) async fn gather_inputs( parent: Some(parent), lsn: lsn.0, size: None, - needed: lsn > next_gc_cutoff, + needed: lsn > next_pitr_cutoff, }, timeline_id: timeline.timeline_id, kind, @@ -357,8 +350,8 @@ pub(super) async fn gather_inputs( segment: Segment { parent: Some(lease_parent), lsn: lsn.0, - size: None, // Filled in later, if necessary - needed: lsn > next_gc_cutoff, // only needed if the point is within rentention. + size: None, // Filled in later, if necessary + needed: lsn > next_pitr_cutoff, // only needed if the point is within rentention. }, timeline_id: timeline.timeline_id, kind: LsnKind::LeaseStart, @@ -398,9 +391,7 @@ pub(super) async fn gather_inputs( last_record: last_record_lsn, // this is not used above, because it might not have updated recently enough latest_gc_cutoff: *timeline.get_latest_gc_cutoff_lsn(), - horizon_cutoff, - pitr_cutoff, - next_gc_cutoff, + next_pitr_cutoff, retention_param_cutoff, lease_points, }); @@ -742,9 +733,7 @@ fn verify_size_for_multiple_branches() { "ancestor_lsn": "0/18D3D98", "last_record": "0/2230CD0", "latest_gc_cutoff": "0/1698C48", - "horizon_cutoff": "0/2210CD0", - "pitr_cutoff": "0/2210CD0", - "next_gc_cutoff": "0/2210CD0", + "next_pitr_cutoff": "0/2210CD0", "retention_param_cutoff": null, "lease_points": [] }, @@ -753,9 +742,7 @@ fn verify_size_for_multiple_branches() { "ancestor_lsn": "0/176D998", "last_record": "0/1837770", "latest_gc_cutoff": "0/1698C48", - "horizon_cutoff": "0/1817770", - "pitr_cutoff": "0/1817770", - "next_gc_cutoff": "0/1817770", + "next_pitr_cutoff": "0/1817770", "retention_param_cutoff": null, "lease_points": [] }, @@ -764,9 +751,7 @@ fn verify_size_for_multiple_branches() { "ancestor_lsn": "0/0", "last_record": "0/18D3D98", "latest_gc_cutoff": "0/1698C48", - "horizon_cutoff": "0/18B3D98", - "pitr_cutoff": "0/18B3D98", - "next_gc_cutoff": "0/18B3D98", + "next_pitr_cutoff": "0/18B3D98", "retention_param_cutoff": null, "lease_points": [] } @@ -820,9 +805,7 @@ fn verify_size_for_one_branch() { "ancestor_lsn": "0/0", "last_record": "47/280A5860", "latest_gc_cutoff": "47/240A5860", - "horizon_cutoff": "47/240A5860", - "pitr_cutoff": "47/240A5860", - "next_gc_cutoff": "47/240A5860", + "next_pitr_cutoff": "47/240A5860", "retention_param_cutoff": "0/0", "lease_points": [] } diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 48a5b2d32bf7..3d3d3ac34de1 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -478,37 +478,32 @@ impl GcInfo { } } -/// The `GcInfo` component describing which Lsns need to be retained. +/// The `GcInfo` component describing which Lsns need to be retained. Functionally, this +/// is a single number (the oldest LSN which we must retain), but it internally distinguishes +/// between time-based and space-based retention for observability and consumption metrics purposes. #[derive(Debug)] pub(crate) struct GcCutoffs { - /// Keep everything newer than this point. - /// - /// This is calculated by subtracting 'gc_horizon' setting from - /// last-record LSN - /// - /// FIXME: is this inclusive or exclusive? - pub(crate) horizon: Lsn, + /// Calculated from the [`TenantConf::gc_horizon`], this LSN indicates how much + /// history we must keep to retain a specified number of bytes of WAL. + pub(crate) space: Lsn, - /// In addition to 'retain_lsns' and 'horizon_cutoff', keep everything newer than this - /// point. - /// - /// This is calculated by finding a number such that a record is needed for PITR - /// if only if its LSN is larger than 'pitr_cutoff'. - pub(crate) pitr: Lsn, + /// Calculated from [`TenantConf::pitr_interval`], this LSN indicates how much + /// history we must keep to enable reading back at least the PITR interval duration. + pub(crate) time: Lsn, } impl Default for GcCutoffs { fn default() -> Self { Self { - horizon: Lsn::INVALID, - pitr: Lsn::INVALID, + space: Lsn::INVALID, + time: Lsn::INVALID, } } } impl GcCutoffs { fn select_min(&self) -> Lsn { - std::cmp::min(self.horizon, self.pitr) + std::cmp::min(self.space, self.time) } } @@ -867,7 +862,7 @@ impl Timeline { let gc_info = self.gc_info.read().unwrap(); let history = self .get_last_record_lsn() - .checked_sub(gc_info.cutoffs.pitr) + .checked_sub(gc_info.cutoffs.time) .unwrap_or(Lsn(0)) .0; (history, gc_info.within_ancestor_pitr) @@ -1566,7 +1561,7 @@ impl Timeline { ) -> anyhow::Result<()> { ensure!( lsn >= **latest_gc_cutoff_lsn, - "LSN {} is earlier than latest GC horizon {} (we might've already garbage collected needed data)", + "LSN {} is earlier than latest GC cutoff {} (we might've already garbage collected needed data)", lsn, **latest_gc_cutoff_lsn, ); @@ -4944,18 +4939,18 @@ impl Timeline { /// garbage collection. /// /// We calculate two cutoffs, one based on time and one based on WAL size. `pitr` - /// controls the time cutoff (or ZERO to disable time-based retention), and `cutoff_horizon` controls + /// controls the time cutoff (or ZERO to disable time-based retention), and `space_cutoff` controls /// the space-based retention. /// /// This function doesn't simply to calculate time & space based retention: it treats time-based /// retention as authoritative if enabled, and falls back to space-based retention if calculating /// the LSN for a time point isn't possible. Therefore the GcCutoffs::horizon in the response might - /// be different to the `cutoff_horizon` input. Callers should treat the min() of the two cutoffs + /// be different to the `space_cutoff` input. Callers should treat the min() of the two cutoffs /// in the response as the GC cutoff point for the timeline. #[instrument(skip_all, fields(timeline_id=%self.timeline_id))] pub(super) async fn find_gc_cutoffs( &self, - cutoff_horizon: Lsn, + space_cutoff: Lsn, pitr: Duration, cancel: &CancellationToken, ctx: &RequestContext, @@ -4972,8 +4967,8 @@ impl Timeline { // Unit tests which specify zero PITR interval expect to avoid doing any I/O for timestamp lookup if pitr == Duration::ZERO { return Ok(GcCutoffs { - pitr: self.get_last_record_lsn(), - horizon: cutoff_horizon, + time: self.get_last_record_lsn(), + space: space_cutoff, }); } } @@ -4981,8 +4976,7 @@ impl Timeline { // Calculate a time-based limit on how much to retain: // - if PITR interval is set, then this is our cutoff. // - if PITR interval is not set, then we do a lookup - // based on DEFAULT_PITR_INTERVAL, so that size-based retention (horizon) - // does not result in keeping history around permanently on idle databases. + // based on DEFAULT_PITR_INTERVAL, so that size-based retention does not result in keeping history around permanently on idle databases. let time_cutoff = { let now = SystemTime::now(); let time_range = if pitr == Duration::ZERO { @@ -5023,31 +5017,31 @@ impl Timeline { // PITR is not set. Retain the size-based limit, or the default time retention, // whichever requires less data. GcCutoffs { - pitr: std::cmp::max(time_cutoff, cutoff_horizon), - horizon: std::cmp::max(time_cutoff, cutoff_horizon), + time: self.get_last_record_lsn(), + space: std::cmp::max(time_cutoff, space_cutoff), } } (Duration::ZERO, None) => { // PITR is not set, and time lookup failed GcCutoffs { - pitr: self.get_last_record_lsn(), - horizon: cutoff_horizon, + time: self.get_last_record_lsn(), + space: space_cutoff, } } (_, None) => { // PITR interval is set & we didn't look up a timestamp successfully. Conservatively assume PITR // cannot advance beyond what was already GC'd, and respect space-based retention GcCutoffs { - pitr: *self.get_latest_gc_cutoff_lsn(), - horizon: cutoff_horizon, + time: *self.get_latest_gc_cutoff_lsn(), + space: space_cutoff, } } (_, Some(time_cutoff)) => { // PITR interval is set and we looked up timestamp successfully. Ignore // size based retention and make time cutoff authoritative GcCutoffs { - pitr: time_cutoff, - horizon: time_cutoff, + time: time_cutoff, + space: time_cutoff, } } }) @@ -5074,11 +5068,11 @@ impl Timeline { return Err(GcError::TimelineCancelled); } - let (horizon_cutoff, pitr_cutoff, retain_lsns, max_lsn_with_valid_lease) = { + let (space_cutoff, time_cutoff, retain_lsns, max_lsn_with_valid_lease) = { let gc_info = self.gc_info.read().unwrap(); - let horizon_cutoff = min(gc_info.cutoffs.horizon, self.get_disk_consistent_lsn()); - let pitr_cutoff = gc_info.cutoffs.pitr; + let space_cutoff = min(gc_info.cutoffs.space, self.get_disk_consistent_lsn()); + let time_cutoff = gc_info.cutoffs.time; let retain_lsns = gc_info.retain_lsns.clone(); // Gets the maximum LSN that holds the valid lease. @@ -5088,14 +5082,14 @@ impl Timeline { let max_lsn_with_valid_lease = gc_info.leases.last_key_value().map(|(lsn, _)| *lsn); ( - horizon_cutoff, - pitr_cutoff, + space_cutoff, + time_cutoff, retain_lsns, max_lsn_with_valid_lease, ) }; - let mut new_gc_cutoff = Lsn::min(horizon_cutoff, pitr_cutoff); + let mut new_gc_cutoff = Lsn::min(space_cutoff, time_cutoff); let standby_horizon = self.standby_horizon.load(); // Hold GC for the standby, but as a safety guard do it only within some // reasonable lag. @@ -5124,8 +5118,8 @@ impl Timeline { let res = self .gc_timeline( - horizon_cutoff, - pitr_cutoff, + space_cutoff, + time_cutoff, retain_lsns, max_lsn_with_valid_lease, new_gc_cutoff, @@ -5143,8 +5137,8 @@ impl Timeline { async fn gc_timeline( &self, - horizon_cutoff: Lsn, - pitr_cutoff: Lsn, + space_cutoff: Lsn, + time_cutoff: Lsn, retain_lsns: Vec, max_lsn_with_valid_lease: Option, new_gc_cutoff: Lsn, @@ -5205,22 +5199,22 @@ impl Timeline { result.layers_total += 1; // 1. Is it newer than GC horizon cutoff point? - if l.get_lsn_range().end > horizon_cutoff { + if l.get_lsn_range().end > space_cutoff { debug!( - "keeping {} because it's newer than horizon_cutoff {}", + "keeping {} because it's newer than space_cutoff {}", l.layer_name(), - horizon_cutoff, + space_cutoff, ); result.layers_needed_by_cutoff += 1; continue 'outer; } // 2. It is newer than PiTR cutoff point? - if l.get_lsn_range().end > pitr_cutoff { + if l.get_lsn_range().end > time_cutoff { debug!( - "keeping {} because it's newer than pitr_cutoff {}", + "keeping {} because it's newer than time_cutoff {}", l.layer_name(), - pitr_cutoff, + time_cutoff, ); result.layers_needed_by_pitr += 1; continue 'outer; diff --git a/pageserver/src/tenant/timeline/compaction.rs b/pageserver/src/tenant/timeline/compaction.rs index eec5e5e53cf7..cbb330334104 100644 --- a/pageserver/src/tenant/timeline/compaction.rs +++ b/pageserver/src/tenant/timeline/compaction.rs @@ -195,7 +195,7 @@ impl Timeline { tracing::info!( "latest_gc_cutoff: {}, pitr cutoff {}", *latest_gc_cutoff, - self.gc_info.read().unwrap().cutoffs.pitr + self.gc_info.read().unwrap().cutoffs.time ); let layers = self.layers.read().await; @@ -990,7 +990,7 @@ impl Timeline { "enhanced legacy compaction currently does not support retain_lsns (branches)" ))); } - let gc_cutoff = Lsn::min(gc_info.cutoffs.horizon, gc_info.cutoffs.pitr); + let gc_cutoff = gc_info.cutoffs.select_min(); let mut selected_layers = Vec::new(); // TODO: consider retain_lsns drop(gc_info); From b5ab0555265d72b2cdd86ee259d84847409ad8ad Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Fri, 12 Jul 2024 13:46:14 -0500 Subject: [PATCH 118/194] Rename compute migrations to start at 1 This matches what we put into the neon_migration.migration_id table. --- compute_tools/src/migration.rs | 17 +++++++++++++--- ...sql => 0001-neon_superuser_bypass_rls.sql} | 0 ...1-alter_roles.sql => 0002-alter_roles.sql} | 0 ...create_subscription_to_neon_superuser.sql} | 0 ...04-grant_pg_monitor_to_neon_superuser.sql} | 0 ...grant_all_on_tables_to_neon_superuser.sql} | 0 ...nt_all_on_sequences_to_neon_superuser.sql} | 0 ...s_to_neon_superuser_with_grant_option.sql} | 0 ...s_to_neon_superuser_with_grant_option.sql} | 0 ...lication_for_previously_allowed_roles.sql} | 0 ...nchronization_funcs_to_neon_superuser.sql} | 0 compute_tools/src/spec.rs | 20 +++++++++---------- 12 files changed, 24 insertions(+), 13 deletions(-) rename compute_tools/src/migrations/{0000-neon_superuser_bypass_rls.sql => 0001-neon_superuser_bypass_rls.sql} (100%) rename compute_tools/src/migrations/{0001-alter_roles.sql => 0002-alter_roles.sql} (100%) rename compute_tools/src/migrations/{0002-grant_pg_create_subscription_to_neon_superuser.sql => 0003-grant_pg_create_subscription_to_neon_superuser.sql} (100%) rename compute_tools/src/migrations/{0003-grant_pg_monitor_to_neon_superuser.sql => 0004-grant_pg_monitor_to_neon_superuser.sql} (100%) rename compute_tools/src/migrations/{0004-grant_all_on_tables_to_neon_superuser.sql => 0005-grant_all_on_tables_to_neon_superuser.sql} (100%) rename compute_tools/src/migrations/{0005-grant_all_on_sequences_to_neon_superuser.sql => 0006-grant_all_on_sequences_to_neon_superuser.sql} (100%) rename compute_tools/src/migrations/{0006-grant_all_on_tables_to_neon_superuser_with_grant_option.sql => 0007-grant_all_on_tables_to_neon_superuser_with_grant_option.sql} (100%) rename compute_tools/src/migrations/{0007-grant_all_on_sequences_to_neon_superuser_with_grant_option.sql => 0008-grant_all_on_sequences_to_neon_superuser_with_grant_option.sql} (100%) rename compute_tools/src/migrations/{0008-revoke_replication_for_previously_allowed_roles.sql => 0009-revoke_replication_for_previously_allowed_roles.sql} (100%) rename compute_tools/src/migrations/{0009-grant_snapshot_synchronization_funcs_to_neon_superuser.sql => 0010-grant_snapshot_synchronization_funcs_to_neon_superuser.sql} (100%) diff --git a/compute_tools/src/migration.rs b/compute_tools/src/migration.rs index 61dcf01c8448..241ccd41001a 100644 --- a/compute_tools/src/migration.rs +++ b/compute_tools/src/migration.rs @@ -66,17 +66,28 @@ impl<'m> MigrationRunner<'m> { .context("run_migrations begin")?; while current_migration < self.migrations.len() { + macro_rules! migration_id { + ($cm:expr) => { + ($cm + 1) as i64 + }; + } + let migration = self.migrations[current_migration]; if migration.starts_with("-- SKIP") { - info!("Skipping migration id={}", current_migration); + info!("Skipping migration id={}", migration_id!(current_migration)); } else { info!( "Running migration id={}:\n{}\n", - current_migration, migration + migration_id!(current_migration), + migration ); + self.client.simple_query(migration).with_context(|| { - format!("run_migration current_migration={}", current_migration) + format!( + "run_migration migration id={}", + migration_id!(current_migration) + ) })?; } diff --git a/compute_tools/src/migrations/0000-neon_superuser_bypass_rls.sql b/compute_tools/src/migrations/0001-neon_superuser_bypass_rls.sql similarity index 100% rename from compute_tools/src/migrations/0000-neon_superuser_bypass_rls.sql rename to compute_tools/src/migrations/0001-neon_superuser_bypass_rls.sql diff --git a/compute_tools/src/migrations/0001-alter_roles.sql b/compute_tools/src/migrations/0002-alter_roles.sql similarity index 100% rename from compute_tools/src/migrations/0001-alter_roles.sql rename to compute_tools/src/migrations/0002-alter_roles.sql diff --git a/compute_tools/src/migrations/0002-grant_pg_create_subscription_to_neon_superuser.sql b/compute_tools/src/migrations/0003-grant_pg_create_subscription_to_neon_superuser.sql similarity index 100% rename from compute_tools/src/migrations/0002-grant_pg_create_subscription_to_neon_superuser.sql rename to compute_tools/src/migrations/0003-grant_pg_create_subscription_to_neon_superuser.sql diff --git a/compute_tools/src/migrations/0003-grant_pg_monitor_to_neon_superuser.sql b/compute_tools/src/migrations/0004-grant_pg_monitor_to_neon_superuser.sql similarity index 100% rename from compute_tools/src/migrations/0003-grant_pg_monitor_to_neon_superuser.sql rename to compute_tools/src/migrations/0004-grant_pg_monitor_to_neon_superuser.sql diff --git a/compute_tools/src/migrations/0004-grant_all_on_tables_to_neon_superuser.sql b/compute_tools/src/migrations/0005-grant_all_on_tables_to_neon_superuser.sql similarity index 100% rename from compute_tools/src/migrations/0004-grant_all_on_tables_to_neon_superuser.sql rename to compute_tools/src/migrations/0005-grant_all_on_tables_to_neon_superuser.sql diff --git a/compute_tools/src/migrations/0005-grant_all_on_sequences_to_neon_superuser.sql b/compute_tools/src/migrations/0006-grant_all_on_sequences_to_neon_superuser.sql similarity index 100% rename from compute_tools/src/migrations/0005-grant_all_on_sequences_to_neon_superuser.sql rename to compute_tools/src/migrations/0006-grant_all_on_sequences_to_neon_superuser.sql diff --git a/compute_tools/src/migrations/0006-grant_all_on_tables_to_neon_superuser_with_grant_option.sql b/compute_tools/src/migrations/0007-grant_all_on_tables_to_neon_superuser_with_grant_option.sql similarity index 100% rename from compute_tools/src/migrations/0006-grant_all_on_tables_to_neon_superuser_with_grant_option.sql rename to compute_tools/src/migrations/0007-grant_all_on_tables_to_neon_superuser_with_grant_option.sql diff --git a/compute_tools/src/migrations/0007-grant_all_on_sequences_to_neon_superuser_with_grant_option.sql b/compute_tools/src/migrations/0008-grant_all_on_sequences_to_neon_superuser_with_grant_option.sql similarity index 100% rename from compute_tools/src/migrations/0007-grant_all_on_sequences_to_neon_superuser_with_grant_option.sql rename to compute_tools/src/migrations/0008-grant_all_on_sequences_to_neon_superuser_with_grant_option.sql diff --git a/compute_tools/src/migrations/0008-revoke_replication_for_previously_allowed_roles.sql b/compute_tools/src/migrations/0009-revoke_replication_for_previously_allowed_roles.sql similarity index 100% rename from compute_tools/src/migrations/0008-revoke_replication_for_previously_allowed_roles.sql rename to compute_tools/src/migrations/0009-revoke_replication_for_previously_allowed_roles.sql diff --git a/compute_tools/src/migrations/0009-grant_snapshot_synchronization_funcs_to_neon_superuser.sql b/compute_tools/src/migrations/0010-grant_snapshot_synchronization_funcs_to_neon_superuser.sql similarity index 100% rename from compute_tools/src/migrations/0009-grant_snapshot_synchronization_funcs_to_neon_superuser.sql rename to compute_tools/src/migrations/0010-grant_snapshot_synchronization_funcs_to_neon_superuser.sql diff --git a/compute_tools/src/spec.rs b/compute_tools/src/spec.rs index 1d12b88c7ce3..6a872638219f 100644 --- a/compute_tools/src/spec.rs +++ b/compute_tools/src/spec.rs @@ -777,21 +777,21 @@ pub fn handle_migrations(client: &mut Client) -> Result<()> { // Add new migrations in numerical order. let migrations = [ - include_str!("./migrations/0000-neon_superuser_bypass_rls.sql"), - include_str!("./migrations/0001-alter_roles.sql"), - include_str!("./migrations/0002-grant_pg_create_subscription_to_neon_superuser.sql"), - include_str!("./migrations/0003-grant_pg_monitor_to_neon_superuser.sql"), - include_str!("./migrations/0004-grant_all_on_tables_to_neon_superuser.sql"), - include_str!("./migrations/0005-grant_all_on_sequences_to_neon_superuser.sql"), + include_str!("./migrations/0001-neon_superuser_bypass_rls.sql"), + include_str!("./migrations/0002-alter_roles.sql"), + include_str!("./migrations/0003-grant_pg_create_subscription_to_neon_superuser.sql"), + include_str!("./migrations/0004-grant_pg_monitor_to_neon_superuser.sql"), + include_str!("./migrations/0005-grant_all_on_tables_to_neon_superuser.sql"), + include_str!("./migrations/0006-grant_all_on_sequences_to_neon_superuser.sql"), include_str!( - "./migrations/0006-grant_all_on_tables_to_neon_superuser_with_grant_option.sql" + "./migrations/0007-grant_all_on_tables_to_neon_superuser_with_grant_option.sql" ), include_str!( - "./migrations/0007-grant_all_on_sequences_to_neon_superuser_with_grant_option.sql" + "./migrations/0008-grant_all_on_sequences_to_neon_superuser_with_grant_option.sql" ), - include_str!("./migrations/0008-revoke_replication_for_previously_allowed_roles.sql"), + include_str!("./migrations/0009-revoke_replication_for_previously_allowed_roles.sql"), include_str!( - "./migrations/0009-grant_snapshot_synchronization_funcs_to_neon_superuser.sql" + "./migrations/0010-grant_snapshot_synchronization_funcs_to_neon_superuser.sql" ), ]; From ba17025a57bc4916b3efeb0fd068f2ada7f668a8 Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Fri, 12 Jul 2024 13:38:51 -0500 Subject: [PATCH 119/194] Run each migration in its own transaction Previously, every migration was run in the same transaction. This is preparatory work for fixing CVE-2024-4317. --- compute_tools/src/migration.rs | 46 +++++++++++--------------- test_runner/fixtures/neon_fixtures.py | 6 ++-- test_runner/regress/test_migrations.py | 7 +--- 3 files changed, 24 insertions(+), 35 deletions(-) diff --git a/compute_tools/src/migration.rs b/compute_tools/src/migration.rs index 241ccd41001a..22ab145edaec 100644 --- a/compute_tools/src/migration.rs +++ b/compute_tools/src/migration.rs @@ -9,6 +9,9 @@ pub(crate) struct MigrationRunner<'m> { impl<'m> MigrationRunner<'m> { pub fn new(client: &'m mut Client, migrations: &'m [&'m str]) -> Self { + // The neon_migration.migration_id::id column is a bigint, which is equivalent to an i64 + assert!(migrations.len() + 1 < i64::MAX as usize); + Self { client, migrations } } @@ -22,11 +25,8 @@ impl<'m> MigrationRunner<'m> { Ok(row.get::<&str, i64>("id")) } - fn update_migration_id(&mut self) -> Result<()> { - let setval = format!( - "UPDATE neon_migration.migration_id SET id={}", - self.migrations.len() - ); + fn update_migration_id(&mut self, migration_id: i64) -> Result<()> { + let setval = format!("UPDATE neon_migration.migration_id SET id={}", migration_id); self.client .simple_query(&setval) @@ -57,14 +57,7 @@ impl<'m> MigrationRunner<'m> { pub fn run_migrations(mut self) -> Result<()> { self.prepare_migrations()?; - let mut current_migration: usize = self.get_migration_id()? as usize; - let starting_migration_id = current_migration; - - let query = "BEGIN"; - self.client - .simple_query(query) - .context("run_migrations begin")?; - + let mut current_migration = self.get_migration_id()? as usize; while current_migration < self.migrations.len() { macro_rules! migration_id { ($cm:expr) => { @@ -83,28 +76,29 @@ impl<'m> MigrationRunner<'m> { migration ); + self.client + .simple_query("BEGIN") + .context("begin migration")?; + self.client.simple_query(migration).with_context(|| { format!( - "run_migration migration id={}", + "run_migrations migration id={}", migration_id!(current_migration) ) })?; - } - current_migration += 1; - } + // Migration IDs start at 1 + self.update_migration_id(migration_id!(current_migration))?; - self.update_migration_id()?; + self.client + .simple_query("COMMIT") + .context("commit migration")?; - let query = "COMMIT"; - self.client - .simple_query(query) - .context("run_migrations commit")?; + info!("Finished migration id={}", migration_id!(current_migration)); + } - info!( - "Ran {} migrations", - (self.migrations.len() - starting_migration_id) - ); + current_migration += 1; + } Ok(()) } diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 625e9096f58f..4766b7251624 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -3798,13 +3798,13 @@ def respec(self, **kwargs): json.dump(dict(data_dict, **kwargs), file, indent=4) # Please note: Migrations only run if pg_skip_catalog_updates is false - def wait_for_migrations(self): + def wait_for_migrations(self, num_migrations: int = 10): with self.cursor() as cur: def check_migrations_done(): cur.execute("SELECT id FROM neon_migration.migration_id") - migration_id = cur.fetchall()[0][0] - assert migration_id != 0 + migration_id: int = cur.fetchall()[0][0] + assert migration_id >= num_migrations wait_until(20, 0.5, check_migrations_done) diff --git a/test_runner/regress/test_migrations.py b/test_runner/regress/test_migrations.py index 91bd3ea50caf..880dead4e8d9 100644 --- a/test_runner/regress/test_migrations.py +++ b/test_runner/regress/test_migrations.py @@ -11,17 +11,14 @@ def test_migrations(neon_simple_env: NeonEnv): endpoint.respec(skip_pg_catalog_updates=False) endpoint.start() - endpoint.wait_for_migrations() - num_migrations = 10 + endpoint.wait_for_migrations(num_migrations=num_migrations) with endpoint.cursor() as cur: cur.execute("SELECT id FROM neon_migration.migration_id") migration_id = cur.fetchall() assert migration_id[0][0] == num_migrations - endpoint.assert_log_contains(f"INFO handle_migrations: Ran {num_migrations} migrations") - endpoint.stop() endpoint.start() # We don't have a good way of knowing that the migrations code path finished executing @@ -31,5 +28,3 @@ def test_migrations(neon_simple_env: NeonEnv): cur.execute("SELECT id FROM neon_migration.migration_id") migration_id = cur.fetchall() assert migration_id[0][0] == num_migrations - - endpoint.assert_log_contains("INFO handle_migrations: Ran 0 migrations") From b197cc20fc4d2c474eec03d57ce855203e24c704 Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Mon, 15 Jul 2024 10:30:04 -0500 Subject: [PATCH 120/194] Hide import behind TYPE_CHECKING --- test_runner/regress/test_migrations.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test_runner/regress/test_migrations.py b/test_runner/regress/test_migrations.py index 880dead4e8d9..bdc5ca907ec1 100644 --- a/test_runner/regress/test_migrations.py +++ b/test_runner/regress/test_migrations.py @@ -1,6 +1,10 @@ +from __future__ import annotations + import time +from typing import TYPE_CHECKING -from fixtures.neon_fixtures import NeonEnv +if TYPE_CHECKING: + from fixtures.neon_fixtures import NeonEnv def test_migrations(neon_simple_env: NeonEnv): From 7cf59ae5b4b2ebf5a7685976cb74ae28dd25db08 Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Mon, 15 Jul 2024 10:35:49 -0500 Subject: [PATCH 121/194] Add some typing to Endpoint.respec() --- test_runner/fixtures/neon_fixtures.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 4766b7251624..2765ff916e63 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -3785,12 +3785,12 @@ def reconfigure( self.endpoint_id, self.tenant_id, pageserver_id, self.active_safekeepers ) - def respec(self, **kwargs): + def respec(self, **kwargs: Any) -> None: """Update the endpoint.json file used by control_plane.""" # Read config config_path = os.path.join(self.endpoint_path(), "endpoint.json") with open(config_path, "r") as f: - data_dict = json.load(f) + data_dict: dict[str, Any] = json.load(f) # Write it back updated with open(config_path, "w") as file: From 0950866fa8728896d04ac0fdf707813299f1d621 Mon Sep 17 00:00:00 2001 From: "Alex Chi Z." Date: Tue, 16 Jul 2024 15:43:24 -0400 Subject: [PATCH 122/194] fix(pageserver): limit num of delta layers for l0 compaction (#8391) ## Problem close https://github.com/neondatabase/neon/issues/8389 ## Summary of changes A quick mitigation for tenants with fast writes. We compact at most 60 delta layers at a time, expecting a memory footprint of 15GB. We will pick the oldest 60 L0 layers. This should be a relatively safe change so no test is added. Question is whether to make this parameter configurable via tenant config. --------- Signed-off-by: Alex Chi Z Co-authored-by: John Spray --- pageserver/src/tenant/timeline/compaction.rs | 31 ++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/pageserver/src/tenant/timeline/compaction.rs b/pageserver/src/tenant/timeline/compaction.rs index cbb330334104..f251b667c2fb 100644 --- a/pageserver/src/tenant/timeline/compaction.rs +++ b/pageserver/src/tenant/timeline/compaction.rs @@ -26,6 +26,7 @@ use utils::id::TimelineId; use crate::context::{AccessStatsBehavior, RequestContext, RequestContextBuilder}; use crate::page_cache; +use crate::tenant::config::defaults::{DEFAULT_CHECKPOINT_DISTANCE, DEFAULT_COMPACTION_THRESHOLD}; use crate::tenant::storage_layer::{AsLayerDesc, PersistentLayerDesc}; use crate::tenant::timeline::{drop_rlock, DeltaLayerWriter, ImageLayerWriter}; use crate::tenant::timeline::{Hole, ImageLayerCreationOutcome}; @@ -415,6 +416,7 @@ impl Timeline { .map(|x| guard.get_from_desc(&x)) .collect_vec(); stats.level0_deltas_count = Some(level0_deltas.len()); + // Only compact if enough layers have accumulated. let threshold = self.get_compaction_threshold(); if level0_deltas.is_empty() || level0_deltas.len() < threshold { @@ -445,6 +447,22 @@ impl Timeline { let mut prev_lsn_end = first_level0_delta.layer_desc().lsn_range.end; let mut deltas_to_compact = Vec::with_capacity(level0_deltas.len()); + // Accumulate the size of layers in `deltas_to_compact` + let mut deltas_to_compact_bytes = 0; + + // Under normal circumstances, we will accumulate up to compaction_interval L0s of size + // checkpoint_distance each. To avoid edge cases using extra system resources, bound our + // work in this function to only operate on this much delta data at once. + // + // Take the max of the configured value & the default, so that tests that configure tiny values + // can still use a sensible amount of memory, but if a deployed system configures bigger values we + // still let them compact a full stack of L0s in one go. + let delta_size_limit = std::cmp::max( + self.get_compaction_threshold(), + DEFAULT_COMPACTION_THRESHOLD, + ) as u64 + * std::cmp::max(self.get_checkpoint_distance(), DEFAULT_CHECKPOINT_DISTANCE); + deltas_to_compact.push(first_level0_delta.download_and_keep_resident().await?); for l in level0_deltas_iter { let lsn_range = &l.layer_desc().lsn_range; @@ -453,7 +471,20 @@ impl Timeline { break; } deltas_to_compact.push(l.download_and_keep_resident().await?); + deltas_to_compact_bytes += l.metadata().file_size; prev_lsn_end = lsn_range.end; + + if deltas_to_compact_bytes >= delta_size_limit { + info!( + l0_deltas_selected = deltas_to_compact.len(), + l0_deltas_total = level0_deltas.len(), + "L0 compaction picker hit max delta layer size limit: {}", + delta_size_limit + ); + + // Proceed with compaction, but only a subset of L0s + break; + } } let lsn_range = Range { start: deltas_to_compact From f4f0869dc841374921e7fb3ff353ecbc2b2267a0 Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 16 Jul 2024 20:55:29 +0100 Subject: [PATCH 123/194] pageserver: exclude un-read layers from short residence statistic (#8396) ## Problem The `evictions_with_low_residence_duration` is used as an indicator of cache thrashing. However, there are situations where it is quite legitimate to only have a short residence during compaction, where a delta is downloaded, used to generate an image layer, and then discarded. This can lead to false positive alerts. ## Summary of changes - Only track low residence duration for layers that have been accessed at least once (compaction doesn't count as an access). This will give us a metric that indicates thrashing on layers that the _user_ is using, rather than those we're downloading for housekeeping purposes. Once we add "layer visibility" as an explicit property of layers, this can also be used as a cleaner condition (residence of non-visible layers should never be alertable) --- pageserver/src/tenant/storage_layer.rs | 20 ++++++++++++++++++++ pageserver/src/tenant/storage_layer/layer.rs | 20 ++++++++++++++------ test_runner/regress/test_tenant_conf.py | 11 +++++++++++ 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index 62730f88b260..2f0c45317d9a 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -676,6 +676,26 @@ impl LayerAccessStats { }, } } + + /// Whether this layer has been accessed (excluding in [`AccessStatsBehavior::Skip`]). + /// + /// This indicates whether the layer has been used for some purpose that would motivate + /// us to keep it on disk, such as for serving a getpage request. + fn accessed(&self) -> bool { + let locked = self.0.lock().unwrap(); + let inner = &locked.for_eviction_policy; + + // Consider it accessed if the most recent access is more recent than + // the most recent change in residence status. + match ( + inner.last_accesses.recent(), + inner.last_residence_changes.recent(), + ) { + (None, _) => false, + (Some(_), None) => true, + (Some(a), Some(r)) => a.when >= r.timestamp, + } + } } /// Get a layer descriptor from a layer. diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index 02069c29d264..4500bc94dd66 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -1469,14 +1469,22 @@ impl LayerInner { let duration = SystemTime::now().duration_since(local_layer_mtime); match duration { Ok(elapsed) => { - timeline - .metrics - .evictions_with_low_residence_duration - .read() - .unwrap() - .observe(elapsed); + let accessed = self.access_stats.accessed(); + if accessed { + // Only layers used for reads contribute to our "low residence" metric that is used + // to detect thrashing. Layers promoted for other reasons (e.g. compaction) are allowed + // to be rapidly evicted without contributing to this metric. + timeline + .metrics + .evictions_with_low_residence_duration + .read() + .unwrap() + .observe(elapsed); + } + tracing::info!( residence_millis = elapsed.as_millis(), + accessed, "evicted layer after known residence period" ); } diff --git a/test_runner/regress/test_tenant_conf.py b/test_runner/regress/test_tenant_conf.py index 1a8bc3b98363..9fb7324fa15c 100644 --- a/test_runner/regress/test_tenant_conf.py +++ b/test_runner/regress/test_tenant_conf.py @@ -8,6 +8,7 @@ from fixtures.pageserver.utils import assert_tenant_state, wait_for_upload from fixtures.remote_storage import LocalFsStorage, RemoteStorageKind from fixtures.utils import wait_until +from fixtures.workload import Workload def test_tenant_config(neon_env_builder: NeonEnvBuilder): @@ -265,6 +266,13 @@ def test_live_reconfig_get_evictions_low_residence_duration_metric_threshold( (tenant_id, timeline_id) = env.initial_tenant, env.initial_timeline ps_http = env.pageserver.http_client() + # When we evict/download layers, we will use this Workload to generate getpage requests + # that touch some layers, as otherwise the pageserver doesn't report totally unused layers + # as problems when they have short residence duration. + workload = Workload(env, tenant_id, timeline_id) + workload.init() + workload.write_rows(100) + def get_metric(): metrics = ps_http.get_metrics() metric = metrics.query_one( @@ -285,6 +293,7 @@ def get_metric(): assert default_value == "1day" ps_http.download_all_layers(tenant_id, timeline_id) + workload.validate() ps_http.evict_all_layers(tenant_id, timeline_id) metric = get_metric() assert int(metric.value) > 0, "metric is updated" @@ -305,6 +314,7 @@ def get_metric(): assert int(metric.value) == 0 ps_http.download_all_layers(tenant_id, timeline_id) + workload.validate() ps_http.evict_all_layers(tenant_id, timeline_id) metric = get_metric() assert int(metric.labels["low_threshold_secs"]) == 2 * 24 * 60 * 60 @@ -318,6 +328,7 @@ def get_metric(): assert int(metric.value) == 0, "value resets if label changes" ps_http.download_all_layers(tenant_id, timeline_id) + workload.validate() ps_http.evict_all_layers(tenant_id, timeline_id) metric = get_metric() assert int(metric.labels["low_threshold_secs"]) == 2 * 60 * 60 From 4a90423292a2c6abec84a75d8c4cb2c3306baeed Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 16 Jul 2024 21:36:17 +0100 Subject: [PATCH 124/194] pageserver: reduce size of delta layer ValueRef (#8401) ## Problem ValueRef is an unnecessarily large structure, because it carries a cursor. L0 compaction currently instantiates gigabytes of these under some circumstances. ## Summary of changes - Carry a ref to the parent layer instead of a cursor, and construct a cursor on demand. This reduces RSS high watermark during L0 compaction by about 20%. --- .../src/tenant/storage_layer/delta_layer.rs | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 2d36ac744277..64412fe4afd1 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -1180,9 +1180,7 @@ impl DeltaLayerInner { let delta_key = DeltaKey::from_slice(key); let val_ref = ValueRef { blob_ref: BlobRef(value), - reader: BlockCursor::new(crate::tenant::block_io::BlockReaderRef::Adapter( - Adapter(self), - )), + layer: self, }; let pos = BlobRef(value).pos(); if let Some(last) = all_keys.last_mut() { @@ -1426,7 +1424,7 @@ impl DeltaLayerInner { let keys = self.load_keys(ctx).await?; async fn dump_blob(val: &ValueRef<'_>, ctx: &RequestContext) -> anyhow::Result { - let buf = val.reader.read_blob(val.blob_ref.pos(), ctx).await?; + let buf = val.load_raw(ctx).await?; let val = Value::des(&buf)?; let desc = match val { Value::Image(img) => { @@ -1461,8 +1459,7 @@ impl DeltaLayerInner { use pageserver_api::key::CHECKPOINT_KEY; use postgres_ffi::CheckPoint; if key == CHECKPOINT_KEY { - let buf = val.reader.read_blob(val.blob_ref.pos(), ctx).await?; - let val = Value::des(&buf)?; + let val = val.load(ctx).await?; match val { Value::Image(img) => { let checkpoint = CheckPoint::decode(&img)?; @@ -1547,17 +1544,24 @@ pub struct DeltaEntry<'a> { /// Reference to an on-disk value pub struct ValueRef<'a> { blob_ref: BlobRef, - reader: BlockCursor<'a>, + layer: &'a DeltaLayerInner, } impl<'a> ValueRef<'a> { /// Loads the value from disk pub async fn load(&self, ctx: &RequestContext) -> Result { - // theoretically we *could* record an access time for each, but it does not really matter - let buf = self.reader.read_blob(self.blob_ref.pos(), ctx).await?; + let buf = self.load_raw(ctx).await?; let val = Value::des(&buf)?; Ok(val) } + + async fn load_raw(&self, ctx: &RequestContext) -> Result> { + let reader = BlockCursor::new(crate::tenant::block_io::BlockReaderRef::Adapter(Adapter( + self.layer, + ))); + let buf = reader.read_blob(self.blob_ref.pos(), ctx).await?; + Ok(buf) + } } pub(crate) struct Adapter(T); From f7131834eb55efc2d49a4e660a763d590c74a0a2 Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 17 Jul 2024 15:25:35 +0100 Subject: [PATCH 125/194] docs/rfcs: timeline ancestor detach API (#6888) ## Problem When a tenant creates a new timeline that they will treat as their 'main' history, it is awkward to permanently retain an 'old main' timeline as its ancestor. Currently this is necessary because it is forbidden to delete a timeline which has descendents. ## Summary of changes A new pageserver API is proposed to 'adopt' data from a parent timeline into one of its children, such that the link between ancestor and child can be severed, leaving the parent in a state where it may then be deleted. --------- Co-authored-by: Joonas Koivunen --- docs/rfcs/034-ancestor-deletion.md | 252 +++++++++++++++++++++++++++++ 1 file changed, 252 insertions(+) create mode 100644 docs/rfcs/034-ancestor-deletion.md diff --git a/docs/rfcs/034-ancestor-deletion.md b/docs/rfcs/034-ancestor-deletion.md new file mode 100644 index 000000000000..7341d930e26d --- /dev/null +++ b/docs/rfcs/034-ancestor-deletion.md @@ -0,0 +1,252 @@ +# Ancestor Timeline Deletion + +Created on: 2024-02-23 + +Author: John Spray + +# Summary + +When a tenant creates a new timeline that they will treat as their 'main' history, +it is awkward to permanently retain an 'old main' timeline as its ancestor. Currently +this is necessary because it is forbidden to delete a timeline which has descendents. + +A new pageserver API is proposed to 'adopt' data from a parent timeline into +one of its children, such that the link between ancestor and child can be severed, +leaving the parent in a state where it may then be deleted. + +# Motivation + +Retaining parent timelines currently has two costs: + +- Cognitive load on users, who have to remember which is the "real" main timeline. +- Storage capacity cost, as the parent timeline will retain layers up to the + child's timeline point, even if the child fully covers its keyspace with image + layers and will never actually read from the parent. + +# Solution + +A new pageserver API `PUT /v1/tenant/:tenant_id/timeline/:timeline_id/detach_ancestor` +will be added. The `timeline_id` in this URL is that of the _child_ timeline that we +wish to detach from its parent. + +On success, this API will leave the following state: + +- The detached child timeline will no longer have an ancestor, and will contain all + the data needed to service reads without recursing into an ancestor. +- Any other children of the parent whose timeline points were at a lower LSN than + the detached child timeline will be modified to have the child timeline as their + new parent. +- The parent timeline will still exist, but the child will no longer have it as an + ancestor. If this was the last timeline that depended on the parent, then the + parent will become deletable. + +This API's implementation will consist of a series of retryable steps, such that +on failures/timeout it can safely be called again to reach the target state. + +## Example + +### Before + +The user has "rolled back" their project to LSN X, resulting in a "new main" +timeline. The parent "old main" timeline still exists, and they would like +to clean it up. + +They have two other timelines A and B. A is from before the rollback point, +and B is from after the rollback point. + +``` +----"old main" timeline-------X--------------------------------------------> + | | | + |-> child A | | + |-> "new main" timeline | + -> child B + +``` + +### After calling detach ancestor API + +The "new main" timeline is no longer dependent on old main, and neither +is child A, because it had a branch point before X. + +The user may now choose to delete child B and "old main" to get to +a pristine state. Child B is likely to be unwanted since the user +chose to roll back to X, and it branches from after X. However, we +don't assume this in the API; it is up to the user to delete it. + +``` +|----"old main" timeline----------------------------------------------------> + | + | + | + -> child B + +|----"new main" timeline---------> + | + |-> child A + + +``` + +### After removing timelines + +We end up with a totally clean state that leaves no trace that a rollback +ever happened: there is only one root timeline. + +``` +| ----"new main" timeline-----------> + | + |-> child A + + +``` + +## Caveats + +Important things for API users to bear in mind: + +- this API does not delete the parent timeline: you must still do that explicitly. +- if there are other child timelines ahead of the branch point of the detached + child, the parent won't be deletable: you must either delete or detach those + children. +- do _not_ simply loop over all children and detach them all: this can have an + extremely high storage cost. The detach ancestor API is intended for use on a single + timeline to make it the new "main". +- The detach ancestor API should also not be + exposed directly to the user as button/API, because they might decide + to click it for all the children and thereby generate many copies of the + parent's data -- the detach ancestor API should be used as part + of a high level "clean up after rollback" feature. + +## `detach_ancestor` API implementation + +Terms used in the following sections: + +- "the child": the timeline whose ID is specified in the detach ancestor API URL, also + called "new main" in the example. +- "the parent": the parent of "the child". Also called "old main" in the example. +- "the branch point" the ancestor_lsn of "the child" + +### Phase 1: write out adopted layers to S3 + +The child will "adopt" layers from the parent, such that its end state contains +all the parent's history as well as its own. + +For all layers in the parent's layer map whose high LSN is below the branch +point, issue S3 CopyObject requests to duplicate them into the child timeline's +prefix. Do not add them to the child's layer map yet. + +For delta layers in the parent's layer map which straddle the branch point, read them +and write out only content up to the branch point into new layer objects. + +This is a long running operation if the parent has many layers: it should be +implemented in a way that resumes rather than restarting from scratch, if the API +times out and is called again. + +As an optimization, if there are no other timelines that will be adopted into +the child, _and_ the child's image layers already full cover the branch LSN, +then we may skip adopting layers. + +### Phase 2: update the child's index + +Having written out all needed layers in phase 1, atomically link them all +into the child's IndexPart and upload to S3. This may be done while the +child Timeline is still running. + +### Phase 3: modify timelines ancestry + +Modify the child's ancestor to None, and upload its IndexPart to persist the change. + +For all timelines which have the same parent as the child, and have a branch +point lower than our branch point, switch their ancestor_timeline to the child, +and upload their IndexPart to persist the change. + +## Alternatives considered + +### Generate full image layer on child, rather than adopting parent deltas + +This would work for the case of a single child, but would prevent re-targeting +other timelines that depended on the parent. If we detached many children this +way, the storage cost would become prohibitive (consider a 1TB database with +100 child timelines: it would cost 100TiB if they all generated their own image layers). + +### Don't rewrite anything: just fake it in the API + +We could add a layer of indirection that let a child "pretend" that it had no +ancestor, when in reality it still had the parent. The pageserver API could +accept deletion of ancestor timelines, and just update child metadata to make +them look like they have no ancestor. + +This would not achieve the desired reduction in storage cost, and may well be more +complex to maintain than simply implementing the API described in this RFC. + +### Avoid copying objects: enable child index to use parent layers directly + +We could teach IndexPart to store a TimelineId for each layer, such that a child +timeline could reference a parent's layers directly, rather than copying them +into the child's prefix. + +This would impose a cost for the normal case of indices that only target the +timeline's own layers, add complexity, and break the useful simplifying +invariant that timelines "own" their own path. If child timelines were +referencing layers from the parent, we would have to ensure that the parent +never runs GC/compaction again, which would make the API less flexible (the +proposal in this RFC enables deletion of the parent but doesn't require it.) + +## Performance + +### Adopting layers + +- CopyObject is a relatively cheap operation, but we may need to issue tens of thousands + of such requests: this can take up to tens of seconds and will compete for RemoteStorage + semaphore units with other activity on the pageserver. +- If we are running on storage backend that doesn't implement CopyObject, then + this part will be much more expensive as we would stream all layer content + through the pageserver. This is no different to issuing a lot + of reads to a timeline that does not have a warm local cache: it will move + a lot of gigabytes, but that shouldn't break anything. +- Generating truncated layers for delta that straddle the branch point will + require streaming read/write of all the layers in question. + +### Updating timeline ancestry + +The simplest way to update timeline ancestry will probably be to stop and start +all the Timeline objects: this is preferable to the complexity of making their +ancestry mutable at runtime. + +There will be a corresponding "stutter" in the availability of the timelines, +of the order 10-100ms, which is the time taken to upload their IndexPart, and +restart the Timeline. + +# Interaction with other features + +## Concurrent timeline creation + +If new historic timelines are created using the parent as an ancestor while the +detach ancestor API is running, they will not be re-parented to the child. This +doesn't break anything, but it leaves the parent in a state where it might not +be possible to delete it. + +Since timeline creations are an explicit user action, this is not something we need to +worry about as the storage layer: a user who wants to delete their parent timeline will not create +new children, and if they do, they can choose to delete those children to +enable deleting the parent. + +For the least surprise to the user, before starting the detach ancestor branch +operation, the control plane should wait until all branches are created and not +allow any branches to be created before the branch point on the ancestor branch +while the operation is ongoing. + +## WAL based disaster recovery + +WAL based disaster recovery currently supports only restoring of the main +branch. Enabling WAL based disaster recovery in the future requires that we +keep a record which timeline generated the WAL and at which LSN was a parent +detached. Keep a list of timeline ids and the LSN in which they were detached in +the `index_part.json`. Limit the size of the list to 100 first entries, after +which the WAL disaster recovery will not be possible. + +## Sharded tenants + +For sharded tenants, calls to the detach ancestor API will pass through the storage +controller, which will handle them the same as timeline creations: invoke first +on shard zero, and then on all the other shards. From f2b8e390e77c157d8f7ebef573bb226a313a8478 Mon Sep 17 00:00:00 2001 From: Peter Bendel Date: Wed, 17 Jul 2024 16:56:32 +0200 Subject: [PATCH 126/194] Bodobolero/pgbench compare azure (#8409) ## Problem We want to run performance tests on all supported cloud providers. We want to run most tests on the postgres version which is default for new projects in production, currently (July 24) this is postgres version 16 ## Summary of changes - change default postgres version for some (performance) tests to 16 (which is our default for new projects in prod anyhow) - add azure region to pgbench_compare jobs - add azure region to pgvector benchmarking jobs - re-used project `weathered-snowflake-88107345` was prepared with 1 million embeddings running on 7 minCU 7 maxCU in azure region to compare with AWS region (pgvector indexing and hnsw queries) - see job pgbench-pgvector - Note we now have a 11 environments combinations where we run pgbench-compare and 5 are for k8s-pod (deprecated) which we can remove in the future once auto-scaling team approves. ## Logs A current run with the changes from this pull request is running here https://github.com/neondatabase/neon/actions/runs/9972096222 Note that we currently expect some failures due to - https://github.com/neondatabase/neon/issues/8275 - instability of projects on azure region --- .../actions/neon-project-create/action.yml | 4 +- .github/workflows/benchmarking.yml | 70 ++++++++++++++----- 2 files changed, 56 insertions(+), 18 deletions(-) diff --git a/.github/actions/neon-project-create/action.yml b/.github/actions/neon-project-create/action.yml index 16759ad03820..d4029bd37c1b 100644 --- a/.github/actions/neon-project-create/action.yml +++ b/.github/actions/neon-project-create/action.yml @@ -9,8 +9,8 @@ inputs: description: 'Region ID, if not set the project will be created in the default region' default: aws-us-east-2 postgres_version: - description: 'Postgres version; default is 15' - default: '15' + description: 'Postgres version; default is 16' + default: '16' api_host: description: 'Neon API host' default: console-stage.neon.build diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index d038f64f15b0..d785156a29b1 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -59,7 +59,7 @@ jobs: strategy: matrix: include: - - DEFAULT_PG_VERSION: 14 + - DEFAULT_PG_VERSION: 16 PLATFORM: "neon-staging" region_id: ${{ github.event.inputs.region_id || 'aws-us-east-2' }} provisioner: 'k8s-pod' @@ -146,6 +146,7 @@ jobs: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} replication-tests: + if: ${{ github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null }} env: POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install DEFAULT_PG_VERSION: 14 @@ -190,6 +191,7 @@ jobs: run_in_parallel: false save_perf_report: ${{ env.SAVE_PERF_REPORT }} extra_params: -m remote_cluster --timeout 5400 + pg_version: ${{ env.DEFAULT_PG_VERSION }} env: VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" @@ -215,11 +217,14 @@ jobs: # Available platforms: # - neon-captest-new: Freshly created project (1 CU) # - neon-captest-freetier: Use freetier-sized compute (0.25 CU) + # - neonvm-captest-azure-new: Freshly created project (1 CU) in azure region + # - neonvm-captest-azure-freetier: Use freetier-sized compute (0.25 CU) in azure region # - neon-captest-reuse: Reusing existing project # - rds-aurora: Aurora Postgres Serverless v2 with autoscaling from 0.5 to 2 ACUs # - rds-postgres: RDS Postgres db.m5.large instance (2 vCPU, 8 GiB) with gp3 EBS storage env: RUN_AWS_RDS_AND_AURORA: ${{ github.event.inputs.run_AWS_RDS_AND_AURORA || 'false' }} + DEFAULT_REGION_ID: ${{ github.event.inputs.region_id || 'aws-us-east-2' }} runs-on: ubuntu-22.04 outputs: pgbench-compare-matrix: ${{ steps.pgbench-compare-matrix.outputs.matrix }} @@ -230,23 +235,33 @@ jobs: - name: Generate matrix for pgbench benchmark id: pgbench-compare-matrix run: | + region_id_default=${{ env.DEFAULT_REGION_ID }} matrix='{ + "pg_version" : [ + 16 + ], + "region_id" : [ + "'"$region_id_default"'" + ], "platform": [ "neon-captest-new", "neon-captest-reuse", "neonvm-captest-new" ], "db_size": [ "10gb" ], - "include": [{ "platform": "neon-captest-freetier", "db_size": "3gb" }, - { "platform": "neon-captest-new", "db_size": "50gb" }, - { "platform": "neonvm-captest-freetier", "db_size": "3gb" }, - { "platform": "neonvm-captest-new", "db_size": "50gb" }, - { "platform": "neonvm-captest-sharding-reuse", "db_size": "50gb" }] + "include": [{ "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neon-captest-freetier", "db_size": "3gb" }, + { "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neon-captest-new", "db_size": "50gb" }, + { "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-freetier", "db_size": "3gb" }, + { "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new", "db_size": "50gb" }, + { "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-freetier", "db_size": "3gb" }, + { "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-new", "db_size": "10gb" }, + { "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-new", "db_size": "50gb" }, + { "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-sharding-reuse", "db_size": "50gb" }] }' if [ "$(date +%A)" = "Saturday" ]; then - matrix=$(echo "$matrix" | jq '.include += [{ "platform": "rds-postgres", "db_size": "10gb"}, - { "platform": "rds-aurora", "db_size": "50gb"}]') + matrix=$(echo "$matrix" | jq '.include += [{ "pg_version": 14, "region_id": "'"$region_id_default"'", "platform": "rds-postgres", "db_size": "10gb"}, + { "pg_version": 14, "region_id": "'"$region_id_default"'", "platform": "rds-aurora", "db_size": "50gb"}]') fi echo "matrix=$(echo "$matrix" | jq --compact-output '.')" >> $GITHUB_OUTPUT @@ -298,7 +313,7 @@ jobs: TEST_PG_BENCH_DURATIONS_MATRIX: "60m" TEST_PG_BENCH_SCALES_MATRIX: ${{ matrix.db_size }} POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install - DEFAULT_PG_VERSION: 14 + DEFAULT_PG_VERSION: ${{ matrix.pg_version }} TEST_OUTPUT: /tmp/test_output BUILD_TYPE: remote SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }} @@ -323,14 +338,14 @@ jobs: prefix: latest - name: Create Neon Project - if: contains(fromJson('["neon-captest-new", "neon-captest-freetier", "neonvm-captest-new", "neonvm-captest-freetier"]'), matrix.platform) + if: contains(fromJson('["neon-captest-new", "neon-captest-freetier", "neonvm-captest-new", "neonvm-captest-freetier", "neonvm-azure-captest-freetier", "neonvm-azure-captest-new"]'), matrix.platform) id: create-neon-project uses: ./.github/actions/neon-project-create with: - region_id: ${{ github.event.inputs.region_id || 'aws-us-east-2' }} + region_id: ${{ matrix.region_id }} postgres_version: ${{ env.DEFAULT_PG_VERSION }} api_key: ${{ secrets.NEON_STAGING_API_KEY }} - compute_units: ${{ (matrix.platform == 'neon-captest-freetier' && '[0.25, 0.25]') || '[1, 1]' }} + compute_units: ${{ (contains(matrix.platform, 'captest-freetier') && '[0.25, 0.25]') || '[1, 1]' }} provisioner: ${{ (contains(matrix.platform, 'neonvm-') && 'k8s-neonvm') || 'k8s-pod' }} - name: Set up Connection String @@ -343,7 +358,7 @@ jobs: neonvm-captest-sharding-reuse) CONNSTR=${{ secrets.BENCHMARK_CAPTEST_SHARDING_CONNSTR }} ;; - neon-captest-new | neon-captest-freetier | neonvm-captest-new | neonvm-captest-freetier) + neon-captest-new | neon-captest-freetier | neonvm-captest-new | neonvm-captest-freetier | neonvm-azure-captest-new | neonvm-azure-captest-freetier) CONNSTR=${{ steps.create-neon-project.outputs.dsn }} ;; rds-aurora) @@ -368,6 +383,7 @@ jobs: run_in_parallel: false save_perf_report: ${{ env.SAVE_PERF_REPORT }} extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_init + pg_version: ${{ env.DEFAULT_PG_VERSION }} env: BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }} VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" @@ -381,6 +397,7 @@ jobs: run_in_parallel: false save_perf_report: ${{ env.SAVE_PERF_REPORT }} extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_simple_update + pg_version: ${{ env.DEFAULT_PG_VERSION }} env: BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }} VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" @@ -394,6 +411,7 @@ jobs: run_in_parallel: false save_perf_report: ${{ env.SAVE_PERF_REPORT }} extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_select_only + pg_version: ${{ env.DEFAULT_PG_VERSION }} env: BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }} VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" @@ -420,6 +438,12 @@ jobs: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} pgbench-pgvector: + strategy: + matrix: + include: + - PLATFORM: "neon-captest-pgvector" + - PLATFORM: "azure-captest-pgvector" + env: TEST_PG_BENCH_DURATIONS_MATRIX: "15m" TEST_PG_BENCH_SCALES_MATRIX: "1" @@ -428,7 +452,7 @@ jobs: TEST_OUTPUT: /tmp/test_output BUILD_TYPE: remote SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }} - PLATFORM: "neon-captest-pgvector" + PLATFORM: ${{ matrix.PLATFORM }} runs-on: [ self-hosted, us-east-2, x64 ] container: @@ -448,7 +472,18 @@ jobs: - name: Set up Connection String id: set-up-connstr run: | - CONNSTR=${{ secrets.BENCHMARK_PGVECTOR_CONNSTR }} + case "${PLATFORM}" in + neon-captest-pgvector) + CONNSTR=${{ secrets.BENCHMARK_PGVECTOR_CONNSTR }} + ;; + azure-captest-pgvector) + CONNSTR=${{ secrets.BENCHMARK_PGVECTOR_CONNSTR_AZURE }} + ;; + *) + echo >&2 "Unknown PLATFORM=${PLATFORM}" + exit 1 + ;; + esac echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT @@ -460,6 +495,7 @@ jobs: run_in_parallel: false save_perf_report: ${{ env.SAVE_PERF_REPORT }} extra_params: -m remote_cluster --timeout 21600 -k test_pgvector_indexing + pg_version: ${{ env.DEFAULT_PG_VERSION }} env: VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" @@ -473,6 +509,7 @@ jobs: run_in_parallel: false save_perf_report: ${{ env.SAVE_PERF_REPORT }} extra_params: -m remote_cluster --timeout 21600 + pg_version: ${{ env.DEFAULT_PG_VERSION }} env: BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }} VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" @@ -487,7 +524,7 @@ jobs: uses: slackapi/slack-github-action@v1 with: channel-id: "C033QLM5P7D" # dev-staging-stream - slack-message: "Periodic perf testing neon-captest-pgvector: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + slack-message: "Periodic perf testing ${PLATFORM}: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} @@ -735,6 +772,7 @@ jobs: run_in_parallel: false save_perf_report: ${{ env.SAVE_PERF_REPORT }} extra_params: -m remote_cluster --timeout 21600 -k test_user_examples + pg_version: ${{ env.DEFAULT_PG_VERSION }} env: VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" From 839a5724a4d28b775fbcab03c9e3b3643e2f0086 Mon Sep 17 00:00:00 2001 From: "Alex Chi Z." Date: Wed, 17 Jul 2024 11:22:38 -0400 Subject: [PATCH 127/194] test(pageserver): more k-merge tests on duplicated keys (#8404) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Existing tenants and some selection of layers might produce duplicated keys. Add tests to ensure the k-merge iterator handles it correctly. We also enforced ordering of the k-merge iterator to put images before deltas. part of https://github.com/neondatabase/neon/issues/8002 --------- Signed-off-by: Alex Chi Z Co-authored-by: Arpad Müller --- .../src/tenant/storage_layer/delta_layer.rs | 16 +- .../tenant/storage_layer/merge_iterator.rs | 163 ++++++++++++++++-- 2 files changed, 163 insertions(+), 16 deletions(-) diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 64412fe4afd1..43941b6e1739 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -1672,6 +1672,7 @@ pub(crate) mod test { use rand::RngCore; use super::*; + use crate::repository::Value; use crate::tenant::harness::TIMELINE_ID; use crate::tenant::vectored_blob_io::StreamingVectoredReadPlanner; use crate::tenant::Tenant; @@ -1681,6 +1682,7 @@ pub(crate) mod test { tenant::{disk_btree::tests::TestDisk, harness::TenantHarness}, DEFAULT_PG_VERSION, }; + use bytes::Bytes; /// Construct an index for a fictional delta layer and and then /// traverse in order to plan vectored reads for a query. Finally, @@ -2249,6 +2251,15 @@ pub(crate) mod test { (k1, l1).cmp(&(k2, l2)) } + pub(crate) fn sort_delta_value( + (k1, l1, v1): &(Key, Lsn, Value), + (k2, l2, v2): &(Key, Lsn, Value), + ) -> std::cmp::Ordering { + let order_1 = if v1.is_image() { 0 } else { 1 }; + let order_2 = if v2.is_image() { 0 } else { 1 }; + (k1, l1, order_1).cmp(&(k2, l2, order_2)) + } + pub(crate) async fn produce_delta_layer( tenant: &Tenant, tline: &Arc, @@ -2257,7 +2268,7 @@ pub(crate) mod test { ) -> anyhow::Result { deltas.sort_by(sort_delta); let (key_start, _, _) = deltas.first().unwrap(); - let (key_max, _, _) = deltas.first().unwrap(); + let (key_max, _, _) = deltas.last().unwrap(); let lsn_min = deltas.iter().map(|(_, lsn, _)| lsn).min().unwrap(); let lsn_max = deltas.iter().map(|(_, lsn, _)| lsn).max().unwrap(); let lsn_end = Lsn(lsn_max.0 + 1); @@ -2302,9 +2313,6 @@ pub(crate) mod test { #[tokio::test] async fn delta_layer_iterator() { - use crate::repository::Value; - use bytes::Bytes; - let harness = TenantHarness::create("delta_layer_iterator").unwrap(); let (tenant, ctx) = harness.load().await; diff --git a/pageserver/src/tenant/storage_layer/merge_iterator.rs b/pageserver/src/tenant/storage_layer/merge_iterator.rs index 68759f758576..0edfd4bd4075 100644 --- a/pageserver/src/tenant/storage_layer/merge_iterator.rs +++ b/pageserver/src/tenant/storage_layer/merge_iterator.rs @@ -96,15 +96,22 @@ impl<'a> std::cmp::PartialOrd for IteratorWrapper<'a> { impl<'a> std::cmp::Ord for IteratorWrapper<'a> { fn cmp(&self, other: &Self) -> std::cmp::Ordering { use std::cmp::Ordering; - let a = self.peek_next_key_lsn(); - let b = other.peek_next_key_lsn(); + let a = self.peek_next_key_lsn_value(); + let b = other.peek_next_key_lsn_value(); match (a, b) { - (Some((k1, l1)), Some((k2, l2))) => { - let loaded_1 = if self.is_loaded() { 1 } else { 0 }; - let loaded_2 = if other.is_loaded() { 1 } else { 0 }; + (Some((k1, l1, v1)), Some((k2, l2, v2))) => { + fn map_value_to_num(val: &Option<&Value>) -> usize { + match val { + None => 0, + Some(Value::Image(_)) => 1, + Some(Value::WalRecord(_)) => 2, + } + } + let order_1 = map_value_to_num(&v1); + let order_2 = map_value_to_num(&v2); // When key_lsn are the same, the unloaded iter will always appear before the loaded one. // And note that we do a reverse at the end of the comparison, so it works with the max heap. - (k1, l1, loaded_1).cmp(&(k2, l2, loaded_2)) + (k1, l1, order_1).cmp(&(k2, l2, order_2)) } (Some(_), None) => Ordering::Less, (None, Some(_)) => Ordering::Greater, @@ -137,13 +144,16 @@ impl<'a> IteratorWrapper<'a> { } } - fn peek_next_key_lsn(&self) -> Option<(&Key, Lsn)> { + fn peek_next_key_lsn_value(&self) -> Option<(&Key, Lsn, Option<&Value>)> { match self { - Self::Loaded { iter } => iter.peek().as_ref().map(|(key, lsn, _)| (key, *lsn)), + Self::Loaded { iter } => iter + .peek() + .as_ref() + .map(|(key, lsn, val)| (key, *lsn, Some(val))), Self::NotLoaded { first_key_lower_bound: (key, lsn), .. - } => Some((key, *lsn)), + } => Some((key, *lsn, None)), } } @@ -191,6 +201,13 @@ impl<'a> IteratorWrapper<'a> { } } +/// A merge iterator over delta/image layer iterators. When duplicated records are +/// found, the iterator will not perform any deduplication, and the caller should handle +/// these situation. By saying duplicated records, there are many possibilities: +/// * Two same delta at the same LSN. +/// * Two same image at the same LSN. +/// * Delta/image at the same LSN where the image has already applied the delta. +/// The iterator will always put the image before the delta. pub struct MergeIterator<'a> { heap: BinaryHeap>, } @@ -245,8 +262,9 @@ mod tests { use crate::{ tenant::{ harness::{TenantHarness, TIMELINE_ID}, - storage_layer::delta_layer::test::{produce_delta_layer, sort_delta}, + storage_layer::delta_layer::test::{produce_delta_layer, sort_delta, sort_delta_value}, }, + walrecord::NeonWalRecord, DEFAULT_PG_VERSION, }; @@ -407,6 +425,127 @@ mod tests { // TODO: test layers are loaded only when needed, reducing num of active iterators in k-merge } - // TODO: image layer merge, delta+image mixed merge - // TODO: is it possible to have duplicated delta at same LSN now? we might need to test that + #[tokio::test] + async fn delta_image_mixed_merge() { + use crate::repository::Value; + use bytes::Bytes; + + let harness = TenantHarness::create("merge_iterator_delta_image_mixed_merge").unwrap(); + let (tenant, ctx) = harness.load().await; + + let tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) + .await + .unwrap(); + + fn get_key(id: u32) -> Key { + let mut key = Key::from_hex("000000000033333333444444445500000000").unwrap(); + key.field6 = id; + key + } + // In this test case, we want to test if the iterator still works correctly with multiple copies + // of a delta+image at the same LSN, for example, the following sequence a@10=+a, a@10=+a, a@10=ab, a@10=ab. + // Duplicated deltas/images are possible for old tenants before the full L0 compaction file name fix. + // An incomplete compaction could produce multiple exactly-the-same delta layers. Force image generation + // could produce overlapping images. Apart from duplicated deltas/images, in the current storage implementation + // one key-lsn could have a delta in the delta layer and one image in the image layer. The iterator should + // correctly process these situations and return everything as-is, and the upper layer of the system + // will handle duplicated LSNs. + let test_deltas1 = vec![ + ( + get_key(0), + Lsn(0x10), + Value::WalRecord(NeonWalRecord::wal_init()), + ), + ( + get_key(0), + Lsn(0x18), + Value::WalRecord(NeonWalRecord::wal_append("a")), + ), + ( + get_key(5), + Lsn(0x10), + Value::WalRecord(NeonWalRecord::wal_init()), + ), + ( + get_key(5), + Lsn(0x18), + Value::WalRecord(NeonWalRecord::wal_append("b")), + ), + ]; + let resident_layer_1 = produce_delta_layer(&tenant, &tline, test_deltas1.clone(), &ctx) + .await + .unwrap(); + let mut test_deltas2 = test_deltas1.clone(); + test_deltas2.push(( + get_key(10), + Lsn(0x20), + Value::Image(Bytes::copy_from_slice(b"test")), + )); + let resident_layer_2 = produce_delta_layer(&tenant, &tline, test_deltas2.clone(), &ctx) + .await + .unwrap(); + let test_deltas3 = vec![ + ( + get_key(0), + Lsn(0x10), + Value::Image(Bytes::copy_from_slice(b"")), + ), + ( + get_key(5), + Lsn(0x18), + Value::Image(Bytes::copy_from_slice(b"b")), + ), + ( + get_key(15), + Lsn(0x20), + Value::Image(Bytes::copy_from_slice(b"test")), + ), + ]; + let resident_layer_3 = produce_delta_layer(&tenant, &tline, test_deltas3.clone(), &ctx) + .await + .unwrap(); + let mut test_deltas4 = test_deltas3.clone(); + test_deltas4.push(( + get_key(20), + Lsn(0x20), + Value::Image(Bytes::copy_from_slice(b"test")), + )); + let resident_layer_4 = produce_delta_layer(&tenant, &tline, test_deltas4.clone(), &ctx) + .await + .unwrap(); + let mut expect = Vec::new(); + expect.extend(test_deltas1); + expect.extend(test_deltas2); + expect.extend(test_deltas3); + expect.extend(test_deltas4); + expect.sort_by(sort_delta_value); + + // Test with different layer order for MergeIterator::create to ensure the order + // is stable. + + let mut merge_iter = MergeIterator::create( + &[ + resident_layer_4.get_as_delta(&ctx).await.unwrap(), + resident_layer_1.get_as_delta(&ctx).await.unwrap(), + resident_layer_3.get_as_delta(&ctx).await.unwrap(), + resident_layer_2.get_as_delta(&ctx).await.unwrap(), + ], + &[], + &ctx, + ); + assert_merge_iter_equal(&mut merge_iter, &expect).await; + + let mut merge_iter = MergeIterator::create( + &[ + resident_layer_1.get_as_delta(&ctx).await.unwrap(), + resident_layer_4.get_as_delta(&ctx).await.unwrap(), + resident_layer_3.get_as_delta(&ctx).await.unwrap(), + resident_layer_2.get_as_delta(&ctx).await.unwrap(), + ], + &[], + &ctx, + ); + assert_merge_iter_equal(&mut merge_iter, &expect).await; + } } From 975f8ac658243640c7d695e2bcc0acad3e72ccdb Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 17 Jul 2024 18:35:27 +0100 Subject: [PATCH 128/194] tests: add test_compaction_l0_memory (#8403) This test reproduces the case of a writer creating a deep stack of L0 layers. It uses realistic layer sizes and writes several gigabytes of data, therefore runs as a performance test although it is validating memory footprint rather than performance per se. It acts a regression test for two recent fixes: - https://github.com/neondatabase/neon/pull/8401 - https://github.com/neondatabase/neon/pull/8391 In future it will demonstrate the larger improvement of using a k-merge iterator for L0 compaction (#8184) This test can be extended to enforce limits on the memory consumption of other housekeeping steps, by restarting the pageserver and then running other things to do the same "how much did RSS increase" measurement. --- test_runner/fixtures/pageserver/http.py | 3 + test_runner/performance/test_compaction.py | 96 ++++++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/test_runner/fixtures/pageserver/http.py b/test_runner/fixtures/pageserver/http.py index f1e3d1a30941..c7cea4ec0476 100644 --- a/test_runner/fixtures/pageserver/http.py +++ b/test_runner/fixtures/pageserver/http.py @@ -117,6 +117,9 @@ def delta_layers(self) -> List[HistoricLayerInfo]: def image_layers(self) -> List[HistoricLayerInfo]: return [x for x in self.historic_layers if x.kind == "Image"] + def delta_l0_layers(self) -> List[HistoricLayerInfo]: + return [x for x in self.historic_layers if x.kind == "Delta" and x.l0] + def historic_by_name(self) -> Set[str]: return set(x.layer_file_name for x in self.historic_layers) diff --git a/test_runner/performance/test_compaction.py b/test_runner/performance/test_compaction.py index 326c4f5c6fad..077b76104ce1 100644 --- a/test_runner/performance/test_compaction.py +++ b/test_runner/performance/test_compaction.py @@ -2,6 +2,7 @@ import pytest from fixtures.compare_fixtures import NeonCompare +from fixtures.log_helper import log from fixtures.neon_fixtures import wait_for_last_flush_lsn @@ -56,3 +57,98 @@ def test_compaction(neon_compare: NeonCompare): pageserver_http.timeline_compact(tenant_id, timeline_id) neon_compare.report_size() + + +def test_compaction_l0_memory(neon_compare: NeonCompare): + """ + Generate a large stack of L0s pending compaction into L1s, and + measure the pageserver's peak RSS while doing so + """ + + env = neon_compare.env + pageserver_http = env.pageserver.http_client() + + tenant_id, timeline_id = env.neon_cli.create_tenant( + conf={ + # Initially disable compaction so that we will build up a stack of L0s + "compaction_period": "0s", + "gc_period": "0s", + } + ) + neon_compare.tenant = tenant_id + neon_compare.timeline = timeline_id + + endpoint = env.endpoints.create_start( + "main", tenant_id=tenant_id, config_lines=["shared_buffers=512MB"] + ) + + # Read tenant effective config and assert on checkpoint_distance and compaction_threshold, + # as we do want to test with defaults (to be same as the field), but this test's workload size makes assumptions about them. + # + # If these assertions fail, it probably means we changed the default. + tenant_conf = pageserver_http.tenant_config(tenant_id) + assert tenant_conf.effective_config["checkpoint_distance"] == 256 * 1024 * 1024 + assert tenant_conf.effective_config["compaction_threshold"] == 10 + + # Aim to write about 20 L0s, so that we will hit the limit on how many + # to compact at once + with closing(endpoint.connect()) as conn: + with conn.cursor() as cur: + for i in range(200): + cur.execute(f"create table tbl{i} (i int, j int);") + cur.execute(f"insert into tbl{i} values (generate_series(1, 1000), 0);") + for j in range(100): + cur.execute(f"update tbl{i} set j = {j};") + + wait_for_last_flush_lsn(env, endpoint, tenant_id, timeline_id) + endpoint.stop() + + # Check we have generated the L0 stack we expected + layers = pageserver_http.layer_map_info(tenant_id, timeline_id) + initial_l0s = len(layers.delta_l0_layers()) + initial_l0s_size = sum(x.layer_file_size for x in layers.delta_l0_layers()) + log.info(f"l0s before compaction {initial_l0s} ({initial_l0s_size})") + + def rss_hwm(): + v = pageserver_http.get_metric_value("libmetrics_maxrss_kb") + assert v is not None + assert v > 0 + return v * 1024 + + before = rss_hwm() + pageserver_http.timeline_compact(tenant_id, timeline_id) + after = rss_hwm() + + log.info(f"RSS across compaction: {before} -> {after} (grew {after - before})") + + layers = pageserver_http.layer_map_info(tenant_id, timeline_id) + final_l0s_size = sum(x.layer_file_size for x in layers.delta_l0_layers()) + log.info(f"l0s after compaction {len(layers.delta_l0_layers())} ({final_l0s_size})") + + assert after > before # If we didn't use some memory the test is probably buggy + compaction_mapped_rss = after - before + + # During L0 compaction, we require as much memory as the physical size of what we compacted, and then some, + # because the key->value mapping in L0s compaction is exhaustive, non-streaming, and does not de-duplicate + # repeated references to the same key. + # + # To be fixed in https://github.com/neondatabase/neon/issues/8184, after which + # this memory estimate can be revised far downwards to something that doesn't scale + # linearly with the layer sizes. + MEMORY_ESTIMATE = (initial_l0s_size - final_l0s_size) * 1.25 + + # If we find that compaction is using more memory, this may indicate a regression + assert compaction_mapped_rss < MEMORY_ESTIMATE + + # If we find that compaction is using <0.5 the expected memory then: + # - maybe we made a big efficiency improvement, in which case update the test + # - maybe something is functionally wrong with the test and it's not driving the system as expected + assert compaction_mapped_rss > MEMORY_ESTIMATE / 2 + + # We should have compacted some but not all of the l0s, based on the limit on how much + # l0 to compact in one go + assert len(layers.delta_l0_layers()) > 0 + assert len(layers.delta_l0_layers()) < initial_l0s + + # The pageserver should have logged when it hit the compaction size limit + env.pageserver.assert_log_contains(".*hit max delta layer size limit.*") From da84a250c69b82362af56360eeae9117d82fb94a Mon Sep 17 00:00:00 2001 From: Yuchen Liang <70461588+yliang412@users.noreply.github.com> Date: Wed, 17 Jul 2024 15:19:40 -0400 Subject: [PATCH 129/194] docs: update storage controller db name in doc (#8411) The db name was renamed to storage_controller from attachment_service. Doc was stale. --- docs/storage_controller.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/storage_controller.md b/docs/storage_controller.md index daf4d0c8b74c..6d2ef929a43d 100644 --- a/docs/storage_controller.md +++ b/docs/storage_controller.md @@ -44,7 +44,7 @@ If you need to modify the database schema, here’s how to create a migration: - Use `diesel migration generate ` to create a new migration - Populate the SQL files in the `migrations/` subdirectory - Use `DATABASE_URL=... diesel migration run` to apply the migration you just wrote: this will update the `[schema.rs](http://schema.rs)` file automatically. - - This requires a running database: the easiest way to do that is to just run `cargo neon init ; cargo neon start`, which will leave a database available at `postgresql://localhost:1235/attachment_service` + - This requires a running database: the easiest way to do that is to just run `cargo neon init ; cargo neon start`, which will leave a database available at `postgresql://localhost:1235/storage_controller` - Commit the migration files and the changes to schema.rs - If you need to iterate, you can rewind migrations with `diesel migration revert -a` and then `diesel migration run` again. - The migrations are build into the storage controller binary, and automatically run at startup after it is deployed, so once you’ve committed a migration no further steps are needed. From 0c236fa465f1f4691f9b814208edc7437f92fa4b Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 17 Jul 2024 21:55:20 +0100 Subject: [PATCH 130/194] pageserver: layer count & size metrics (#8410) ## Problem We lack insight into: - How much of a tenant's physical size is image vs. delta layers - Average sizes of image vs. delta layers - Total layer counts per timeline, indicating size of index_part object As well as general observability love, this is motivated by https://github.com/neondatabase/neon/issues/6738, where we need to define some sensible thresholds for storage amplification, and using total physical size may not work well (if someone does a lot of DROPs then it's legitimate for the physical-synthetic ratio to be huge), but the ratio between image layer size and delta layer size may be a better indicator of whether we're generating unreasonable quantities of image layers. ## Summary of changes - Add pageserver_layer_bytes and pageserver_layer_count metrics, labelled by timeline and `kind` (delta or image) - Add & subtract these with LayerInner's lifetime. I'm intentionally avoiding using a generic metric RAII guard object, to avoid bloating LayerInner: it already has all the information it needs to update metric on new+drop. --- pageserver/src/metrics.rs | 94 ++++++++++++++++++++ pageserver/src/tenant/storage_layer/layer.rs | 21 +++++ test_runner/fixtures/metrics.py | 2 + 3 files changed, 117 insertions(+) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index abad4b44b802..753f5524c55d 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -473,6 +473,31 @@ static PITR_HISTORY_SIZE: Lazy = Lazy::new(|| { .expect("failed to define a metric") }); +#[derive(strum_macros::EnumString, strum_macros::Display, strum_macros::IntoStaticStr)] +#[strum(serialize_all = "kebab_case")] +pub(crate) enum MetricLayerKind { + Delta, + Image, +} + +static TIMELINE_LAYER_SIZE: Lazy = Lazy::new(|| { + register_uint_gauge_vec!( + "pageserver_layer_bytes", + "Sum of layer physical sizes in bytes", + &["tenant_id", "shard_id", "timeline_id", "kind"] + ) + .expect("failed to define a metric") +}); + +static TIMELINE_LAYER_COUNT: Lazy = Lazy::new(|| { + register_uint_gauge_vec!( + "pageserver_layer_count", + "Number of layers that exist", + &["tenant_id", "shard_id", "timeline_id", "kind"] + ) + .expect("failed to define a metric") +}); + static TIMELINE_ARCHIVE_SIZE: Lazy = Lazy::new(|| { register_uint_gauge_vec!( "pageserver_archive_size", @@ -2141,6 +2166,10 @@ pub(crate) struct TimelineMetrics { pub last_record_gauge: IntGauge, pub pitr_history_size: UIntGauge, pub archival_size: UIntGauge, + pub(crate) layer_size_image: UIntGauge, + pub(crate) layer_count_image: UIntGauge, + pub(crate) layer_size_delta: UIntGauge, + pub(crate) layer_count_delta: UIntGauge, pub standby_horizon_gauge: IntGauge, pub resident_physical_size_gauge: UIntGauge, /// copy of LayeredTimeline.current_logical_size @@ -2223,6 +2252,42 @@ impl TimelineMetrics { .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id]) .unwrap(); + let layer_size_image = TIMELINE_LAYER_SIZE + .get_metric_with_label_values(&[ + &tenant_id, + &shard_id, + &timeline_id, + MetricLayerKind::Image.into(), + ]) + .unwrap(); + + let layer_count_image = TIMELINE_LAYER_COUNT + .get_metric_with_label_values(&[ + &tenant_id, + &shard_id, + &timeline_id, + MetricLayerKind::Image.into(), + ]) + .unwrap(); + + let layer_size_delta = TIMELINE_LAYER_SIZE + .get_metric_with_label_values(&[ + &tenant_id, + &shard_id, + &timeline_id, + MetricLayerKind::Delta.into(), + ]) + .unwrap(); + + let layer_count_delta = TIMELINE_LAYER_COUNT + .get_metric_with_label_values(&[ + &tenant_id, + &shard_id, + &timeline_id, + MetricLayerKind::Delta.into(), + ]) + .unwrap(); + let standby_horizon_gauge = STANDBY_HORIZON .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id]) .unwrap(); @@ -2277,6 +2342,10 @@ impl TimelineMetrics { last_record_gauge, pitr_history_size, archival_size, + layer_size_image, + layer_count_image, + layer_size_delta, + layer_count_delta, standby_horizon_gauge, resident_physical_size_gauge, current_logical_size_gauge, @@ -2338,6 +2407,31 @@ impl TimelineMetrics { let _ = TIMELINE_ARCHIVE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]); let _ = PITR_HISTORY_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]); + let _ = TIMELINE_LAYER_SIZE.remove_label_values(&[ + tenant_id, + shard_id, + timeline_id, + MetricLayerKind::Image.into(), + ]); + let _ = TIMELINE_LAYER_COUNT.remove_label_values(&[ + tenant_id, + shard_id, + timeline_id, + MetricLayerKind::Image.into(), + ]); + let _ = TIMELINE_LAYER_SIZE.remove_label_values(&[ + tenant_id, + shard_id, + timeline_id, + MetricLayerKind::Delta.into(), + ]); + let _ = TIMELINE_LAYER_COUNT.remove_label_values(&[ + tenant_id, + shard_id, + timeline_id, + MetricLayerKind::Delta.into(), + ]); + let _ = EVICTIONS.remove_label_values(&[tenant_id, shard_id, timeline_id]); let _ = AUX_FILE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]); let _ = VALID_LSN_LEASE_COUNT.remove_label_values(&[tenant_id, shard_id, timeline_id]); diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index 4500bc94dd66..dbf6c60aaee0 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -693,6 +693,18 @@ impl Drop for LayerInner { // and we could be delaying shutdown for nothing. } + if let Some(timeline) = self.timeline.upgrade() { + // Only need to decrement metrics if the timeline still exists: otherwise + // it will have already de-registered these metrics via TimelineMetrics::shutdown + if self.desc.is_delta() { + timeline.metrics.layer_count_delta.dec(); + timeline.metrics.layer_size_delta.sub(self.desc.file_size); + } else { + timeline.metrics.layer_count_image.dec(); + timeline.metrics.layer_size_image.sub(self.desc.file_size); + } + } + if !*self.wanted_deleted.get_mut() { return; } @@ -791,6 +803,15 @@ impl LayerInner { (heavier_once_cell::OnceCell::default(), 0, Status::Evicted) }; + // This object acts as a RAII guard on these metrics: increment on construction + if desc.is_delta() { + timeline.metrics.layer_count_delta.inc(); + timeline.metrics.layer_size_delta.add(desc.file_size); + } else { + timeline.metrics.layer_count_image.inc(); + timeline.metrics.layer_size_image.add(desc.file_size); + } + LayerInner { conf, debug_str: { diff --git a/test_runner/fixtures/metrics.py b/test_runner/fixtures/metrics.py index c019cbbc7790..4836d42db5be 100644 --- a/test_runner/fixtures/metrics.py +++ b/test_runner/fixtures/metrics.py @@ -146,6 +146,8 @@ def histogram(prefix_without_trailing_underscore: str) -> List[str]: "pageserver_smgr_query_seconds_sum", "pageserver_archive_size", "pageserver_pitr_history_size", + "pageserver_layer_bytes", + "pageserver_layer_count", "pageserver_storage_operations_seconds_count_total", "pageserver_storage_operations_seconds_sum_total", "pageserver_evictions_total", From e250b9e063b27db724032e2c0f0971cc67bb7130 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 18 Jul 2024 00:03:02 +0300 Subject: [PATCH 131/194] test: allow requests to any pageserver get cancelled (#8413) Fix flakyness on `test_sharded_timeline_detach_ancestor` which does not reproduce on a fast enough runner by allowing cancelled request before completing on all pageservers. It was only allowed on half of the pageservers. Failure evidence: https://neon-github-public-dev.s3.amazonaws.com/reports/pr-8352/9972357040/index.html#suites/a1c2be32556270764423c495fad75d47/7cca3e3d94fe12f2 --- .../regress/test_timeline_detach_ancestor.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/test_runner/regress/test_timeline_detach_ancestor.py b/test_runner/regress/test_timeline_detach_ancestor.py index d75ab4c0604f..38f8dfa88553 100644 --- a/test_runner/regress/test_timeline_detach_ancestor.py +++ b/test_runner/regress/test_timeline_detach_ancestor.py @@ -702,20 +702,16 @@ def test_sharded_timeline_detach_ancestor(neon_env_builder: NeonEnvBuilder): # make another of the nodes get stuck, then restart stuck = pageservers[int(shards[0]["node_id"])] - stuck.allowed_errors.append(".*: request was dropped before completing") - env.storage_controller.allowed_errors.append(".*: request was dropped before completing") + log.info(f"stuck pageserver is id={stuck.id}") stuck_http = stuck.http_client() stuck_http.configure_failpoints( ("timeline-detach-ancestor::before_starting_after_locking_pausable", "pause") ) restarted = pageservers[int(shards[1]["node_id"])] - restarted.allowed_errors.extend( - [ - ".*: request was dropped before completing", - ".*: Cancelled request finished with an error: ShuttingDown", - ] - ) + log.info(f"restarted pageserver is id={restarted.id}") + # this might be hit; see `restart_restarted` + restarted.allowed_errors.append(".*: Cancelled request finished with an error: ShuttingDown") assert restarted.id != stuck.id restarted_http = restarted.http_client() restarted_http.configure_failpoints( @@ -724,6 +720,14 @@ def test_sharded_timeline_detach_ancestor(neon_env_builder: NeonEnvBuilder): ] ) + for info in shards: + pageserver = pageservers[int(info["node_id"])] + # the first request can cause these, but does not repeatedly + pageserver.allowed_errors.append(".*: request was dropped before completing") + + # first request again + env.storage_controller.allowed_errors.append(".*: request was dropped before completing") + target = env.storage_controller.pageserver_api() with pytest.raises(ReadTimeout): From 1303d477789a4f131cf04c974b6d0846be88a0f5 Mon Sep 17 00:00:00 2001 From: dotdister Date: Thu, 18 Jul 2024 17:33:46 +0900 Subject: [PATCH 132/194] Fix comment in Control Plane (#8406) ## Problem There are something wrong in the comment of `control_plane/src/broker.rs` and `control_plane/src/pageserver.rs` ## Summary of changes Fixed the comment about component name and their data path in `control_plane/src/broker.rs` and `control_plane/src/pageserver.rs`. --- control_plane/src/broker.rs | 4 ++-- control_plane/src/pageserver.rs | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/control_plane/src/broker.rs b/control_plane/src/broker.rs index c3cfc140da2f..c8ac5d8981a5 100644 --- a/control_plane/src/broker.rs +++ b/control_plane/src/broker.rs @@ -1,9 +1,9 @@ //! Code to manage the storage broker //! -//! In the local test environment, the data for each safekeeper is stored in +//! In the local test environment, the storage broker stores its data directly in //! //! ```text -//! .neon/safekeepers/ +//! .neon //! ``` use std::time::Duration; diff --git a/control_plane/src/pageserver.rs b/control_plane/src/pageserver.rs index 5f2373e95a68..e3d1d0e11004 100644 --- a/control_plane/src/pageserver.rs +++ b/control_plane/src/pageserver.rs @@ -1,8 +1,10 @@ //! Code to manage pageservers //! -//! In the local test environment, the pageserver stores its data directly in +//! In the local test environment, the data for each pageserver is stored in //! -//! .neon/ +//! ```text +//! .neon/pageserver_ +//! ``` //! use std::collections::HashMap; From a2d170b6d06a1ccc8eba3aadfaf7bbf16007978c Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Thu, 18 Jul 2024 10:56:07 +0200 Subject: [PATCH 133/194] NeonEnv.from_repo_dir: use storage_controller_db instead of `attachments.json` (#8382) When `NeonEnv.from_repo_dir` was introduced, storage controller stored its state exclusively `attachments.json`. Since then, it has moved to using Postgres, which stores its state in `storage_controller_db`. But `NeonEnv.from_repo_dir` wasn't adjusted to do this. This PR rectifies the situation. Context for this is failures in `test_pageserver_characterize_throughput_with_n_tenants` CF: https://neondb.slack.com/archives/C033RQ5SPDH/p1721035799502239?thread_ts=1720901332.293769&cid=C033RQ5SPDH Notably, `from_repo_dir` is also used by the backwards- and forwards-compatibility. Thus, the changes in this PR affect those tests as well. However, it turns out that the compatibility snapshot already contains the `storage_controller_db`. Thus, it should just work and in fact we can remove hacks like `fixup_storage_controller`. Follow-ups created as part of this work: * https://github.com/neondatabase/neon/issues/8399 * https://github.com/neondatabase/neon/issues/8400 --- Cargo.lock | 27 +++++ Cargo.toml | 1 + control_plane/Cargo.toml | 1 + control_plane/src/storage_controller.rs | 87 +++++++++++---- storage_controller/src/main.rs | 19 +--- storage_controller/src/persistence.rs | 100 ++---------------- test_runner/fixtures/neon_fixtures.py | 27 ++++- ...er_max_throughput_getpage_at_latest_lsn.py | 8 -- test_runner/regress/test_compatibility.py | 25 ----- 9 files changed, 133 insertions(+), 162 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88973647017a..d08da0babd36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1368,6 +1368,7 @@ dependencies = [ "tracing", "url", "utils", + "whoami", "workspace_hack", ] @@ -4603,6 +4604,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "regex" version = "1.10.2" @@ -6972,6 +6982,12 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.92" @@ -7124,6 +7140,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "whoami" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" +dependencies = [ + "redox_syscall 0.4.1", + "wasite", + "web-sys", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 4f42203683d1..b9b4bafb4f69 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -191,6 +191,7 @@ uuid = { version = "1.6.1", features = ["v4", "v7", "serde"] } walkdir = "2.3.2" rustls-native-certs = "0.7" x509-parser = "0.15" +whoami = "1.5.1" ## TODO replace this with tracing env_logger = "0.10" diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index e62f3b8a4780..487ac8f047ed 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -40,6 +40,7 @@ safekeeper_api.workspace = true postgres_connection.workspace = true storage_broker.workspace = true utils.workspace = true +whoami.workspace = true compute_api.workspace = true workspace_hack.workspace = true diff --git a/control_plane/src/storage_controller.rs b/control_plane/src/storage_controller.rs index 47103a2e0ac5..d7aedd711ae0 100644 --- a/control_plane/src/storage_controller.rs +++ b/control_plane/src/storage_controller.rs @@ -29,7 +29,6 @@ use utils::{ pub struct StorageController { env: LocalEnv, listen: String, - path: Utf8PathBuf, private_key: Option>, public_key: Option, postgres_port: u16, @@ -41,6 +40,8 @@ const COMMAND: &str = "storage_controller"; const STORAGE_CONTROLLER_POSTGRES_VERSION: u32 = 16; +const DB_NAME: &str = "storage_controller"; + #[derive(Serialize, Deserialize)] pub struct AttachHookRequest { pub tenant_shard_id: TenantShardId, @@ -65,10 +66,6 @@ pub struct InspectResponse { impl StorageController { pub fn from_env(env: &LocalEnv) -> Self { - let path = Utf8PathBuf::from_path_buf(env.base_data_dir.clone()) - .unwrap() - .join("attachments.json"); - // Makes no sense to construct this if pageservers aren't going to use it: assume // pageservers have control plane API set let listen_url = env.control_plane_api.clone().unwrap(); @@ -128,7 +125,6 @@ impl StorageController { Self { env: env.clone(), - path, listen, private_key, public_key, @@ -203,7 +199,6 @@ impl StorageController { /// /// Returns the database url pub async fn setup_database(&self) -> anyhow::Result { - const DB_NAME: &str = "storage_controller"; let database_url = format!("postgresql://localhost:{}/{DB_NAME}", self.postgres_port); let pg_bin_dir = self.get_pg_bin_dir().await?; @@ -232,6 +227,30 @@ impl StorageController { Ok(database_url) } + pub async fn connect_to_database( + &self, + ) -> anyhow::Result<( + tokio_postgres::Client, + tokio_postgres::Connection, + )> { + tokio_postgres::Config::new() + .host("localhost") + .port(self.postgres_port) + // The user is the ambient operating system user name. + // That is an impurity which we want to fix in => TODO https://github.com/neondatabase/neon/issues/8400 + // + // Until we get there, use the ambient operating system user name. + // Recent tokio-postgres versions default to this if the user isn't specified. + // But tokio-postgres fork doesn't have this upstream commit: + // https://github.com/sfackler/rust-postgres/commit/cb609be758f3fb5af537f04b584a2ee0cebd5e79 + // => we should rebase our fork => TODO https://github.com/neondatabase/neon/issues/8399 + .user(&whoami::username()) + .dbname(DB_NAME) + .connect(tokio_postgres::NoTls) + .await + .map_err(anyhow::Error::new) + } + pub async fn start(&self, retry_timeout: &Duration) -> anyhow::Result<()> { // Start a vanilla Postgres process used by the storage controller for persistence. let pg_data_path = Utf8PathBuf::from_path_buf(self.env.base_data_dir.clone()) @@ -256,18 +275,21 @@ impl StorageController { if !status.success() { anyhow::bail!("initdb failed with status {status}"); } - - // Write a minimal config file: - // - Specify the port, since this is chosen dynamically - // - Switch off fsync, since we're running on lightweight test environments and when e.g. scale testing - // the storage controller we don't want a slow local disk to interfere with that. - tokio::fs::write( - &pg_data_path.join("postgresql.conf"), - format!("port = {}\nfsync=off\n", self.postgres_port), - ) - .await?; }; + // Write a minimal config file: + // - Specify the port, since this is chosen dynamically + // - Switch off fsync, since we're running on lightweight test environments and when e.g. scale testing + // the storage controller we don't want a slow local disk to interfere with that. + // + // NB: it's important that we rewrite this file on each start command so we propagate changes + // from `LocalEnv`'s config file (`.neon/config`). + tokio::fs::write( + &pg_data_path.join("postgresql.conf"), + format!("port = {}\nfsync=off\n", self.postgres_port), + ) + .await?; + println!("Starting storage controller database..."); let db_start_args = [ "-w", @@ -296,11 +318,38 @@ impl StorageController { // Run migrations on every startup, in case something changed. let database_url = self.setup_database().await?; + // We support running a startup SQL script to fiddle with the database before we launch storcon. + // This is used by the test suite. + let startup_script_path = self + .env + .base_data_dir + .join("storage_controller_db.startup.sql"); + let startup_script = match tokio::fs::read_to_string(&startup_script_path).await { + Ok(script) => { + tokio::fs::remove_file(startup_script_path).await?; + script + } + Err(e) => { + if e.kind() == std::io::ErrorKind::NotFound { + // always run some startup script so that this code path doesn't bit rot + "BEGIN; COMMIT;".to_string() + } else { + anyhow::bail!("Failed to read startup script: {e}") + } + } + }; + let (mut client, conn) = self.connect_to_database().await?; + let conn = tokio::spawn(conn); + let tx = client.build_transaction(); + let tx = tx.start().await?; + tx.batch_execute(&startup_script).await?; + tx.commit().await?; + drop(client); + conn.await??; + let mut args = vec![ "-l", &self.listen, - "-p", - self.path.as_ref(), "--dev", "--database-url", &database_url, diff --git a/storage_controller/src/main.rs b/storage_controller/src/main.rs index f1eb0b30fc38..4bf6b528f49e 100644 --- a/storage_controller/src/main.rs +++ b/storage_controller/src/main.rs @@ -1,5 +1,4 @@ use anyhow::{anyhow, Context}; -use camino::Utf8PathBuf; use clap::Parser; use diesel::Connection; use metrics::launch_timestamp::LaunchTimestamp; @@ -51,10 +50,6 @@ struct Cli { #[arg(long)] compute_hook_url: Option, - /// Path to the .json file to store state (will be created if it doesn't exist) - #[arg(short, long)] - path: Option, - /// URL to connect to postgres, like postgresql://localhost:1234/storage_controller #[arg(long)] database_url: Option, @@ -206,11 +201,10 @@ async fn async_main() -> anyhow::Result<()> { let args = Cli::parse(); tracing::info!( - "version: {}, launch_timestamp: {}, build_tag {}, state at {}, listening on {}", + "version: {}, launch_timestamp: {}, build_tag {}, listening on {}", GIT_VERSION, launch_ts.to_string(), BUILD_TAG, - args.path.as_ref().unwrap_or(&Utf8PathBuf::from("")), args.listen ); @@ -277,8 +271,7 @@ async fn async_main() -> anyhow::Result<()> { .await .context("Running database migrations")?; - let json_path = args.path; - let persistence = Arc::new(Persistence::new(secrets.database_url, json_path.clone())); + let persistence = Arc::new(Persistence::new(secrets.database_url)); let service = Service::spawn(config, persistence.clone()).await?; @@ -316,14 +309,6 @@ async fn async_main() -> anyhow::Result<()> { } tracing::info!("Terminating on signal"); - if json_path.is_some() { - // Write out a JSON dump on shutdown: this is used in compat tests to avoid passing - // full postgres dumps around. - if let Err(e) = persistence.write_tenants_json().await { - tracing::error!("Failed to write JSON on shutdown: {e}") - } - } - // Stop HTTP server first, so that we don't have to service requests // while shutting down Service server_shutdown.cancel(); diff --git a/storage_controller/src/persistence.rs b/storage_controller/src/persistence.rs index 9f7b2f775e97..d8f31e86e589 100644 --- a/storage_controller/src/persistence.rs +++ b/storage_controller/src/persistence.rs @@ -5,8 +5,6 @@ use std::time::Duration; use std::time::Instant; use self::split_state::SplitState; -use camino::Utf8Path; -use camino::Utf8PathBuf; use diesel::pg::PgConnection; use diesel::prelude::*; use diesel::Connection; @@ -55,11 +53,6 @@ use crate::node::Node; /// we can UPDATE a node's scheduling mode reasonably quickly to mark a bad node offline. pub struct Persistence { connection_pool: diesel::r2d2::Pool>, - - // In test environments, we support loading+saving a JSON file. This is temporary, for the benefit of - // test_compatibility.py, so that we don't have to commit to making the database contents fully backward/forward - // compatible just yet. - json_path: Option, } /// Legacy format, for use in JSON compat objects in test environment @@ -124,7 +117,7 @@ impl Persistence { const IDLE_CONNECTION_TIMEOUT: Duration = Duration::from_secs(10); const MAX_CONNECTION_LIFETIME: Duration = Duration::from_secs(60); - pub fn new(database_url: String, json_path: Option) -> Self { + pub fn new(database_url: String) -> Self { let manager = diesel::r2d2::ConnectionManager::::new(database_url); // We will use a connection pool: this is primarily to _limit_ our connection count, rather than to optimize time @@ -139,10 +132,7 @@ impl Persistence { .build(manager) .expect("Could not build connection pool"); - Self { - connection_pool, - json_path, - } + Self { connection_pool } } /// A helper for use during startup, where we would like to tolerate concurrent restarts of the @@ -302,85 +292,13 @@ impl Persistence { /// At startup, load the high level state for shards, such as their config + policy. This will /// be enriched at runtime with state discovered on pageservers. pub(crate) async fn list_tenant_shards(&self) -> DatabaseResult> { - let loaded = self - .with_measured_conn( - DatabaseOperation::ListTenantShards, - move |conn| -> DatabaseResult<_> { - Ok(crate::schema::tenant_shards::table.load::(conn)?) - }, - ) - .await?; - - if loaded.is_empty() { - if let Some(path) = &self.json_path { - if tokio::fs::try_exists(path) - .await - .map_err(|e| DatabaseError::Logical(format!("Error stat'ing JSON file: {e}")))? - { - tracing::info!("Importing from legacy JSON format at {path}"); - return self.list_tenant_shards_json(path).await; - } - } - } - Ok(loaded) - } - - /// Shim for automated compatibility tests: load tenants from a JSON file instead of database - pub(crate) async fn list_tenant_shards_json( - &self, - path: &Utf8Path, - ) -> DatabaseResult> { - let bytes = tokio::fs::read(path) - .await - .map_err(|e| DatabaseError::Logical(format!("Failed to load JSON: {e}")))?; - - let mut decoded = serde_json::from_slice::(&bytes) - .map_err(|e| DatabaseError::Logical(format!("Deserialization error: {e}")))?; - for shard in decoded.tenants.values_mut() { - if shard.placement_policy == "\"Single\"" { - // Backward compat for test data after PR https://github.com/neondatabase/neon/pull/7165 - shard.placement_policy = "{\"Attached\":0}".to_string(); - } - - if shard.scheduling_policy.is_empty() { - shard.scheduling_policy = - serde_json::to_string(&ShardSchedulingPolicy::default()).unwrap(); - } - } - - let tenants: Vec = decoded.tenants.into_values().collect(); - - // Synchronize database with what is in the JSON file - self.insert_tenant_shards(tenants.clone()).await?; - - Ok(tenants) - } - - /// For use in testing environments, where we dump out JSON on shutdown. - pub async fn write_tenants_json(&self) -> anyhow::Result<()> { - let Some(path) = &self.json_path else { - anyhow::bail!("Cannot write JSON if path isn't set (test environment bug)"); - }; - tracing::info!("Writing state to {path}..."); - let tenants = self.list_tenant_shards().await?; - let mut tenants_map = HashMap::new(); - for tsp in tenants { - let tenant_shard_id = TenantShardId { - tenant_id: TenantId::from_str(tsp.tenant_id.as_str())?, - shard_number: ShardNumber(tsp.shard_number as u8), - shard_count: ShardCount::new(tsp.shard_count as u8), - }; - - tenants_map.insert(tenant_shard_id, tsp); - } - let json = serde_json::to_string(&JsonPersistence { - tenants: tenants_map, - })?; - - tokio::fs::write(path, &json).await?; - tracing::info!("Wrote {} bytes to {path}...", json.len()); - - Ok(()) + self.with_measured_conn( + DatabaseOperation::ListTenantShards, + move |conn| -> DatabaseResult<_> { + Ok(crate::schema::tenant_shards::table.load::(conn)?) + }, + ) + .await } /// Tenants must be persisted before we schedule them for the first time. This enables us diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 2765ff916e63..fcfd4ea676b9 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -31,6 +31,7 @@ import httpx import jwt import psycopg2 +import psycopg2.sql import pytest import requests import toml @@ -727,8 +728,30 @@ def from_repo_dir( self.repo_dir / "local_fs_remote_storage", ) - if (attachments_json := Path(repo_dir / "attachments.json")).exists(): - shutil.copyfile(attachments_json, self.repo_dir / attachments_json.name) + # restore storage controller (the db is small, don't bother with overlayfs) + storcon_db_from_dir = repo_dir / "storage_controller_db" + storcon_db_to_dir = self.repo_dir / "storage_controller_db" + log.info(f"Copying storage_controller_db from {storcon_db_from_dir} to {storcon_db_to_dir}") + assert storcon_db_from_dir.is_dir() + assert not storcon_db_to_dir.exists() + + def ignore_postgres_log(path: str, _names): + if Path(path) == storcon_db_from_dir: + return {"postgres.log"} + return set() + + shutil.copytree(storcon_db_from_dir, storcon_db_to_dir, ignore=ignore_postgres_log) + assert not (storcon_db_to_dir / "postgres.log").exists() + # NB: neon_local rewrites postgresql.conf on each start based on neon_local config. No need to patch it. + # However, in this new NeonEnv, the pageservers listen on different ports, and the storage controller + # will currently reject re-attach requests from them because the NodeMetadata isn't identical. + # So, from_repo_dir patches up the the storcon database. + patch_script_path = self.repo_dir / "storage_controller_db.startup.sql" + assert not patch_script_path.exists() + patch_script = "" + for ps in self.env.pageservers: + patch_script += f"UPDATE nodes SET listen_http_port={ps.service_port.http}, listen_pg_port={ps.service_port.pg} WHERE node_id = '{ps.id}';" + patch_script_path.write_text(patch_script) # Update the config with info about tenants and timelines with (self.repo_dir / "config").open("r") as f: diff --git a/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py b/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py index 60861cf939b8..949813c984f9 100644 --- a/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py +++ b/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py @@ -255,11 +255,3 @@ def run_pagebench_benchmark( unit="ms", report=MetricReport.LOWER_IS_BETTER, ) - - env.storage_controller.allowed_errors.append( - # The test setup swaps NeonEnv instances, hence different - # pg instances are used for the storage controller db. This means - # the storage controller doesn't know about the nodes mentioned - # in attachments.json at start-up. - ".* Scheduler missing node 1", - ) diff --git a/test_runner/regress/test_compatibility.py b/test_runner/regress/test_compatibility.py index 1e5e320e0eff..65649e0c0a84 100644 --- a/test_runner/regress/test_compatibility.py +++ b/test_runner/regress/test_compatibility.py @@ -93,29 +93,6 @@ ) -def fixup_storage_controller(env: NeonEnv): - """ - After importing a repo_dir, we need to massage the storage controller's state a bit: it will have - initially started up with no nodes, but some tenants, and thereby those tenants won't be scheduled - anywhere. - - After NeonEnv.start() is done (i.e. nodes are started + registered), call this function to get - the storage controller into a good state. - - This function should go away once compat tests carry the controller database in their snapshots, so - that the controller properly remembers nodes between creating + restoring the snapshot. - """ - env.storage_controller.allowed_errors.extend( - [ - ".*Tenant shard .+ references non-existent node.*", - ".*Failed to schedule tenant .+ at startup.*", - ] - ) - env.storage_controller.stop() - env.storage_controller.start() - env.storage_controller.reconcile_until_idle() - - @pytest.mark.xdist_group("compatibility") @pytest.mark.order(before="test_forward_compatibility") def test_create_snapshot( @@ -198,7 +175,6 @@ def test_backward_compatibility( neon_env_builder.num_safekeepers = 3 env = neon_env_builder.from_repo_dir(compatibility_snapshot_dir / "repo") neon_env_builder.start() - fixup_storage_controller(env) check_neon_works( env, @@ -287,7 +263,6 @@ def test_forward_compatibility( assert not env.pageserver.log_contains("git-env:" + prev_pageserver_version) neon_env_builder.start() - fixup_storage_controller(env) # ensure the specified pageserver is running assert env.pageserver.log_contains("git-env:" + prev_pageserver_version) From 7672e49ab530eed265b39bf62c3d44e7750f8303 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 18 Jul 2024 10:14:56 +0100 Subject: [PATCH 134/194] tests: fix metrics check in test_s3_eviction (#8419) ## Problem This test would occasionally fail its metric check. This could happen in the rare case that the nodes had all been restarted before their most recent eviction. The metric check was added in https://github.com/neondatabase/neon/pull/8348 ## Summary of changes - Check metrics before each restart, accumulate into a bool that we assert on at the end of the test --- test_runner/regress/test_wal_acceptor.py | 43 +++++++++++++----------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/test_runner/regress/test_wal_acceptor.py b/test_runner/regress/test_wal_acceptor.py index 2e906e616051..f02f19c588dd 100644 --- a/test_runner/regress/test_wal_acceptor.py +++ b/test_runner/regress/test_wal_acceptor.py @@ -2242,6 +2242,8 @@ def test_s3_eviction( check_values = [0] * n_timelines + event_metrics_seen = False + n_iters = 20 for _ in range(n_iters): if log.isEnabledFor(logging.DEBUG): @@ -2266,6 +2268,27 @@ def test_s3_eviction( # update remote_consistent_lsn on pageserver ps_client.timeline_checkpoint(env.initial_tenant, timelines[i], wait_until_uploaded=True) + # Do metrics check before restarts, since these will reset to zero across a restart + event_metrics_seen |= any( + sk.http_client().get_metric_value( + "safekeeper_eviction_events_started_total", {"kind": "evict"} + ) + or 0 > 0 + and sk.http_client().get_metric_value( + "safekeeper_eviction_events_completed_total", {"kind": "evict"} + ) + or 0 > 0 + and sk.http_client().get_metric_value( + "safekeeper_eviction_events_started_total", {"kind": "restore"} + ) + or 0 > 0 + and sk.http_client().get_metric_value( + "safekeeper_eviction_events_completed_total", {"kind": "restore"} + ) + or 0 > 0 + for sk in env.safekeepers + ) + # restarting random safekeepers for sk in env.safekeepers: if random.random() < restart_chance: @@ -2280,22 +2303,4 @@ def test_s3_eviction( for sk in env.safekeepers ) - assert any( - sk.http_client().get_metric_value( - "safekeeper_eviction_events_started_total", {"kind": "evict"} - ) - or 0 > 0 - and sk.http_client().get_metric_value( - "safekeeper_eviction_events_completed_total", {"kind": "evict"} - ) - or 0 > 0 - and sk.http_client().get_metric_value( - "safekeeper_eviction_events_started_total", {"kind": "restore"} - ) - or 0 > 0 - and sk.http_client().get_metric_value( - "safekeeper_eviction_events_completed_total", {"kind": "restore"} - ) - or 0 > 0 - for sk in env.safekeepers - ) + assert event_metrics_seen From 9ded2556dfe104c76793e04e7b3fde44b83714d3 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 18 Jul 2024 10:23:17 +0100 Subject: [PATCH 135/194] tests: increase test_pg_regress and test_isolation timeouts (#8418) ## Problem These tests time out ~1 in 50 runs when in debug mode. There is no indication of a real issue: they're just wrappers that have large numbers of individual tests contained within on pytest case. ## Summary of changes - Bump pg_regress timeout from 600 to 900s - Bump test_isolation timeout from 300s (default) to 600s In future it would be nice to break out these tests to run individual cases (or batches thereof) as separate tests, rather than this monolith. --- test_runner/regress/test_pg_regress.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_runner/regress/test_pg_regress.py b/test_runner/regress/test_pg_regress.py index 54b493ec705d..d5b5ac3f7570 100644 --- a/test_runner/regress/test_pg_regress.py +++ b/test_runner/regress/test_pg_regress.py @@ -117,7 +117,7 @@ def post_checks(env: NeonEnv, test_output_dir: Path, db_name: str, endpoint: End # Run the main PostgreSQL regression tests, in src/test/regress. # -@pytest.mark.timeout(600) +@pytest.mark.timeout(900) # Contains many sub-tests, is slow in debug builds @pytest.mark.parametrize("shard_count", [None, 4]) def test_pg_regress( neon_env_builder: NeonEnvBuilder, @@ -186,6 +186,7 @@ def test_pg_regress( # Run the PostgreSQL "isolation" tests, in src/test/isolation. # +@pytest.mark.timeout(600) # Contains many sub-tests, is slow in debug builds @pytest.mark.parametrize("shard_count", [None, 4]) def test_isolation( neon_env_builder: NeonEnvBuilder, From b46175532678fc650bd32a2bbd281a3813e4773e Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 18 Jul 2024 12:59:14 +0100 Subject: [PATCH 136/194] tests: turn on safekeeper eviction by default (#8352) ## Problem Ahead of enabling eviction in the field, where it will become the normal/default mode, let's enable it by default throughout our tests in case any issues become visible there. ## Summary of changes - Make default `extra_opts` for safekeepers enable offload & deletion - Set low timeouts in `extra_opts` so that tests running for tens of seconds have a chance to hit some of these background operations. --- test_runner/fixtures/neon_fixtures.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index fcfd4ea676b9..567ca532f97a 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -4077,6 +4077,22 @@ def __init__( self.id = id self.running = running self.logfile = Path(self.data_dir) / f"safekeeper-{id}.log" + + if extra_opts is None: + # Testing defaults: enable everything, and set short timeouts so that background + # work will happen during short tests. + # **Note**: Any test that explicitly sets extra_opts will not get these defaults. + extra_opts = [ + "--enable-offload", + "--delete-offloaded-wal", + "--partial-backup-timeout", + "10s", + "--control-file-save-interval", + "1s", + "--eviction-min-resident", + "10s", + ] + self.extra_opts = extra_opts def start( From d263b1804e70d3adf482e740fb9ed20e3fbcbe09 Mon Sep 17 00:00:00 2001 From: Arthur Petukhovsky Date: Thu, 18 Jul 2024 13:46:00 +0100 Subject: [PATCH 137/194] Fix partial upload bug with invalid remote state (#8383) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We have an issue that some partial uploaded segments can be actually missing in remote storage. I found this issue when was looking at the logs in staging, and it can be triggered by failed uploads: 1. Code tries to upload `SEG_TERM_LSN_LSN_sk5.partial`, but receives error from S3 2. The failed attempt is saved to `segments` vec 3. After some time, the code tries to upload `SEG_TERM_LSN_LSN_sk5.partial` again 4. This time the upload is successful and code calls `gc()` to delete previous uploads 5. Since new object and old object share the same name, uploaded data gets deleted from remote storage This commit fixes the issue by patching `gc()` not to delete objects with the same name as currently uploaded. --------- Co-authored-by: Arpad Müller --- safekeeper/src/timeline_eviction.rs | 5 +---- safekeeper/src/wal_backup_partial.rs | 12 ++++++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/safekeeper/src/timeline_eviction.rs b/safekeeper/src/timeline_eviction.rs index 0b8d58ee8a52..7947d83eb4bf 100644 --- a/safekeeper/src/timeline_eviction.rs +++ b/safekeeper/src/timeline_eviction.rs @@ -199,10 +199,7 @@ async fn redownload_partial_segment( file.flush().await?; let final_path = local_segment_path(mgr, partial); - info!( - "downloaded {} bytes, renaming to {}", - final_path, final_path, - ); + info!("downloaded {actual_len} bytes, renaming to {final_path}"); if let Err(e) = durable_rename(&tmp_file, &final_path, !mgr.conf.no_sync).await { // Probably rename succeeded, but fsync of it failed. Remove // the file then to avoid using it. diff --git a/safekeeper/src/wal_backup_partial.rs b/safekeeper/src/wal_backup_partial.rs index 825851c97c9a..b1efa9749f19 100644 --- a/safekeeper/src/wal_backup_partial.rs +++ b/safekeeper/src/wal_backup_partial.rs @@ -289,6 +289,18 @@ impl PartialBackup { }) .collect(); + if new_segments.len() == 1 { + // we have an uploaded segment, it must not be deleted from remote storage + segments_to_delete.retain(|name| name != &new_segments[0].name); + } else { + // there should always be zero or one uploaded segment + assert!( + new_segments.is_empty(), + "too many uploaded segments: {:?}", + new_segments + ); + } + info!("deleting objects: {:?}", segments_to_delete); let mut objects_to_delete = vec![]; for seg in segments_to_delete.iter() { From a4434cf1c0b42133de1196c8adc5468637bbb8eb Mon Sep 17 00:00:00 2001 From: "Alex Chi Z." Date: Thu, 18 Jul 2024 12:16:44 -0400 Subject: [PATCH 138/194] pageserver: integrate k-merge with bottom-most compaction (#8415) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the k-merge iterator in the compaction process to reduce memory footprint. part of https://github.com/neondatabase/neon/issues/8002 ## Summary of changes * refactor the bottom-most compaction code to use k-merge iterator * add Send bound on some structs as it is used across the await points --------- Signed-off-by: Alex Chi Z Co-authored-by: Arpad Müller --- pageserver/src/tenant.rs | 2 +- pageserver/src/tenant/disk_btree.rs | 4 +- pageserver/src/tenant/storage_layer.rs | 2 - .../src/tenant/storage_layer/delta_layer.rs | 21 +++--- .../src/tenant/storage_layer/image_layer.rs | 23 +++--- pageserver/src/tenant/storage_layer/layer.rs | 5 +- .../tenant/storage_layer/merge_iterator.rs | 4 ++ pageserver/src/tenant/timeline/compaction.rs | 70 ++++++++----------- pageserver/src/tenant/vectored_blob_io.rs | 2 - 9 files changed, 62 insertions(+), 71 deletions(-) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index dc6f42eaebaf..637051413f16 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -6810,7 +6810,7 @@ mod tests { vec![ // Image layer at GC horizon PersistentLayerKey { - key_range: Key::MIN..get_key(10), + key_range: Key::MIN..Key::MAX, lsn_range: Lsn(0x30)..Lsn(0x31), is_delta: false }, diff --git a/pageserver/src/tenant/disk_btree.rs b/pageserver/src/tenant/disk_btree.rs index 251d2ab4aded..1583a3826af5 100644 --- a/pageserver/src/tenant/disk_btree.rs +++ b/pageserver/src/tenant/disk_btree.rs @@ -262,7 +262,7 @@ where pub fn iter<'a>(self, start_key: &'a [u8; L], ctx: &'a RequestContext) -> DiskBtreeIterator<'a> where - R: 'a, + R: 'a + Send, { DiskBtreeIterator { stream: Box::pin(self.into_stream(start_key, ctx)), @@ -521,7 +521,7 @@ where pub struct DiskBtreeIterator<'a> { #[allow(clippy::type_complexity)] stream: std::pin::Pin< - Box, u64), DiskBtreeError>> + 'a>, + Box, u64), DiskBtreeError>> + 'a + Send>, >, } diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index 2f0c45317d9a..a389358f0d27 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -6,8 +6,6 @@ pub(crate) mod inmemory_layer; pub(crate) mod layer; mod layer_desc; mod layer_name; - -#[cfg(test)] pub mod merge_iterator; use crate::context::{AccessStatsBehavior, RequestContext}; diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 43941b6e1739..c34923320aee 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -33,11 +33,14 @@ use crate::page_cache::{self, FileId, PAGE_SZ}; use crate::repository::{Key, Value, KEY_SIZE}; use crate::tenant::blob_io::BlobWriter; use crate::tenant::block_io::{BlockBuf, BlockCursor, BlockLease, BlockReader, FileBlockReader}; -use crate::tenant::disk_btree::{DiskBtreeBuilder, DiskBtreeReader, VisitDirection}; +use crate::tenant::disk_btree::{ + DiskBtreeBuilder, DiskBtreeIterator, DiskBtreeReader, VisitDirection, +}; use crate::tenant::storage_layer::{Layer, ValueReconstructResult, ValueReconstructState}; use crate::tenant::timeline::GetVectoredError; use crate::tenant::vectored_blob_io::{ - BlobFlag, MaxVectoredReadBytes, VectoredBlobReader, VectoredRead, VectoredReadPlanner, + BlobFlag, MaxVectoredReadBytes, StreamingVectoredReadPlanner, VectoredBlobReader, VectoredRead, + VectoredReadPlanner, }; use crate::tenant::{PageReconstructError, Timeline}; use crate::virtual_file::{self, VirtualFile}; @@ -53,6 +56,7 @@ use pageserver_api::models::{ImageCompressionAlgorithm, LayerAccessKind}; use pageserver_api::shard::TenantShardId; use rand::{distributions::Alphanumeric, Rng}; use serde::{Deserialize, Serialize}; +use std::collections::VecDeque; use std::fs::File; use std::io::SeekFrom; use std::ops::Range; @@ -747,12 +751,10 @@ impl DeltaLayer { } impl DeltaLayerInner { - #[cfg(test)] pub(crate) fn key_range(&self) -> &Range { &self.layer_key_range } - #[cfg(test)] pub(crate) fn lsn_range(&self) -> &Range { &self.layer_lsn_range } @@ -1512,7 +1514,6 @@ impl DeltaLayerInner { offset } - #[cfg(test)] pub(crate) fn iter<'a>(&'a self, ctx: &'a RequestContext) -> DeltaLayerIterator<'a> { let block_reader = FileBlockReader::new(&self.file, self.file_id); let tree_reader = @@ -1523,7 +1524,7 @@ impl DeltaLayerInner { index_iter: tree_reader.iter(&[0; DELTA_KEY_SIZE], ctx), key_values_batch: std::collections::VecDeque::new(), is_end: false, - planner: crate::tenant::vectored_blob_io::StreamingVectoredReadPlanner::new( + planner: StreamingVectoredReadPlanner::new( 1024 * 8192, // The default value. Unit tests might use a different value. 1024 * 8K = 8MB buffer. 1024, // The default value. Unit tests might use a different value ), @@ -1595,17 +1596,15 @@ impl<'a> pageserver_compaction::interface::CompactionDeltaEntry<'a, Key> for Del } } -#[cfg(test)] pub struct DeltaLayerIterator<'a> { delta_layer: &'a DeltaLayerInner, ctx: &'a RequestContext, - planner: crate::tenant::vectored_blob_io::StreamingVectoredReadPlanner, - index_iter: crate::tenant::disk_btree::DiskBtreeIterator<'a>, - key_values_batch: std::collections::VecDeque<(Key, Lsn, Value)>, + planner: StreamingVectoredReadPlanner, + index_iter: DiskBtreeIterator<'a>, + key_values_batch: VecDeque<(Key, Lsn, Value)>, is_end: bool, } -#[cfg(test)] impl<'a> DeltaLayerIterator<'a> { /// Retrieve a batch of key-value pairs into the iterator buffer. async fn next_batch(&mut self) -> anyhow::Result<()> { diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index a88a1e642958..c7f41b66befc 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -29,13 +29,16 @@ use crate::page_cache::{self, FileId, PAGE_SZ}; use crate::repository::{Key, Value, KEY_SIZE}; use crate::tenant::blob_io::BlobWriter; use crate::tenant::block_io::{BlockBuf, BlockReader, FileBlockReader}; -use crate::tenant::disk_btree::{DiskBtreeBuilder, DiskBtreeReader, VisitDirection}; +use crate::tenant::disk_btree::{ + DiskBtreeBuilder, DiskBtreeIterator, DiskBtreeReader, VisitDirection, +}; use crate::tenant::storage_layer::{ LayerAccessStats, ValueReconstructResult, ValueReconstructState, }; use crate::tenant::timeline::GetVectoredError; use crate::tenant::vectored_blob_io::{ - BlobFlag, MaxVectoredReadBytes, VectoredBlobReader, VectoredRead, VectoredReadPlanner, + BlobFlag, MaxVectoredReadBytes, StreamingVectoredReadPlanner, VectoredBlobReader, VectoredRead, + VectoredReadPlanner, }; use crate::tenant::{PageReconstructError, Timeline}; use crate::virtual_file::{self, VirtualFile}; @@ -50,6 +53,7 @@ use pageserver_api::models::LayerAccessKind; use pageserver_api::shard::{ShardIdentity, TenantShardId}; use rand::{distributions::Alphanumeric, Rng}; use serde::{Deserialize, Serialize}; +use std::collections::VecDeque; use std::fs::File; use std::io::SeekFrom; use std::ops::Range; @@ -369,12 +373,10 @@ impl ImageLayer { } impl ImageLayerInner { - #[cfg(test)] pub(crate) fn key_range(&self) -> &Range { &self.key_range } - #[cfg(test)] pub(crate) fn lsn(&self) -> Lsn { self.lsn } @@ -699,7 +701,6 @@ impl ImageLayerInner { } } - #[cfg(test)] pub(crate) fn iter<'a>(&'a self, ctx: &'a RequestContext) -> ImageLayerIterator<'a> { let block_reader = FileBlockReader::new(&self.file, self.file_id); let tree_reader = @@ -708,9 +709,9 @@ impl ImageLayerInner { image_layer: self, ctx, index_iter: tree_reader.iter(&[0; KEY_SIZE], ctx), - key_values_batch: std::collections::VecDeque::new(), + key_values_batch: VecDeque::new(), is_end: false, - planner: crate::tenant::vectored_blob_io::StreamingVectoredReadPlanner::new( + planner: StreamingVectoredReadPlanner::new( 1024 * 8192, // The default value. Unit tests might use a different value. 1024 * 8K = 8MB buffer. 1024, // The default value. Unit tests might use a different value ), @@ -974,17 +975,15 @@ impl Drop for ImageLayerWriter { } } -#[cfg(test)] pub struct ImageLayerIterator<'a> { image_layer: &'a ImageLayerInner, ctx: &'a RequestContext, - planner: crate::tenant::vectored_blob_io::StreamingVectoredReadPlanner, - index_iter: crate::tenant::disk_btree::DiskBtreeIterator<'a>, - key_values_batch: std::collections::VecDeque<(Key, Lsn, Value)>, + planner: StreamingVectoredReadPlanner, + index_iter: DiskBtreeIterator<'a>, + key_values_batch: VecDeque<(Key, Lsn, Value)>, is_end: bool, } -#[cfg(test)] impl<'a> ImageLayerIterator<'a> { /// Retrieve a batch of key-value pairs into the iterator buffer. async fn next_batch(&mut self) -> anyhow::Result<()> { diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index dbf6c60aaee0..d9cbaba529d5 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -385,6 +385,7 @@ impl Layer { } /// Get all key/values in the layer. Should be replaced with an iterator-based API in the future. + #[allow(dead_code)] pub(crate) async fn load_key_values( &self, ctx: &RequestContext, @@ -1918,7 +1919,7 @@ impl ResidentLayer { self.owner.metadata() } - #[cfg(test)] + /// Cast the layer to a delta, return an error if it is an image layer. pub(crate) async fn get_as_delta( &self, ctx: &RequestContext, @@ -1930,7 +1931,7 @@ impl ResidentLayer { } } - #[cfg(test)] + /// Cast the layer to an image, return an error if it is a delta layer. pub(crate) async fn get_as_image( &self, ctx: &RequestContext, diff --git a/pageserver/src/tenant/storage_layer/merge_iterator.rs b/pageserver/src/tenant/storage_layer/merge_iterator.rs index 0edfd4bd4075..6f59b2fd7765 100644 --- a/pageserver/src/tenant/storage_layer/merge_iterator.rs +++ b/pageserver/src/tenant/storage_layer/merge_iterator.rs @@ -547,5 +547,9 @@ mod tests { &ctx, ); assert_merge_iter_equal(&mut merge_iter, &expect).await; + + is_send(merge_iter); } + + fn is_send(_: impl Send) {} } diff --git a/pageserver/src/tenant/timeline/compaction.rs b/pageserver/src/tenant/timeline/compaction.rs index f251b667c2fb..a648432b4d08 100644 --- a/pageserver/src/tenant/timeline/compaction.rs +++ b/pageserver/src/tenant/timeline/compaction.rs @@ -27,6 +27,7 @@ use utils::id::TimelineId; use crate::context::{AccessStatsBehavior, RequestContext, RequestContextBuilder}; use crate::page_cache; use crate::tenant::config::defaults::{DEFAULT_CHECKPOINT_DISTANCE, DEFAULT_COMPACTION_THRESHOLD}; +use crate::tenant::storage_layer::merge_iterator::MergeIterator; use crate::tenant::storage_layer::{AsLayerDesc, PersistentLayerDesc}; use crate::tenant::timeline::{drop_rlock, DeltaLayerWriter, ImageLayerWriter}; use crate::tenant::timeline::{Hole, ImageLayerCreationOutcome}; @@ -1039,10 +1040,12 @@ impl Timeline { ); // Step 1: (In the future) construct a k-merge iterator over all layers. For now, simply collect all keys + LSNs. // Also, collect the layer information to decide when to split the new delta layers. - let mut all_key_values = Vec::new(); + let mut downloaded_layers = Vec::new(); let mut delta_split_points = BTreeSet::new(); for layer in &layer_selection { - all_key_values.extend(layer.load_key_values(ctx).await?); + let resident_layer = layer.download_and_keep_resident().await?; + downloaded_layers.push(resident_layer); + let desc = layer.layer_desc(); if desc.is_delta() { // TODO: is it correct to only record split points for deltas intersecting with the GC horizon? (exclude those below/above the horizon) @@ -1052,44 +1055,28 @@ impl Timeline { delta_split_points.insert(key_range.end); } } - // Key small to large, LSN low to high, if the same LSN has both image and delta due to the merge of delta layers and - // image layers, make image appear before than delta. - struct ValueWrapper<'a>(&'a crate::repository::Value); - impl Ord for ValueWrapper<'_> { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - use crate::repository::Value; - use std::cmp::Ordering; - match (self.0, other.0) { - (Value::Image(_), Value::WalRecord(_)) => Ordering::Less, - (Value::WalRecord(_), Value::Image(_)) => Ordering::Greater, - _ => Ordering::Equal, - } - } - } - impl PartialOrd for ValueWrapper<'_> { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } - } - impl PartialEq for ValueWrapper<'_> { - fn eq(&self, other: &Self) -> bool { - self.cmp(other) == std::cmp::Ordering::Equal + let mut delta_layers = Vec::new(); + let mut image_layers = Vec::new(); + for resident_layer in &downloaded_layers { + if resident_layer.layer_desc().is_delta() { + let layer = resident_layer.get_as_delta(ctx).await?; + delta_layers.push(layer); + } else { + let layer = resident_layer.get_as_image(ctx).await?; + image_layers.push(layer); } } - impl Eq for ValueWrapper<'_> {} - all_key_values.sort_by(|(k1, l1, v1), (k2, l2, v2)| { - (k1, l1, ValueWrapper(v1)).cmp(&(k2, l2, ValueWrapper(v2))) - }); + let mut merge_iter = MergeIterator::create(&delta_layers, &image_layers, ctx); // Step 2: Produce images+deltas. TODO: ensure newly-produced delta does not overlap with other deltas. // Data of the same key. let mut accumulated_values = Vec::new(); - let mut last_key = all_key_values.first().unwrap().0; // TODO: assert all_key_values not empty + let mut last_key: Option = None; /// Take a list of images and deltas, produce an image at the GC horizon, and a list of deltas above the GC horizon. async fn flush_accumulated_states( tline: &Arc, key: Key, - accumulated_values: &[&(Key, Lsn, crate::repository::Value)], + accumulated_values: &[(Key, Lsn, crate::repository::Value)], horizon: Lsn, ) -> anyhow::Result<(Vec<(Key, Lsn, crate::repository::Value)>, bytes::Bytes)> { let mut base_image = None; @@ -1190,7 +1177,7 @@ impl Timeline { self.conf, self.timeline_id, self.tenant_shard_id, - &(all_key_values.first().unwrap().0..all_key_values.last().unwrap().0.next()), + &(Key::MIN..Key::MAX), // covers the full key range gc_cutoff, ctx, ) @@ -1200,20 +1187,24 @@ impl Timeline { let delta_split_points = delta_split_points.into_iter().collect_vec(); let mut current_delta_split_point = 0; let mut delta_layers = Vec::new(); - for item @ (key, _, _) in &all_key_values { - if &last_key == key { - accumulated_values.push(item); + while let Some((key, lsn, val)) = merge_iter.next().await? { + if last_key.is_none() || last_key.as_ref() == Some(&key) { + if last_key.is_none() { + last_key = Some(key); + } + accumulated_values.push((key, lsn, val)); } else { + let last_key = last_key.as_mut().unwrap(); let (deltas, image) = - flush_accumulated_states(self, last_key, &accumulated_values, gc_cutoff) + flush_accumulated_states(self, *last_key, &accumulated_values, gc_cutoff) .await?; // Put the image into the image layer. Currently we have a single big layer for the compaction. - image_layer_writer.put_image(last_key, image, ctx).await?; + image_layer_writer.put_image(*last_key, image, ctx).await?; delta_values.extend(deltas); delta_layers.extend( flush_deltas( &mut delta_values, - last_key, + *last_key, &delta_split_points, &mut current_delta_split_point, self, @@ -1223,11 +1214,12 @@ impl Timeline { .await?, ); accumulated_values.clear(); - accumulated_values.push(item); - last_key = *key; + *last_key = key; + accumulated_values.push((key, lsn, val)); } } + let last_key = last_key.expect("no keys produced during compaction"); // TODO: move this part to the loop body let (deltas, image) = flush_accumulated_states(self, last_key, &accumulated_values, gc_cutoff).await?; diff --git a/pageserver/src/tenant/vectored_blob_io.rs b/pageserver/src/tenant/vectored_blob_io.rs index 5a0986ea12ec..54a3ad789b9f 100644 --- a/pageserver/src/tenant/vectored_blob_io.rs +++ b/pageserver/src/tenant/vectored_blob_io.rs @@ -396,7 +396,6 @@ impl<'a> VectoredBlobReader<'a> { /// Read planner used in [`crate::tenant::storage_layer::image_layer::ImageLayerIterator`]. It provides a streaming API for /// getting read blobs. It returns a batch when `handle` gets called and when the current key would just exceed the read_size and /// max_cnt constraints. -#[cfg(test)] pub struct StreamingVectoredReadPlanner { read_builder: Option, // Arguments for previous blob passed into [`StreamingVectoredReadPlanner::handle`] @@ -410,7 +409,6 @@ pub struct StreamingVectoredReadPlanner { cnt: usize, } -#[cfg(test)] impl StreamingVectoredReadPlanner { pub fn new(max_read_size: u64, max_cnt: usize) -> Self { assert!(max_cnt > 0); From 841b76ea7cd7a6f8cf47cbacdbbc1b4fefce331f Mon Sep 17 00:00:00 2001 From: Peter Bendel Date: Thu, 18 Jul 2024 18:18:18 +0200 Subject: [PATCH 139/194] Temporarily use vanilla pgbench and psql (client) for running pgvector benchmark (#8422) ## Problem https://github.com/neondatabase/neon/issues/8275 is not yet fixed Periodic benchmarking fails with SIGABRT in pgvector step, see https://github.com/neondatabase/neon/actions/runs/9967453263/job/27541159738#step:7:393 ## Summary of changes Instead of using pgbench and psql from Neon artifacts, download vanilla postgres binaries into the container and use those to run the client side of the test. --- .github/workflows/benchmarking.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index d785156a29b1..833a4ce33c7f 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -457,17 +457,21 @@ jobs: runs-on: [ self-hosted, us-east-2, x64 ] container: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:pinned - options: --init + options: --init --user root steps: - uses: actions/checkout@v4 - - name: Download Neon artifact - uses: ./.github/actions/download - with: - name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact - path: /tmp/neon/ - prefix: latest + # until https://github.com/neondatabase/neon/issues/8275 is fixed we temporarily install postgresql-16 + # instead of using Neon artifacts containing pgbench + - name: Install postgresql-16 where pytest expects it + run: | + apt-get update && apt-get install -y postgresql-common + /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y + apt-get -y install postgresql-16 + mkdir -p /tmp/neon/pg_install/v16/bin + ln -s /usr/bin/pgbench /tmp/neon/pg_install/v16/bin/pgbench + ln -s /usr/bin/psql /tmp/neon/pg_install/v16/bin/psql - name: Set up Connection String id: set-up-connstr From 5a772761ee799d1d4537fcb2ab5a973e7be4d754 Mon Sep 17 00:00:00 2001 From: Arthur Petukhovsky Date: Thu, 18 Jul 2024 17:26:27 +0100 Subject: [PATCH 140/194] Change log level for GuardDrop error (#8305) The error means that manager exited earlier than `ResidenceGuard` and it's not unexpected with current deletion implementation. This commit changes log level to reduse noise. --- safekeeper/src/timeline_guard.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/safekeeper/src/timeline_guard.rs b/safekeeper/src/timeline_guard.rs index e249c859b4bb..dbdf46412ddb 100644 --- a/safekeeper/src/timeline_guard.rs +++ b/safekeeper/src/timeline_guard.rs @@ -4,7 +4,7 @@ use std::collections::HashSet; -use tracing::{debug, warn}; +use tracing::debug; use crate::timeline_manager::ManagerCtlMessage; @@ -23,7 +23,7 @@ impl Drop for ResidenceGuard { .manager_tx .send(ManagerCtlMessage::GuardDrop(self.guard_id)); if let Err(e) = res { - warn!("failed to send GuardDrop message: {:?}", e); + debug!("failed to send GuardDrop message: {:?}", e); } } } From c96e8012ce2472964f7dff13110d57f7ba5db2b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Thu, 18 Jul 2024 20:09:57 +0200 Subject: [PATCH 141/194] Enable zstd in tests (#8368) Successor of #8288 , just enable zstd in tests. Also adds a test that creates easily compressable data. Part of #5431 --------- Co-authored-by: John Spray Co-authored-by: Joonas Koivunen --- pageserver/src/metrics.rs | 16 ++++ .../src/tenant/storage_layer/image_layer.rs | 10 ++ test_runner/fixtures/neon_fixtures.py | 1 + test_runner/regress/test_compaction.py | 93 ++++++++++++++++++- .../regress/test_disk_usage_eviction.py | 3 + 5 files changed, 122 insertions(+), 1 deletion(-) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 753f5524c55d..c03567f6efb2 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -610,6 +610,22 @@ pub(crate) static CIRCUIT_BREAKERS_UNBROKEN: Lazy = Lazy::new(|| { .expect("failed to define a metric") }); +pub(crate) static COMPRESSION_IMAGE_INPUT_BYTES: Lazy = Lazy::new(|| { + register_int_counter!( + "pageserver_compression_image_in_bytes_total", + "Size of uncompressed data written into image layers" + ) + .expect("failed to define a metric") +}); + +pub(crate) static COMPRESSION_IMAGE_OUTPUT_BYTES: Lazy = Lazy::new(|| { + register_int_counter!( + "pageserver_compression_image_out_bytes_total", + "Size of compressed image layer written" + ) + .expect("failed to define a metric") +}); + pub(crate) mod initial_logical_size { use metrics::{register_int_counter, register_int_counter_vec, IntCounter, IntCounterVec}; use once_cell::sync::Lazy; diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index c7f41b66befc..45b47bb62b0c 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -738,6 +738,9 @@ struct ImageLayerWriterInner { key_range: Range, lsn: Lsn, + // Total uncompressed bytes passed into put_image + uncompressed_bytes: u64, + blob_writer: BlobWriter, tree: DiskBtreeBuilder, } @@ -793,6 +796,7 @@ impl ImageLayerWriterInner { lsn, tree: tree_builder, blob_writer, + uncompressed_bytes: 0, }; Ok(writer) @@ -811,6 +815,7 @@ impl ImageLayerWriterInner { ) -> anyhow::Result<()> { ensure!(self.key_range.contains(&key)); let compression = self.conf.image_compression; + self.uncompressed_bytes += img.len() as u64; let (_img, res) = self .blob_writer .write_blob_maybe_compressed(img, ctx, compression) @@ -836,6 +841,11 @@ impl ImageLayerWriterInner { let index_start_blk = ((self.blob_writer.size() + PAGE_SZ as u64 - 1) / PAGE_SZ as u64) as u32; + // Calculate compression ratio + let compressed_size = self.blob_writer.size() - PAGE_SZ as u64; // Subtract PAGE_SZ for header + crate::metrics::COMPRESSION_IMAGE_INPUT_BYTES.inc_by(self.uncompressed_bytes); + crate::metrics::COMPRESSION_IMAGE_OUTPUT_BYTES.inc_by(compressed_size); + let mut file = self.blob_writer.into_inner(); // Write out the index diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 567ca532f97a..db7269ad4148 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1158,6 +1158,7 @@ def __init__(self, config: NeonEnvBuilder): "listen_http_addr": f"localhost:{pageserver_port.http}", "pg_auth_type": pg_auth_type, "http_auth_type": http_auth_type, + "image_compression": "zstd", } if self.pageserver_virtual_file_io_engine is not None: ps_cfg["virtual_file_io_engine"] = self.pageserver_virtual_file_io_engine diff --git a/test_runner/regress/test_compaction.py b/test_runner/regress/test_compaction.py index f321c09b2729..be787e064262 100644 --- a/test_runner/regress/test_compaction.py +++ b/test_runner/regress/test_compaction.py @@ -6,7 +6,10 @@ import pytest from fixtures.log_helper import log -from fixtures.neon_fixtures import NeonEnvBuilder, generate_uploads_and_deletions +from fixtures.neon_fixtures import ( + NeonEnvBuilder, + generate_uploads_and_deletions, +) from fixtures.pageserver.http import PageserverApiException from fixtures.utils import wait_until from fixtures.workload import Workload @@ -142,6 +145,10 @@ def test_sharding_compaction( "image_layer_creation_check_threshold": 0, } + # Disable compression, as we can't estimate the size of layers with compression enabled + # TODO: implement eager layer cutting during compaction + neon_env_builder.pageserver_config_override = "image_compression='disabled'" + neon_env_builder.num_pageservers = 1 if shard_count is None else shard_count env = neon_env_builder.init_start( initial_tenant_conf=TENANT_CONF, @@ -320,3 +327,87 @@ def assert_broken(): or 0 ) == 0 assert not env.pageserver.log_contains(".*Circuit breaker failure ended.*") + + +@pytest.mark.parametrize("enabled", [True, False]) +def test_image_layer_compression(neon_env_builder: NeonEnvBuilder, enabled: bool): + tenant_conf = { + # small checkpointing and compaction targets to ensure we generate many upload operations + "checkpoint_distance": f"{128 * 1024}", + "compaction_threshold": "1", + "compaction_target_size": f"{128 * 1024}", + # no PITR horizon, we specify the horizon when we request on-demand GC + "pitr_interval": "0s", + # disable background compaction and GC. We invoke it manually when we want it to happen. + "gc_period": "0s", + "compaction_period": "0s", + # create image layers as eagerly as possible + "image_creation_threshold": "1", + "image_layer_creation_check_threshold": "0", + } + + # Explicitly enable/disable compression, rather than using default + if enabled: + neon_env_builder.pageserver_config_override = "image_compression='zstd'" + else: + neon_env_builder.pageserver_config_override = "image_compression='disabled'" + + env = neon_env_builder.init_start(initial_tenant_conf=tenant_conf) + + tenant_id = env.initial_tenant + timeline_id = env.initial_timeline + + pageserver = env.pageserver + ps_http = env.pageserver.http_client() + with env.endpoints.create_start( + "main", tenant_id=tenant_id, pageserver_id=pageserver.id + ) as endpoint: + endpoint.safe_psql("CREATE TABLE foo (id INTEGER PRIMARY KEY, val text)") + # Generate around 800k worth of easily compressible data to store + for v in range(100): + endpoint.safe_psql( + f"INSERT INTO foo (id, val) VALUES ({v}, repeat('abcde{v:0>3}', 500))" + ) + # run compaction to create image layers + ps_http.timeline_checkpoint(tenant_id, timeline_id, wait_until_uploaded=True) + + layer_map = ps_http.layer_map_info(tenant_id, timeline_id) + image_layer_count = 0 + delta_layer_count = 0 + for layer in layer_map.historic_layers: + if layer.kind == "Image": + image_layer_count += 1 + elif layer.kind == "Delta": + delta_layer_count += 1 + assert image_layer_count > 0 + assert delta_layer_count > 0 + + log.info(f"images: {image_layer_count}, deltas: {delta_layer_count}") + + bytes_in = pageserver.http_client().get_metric_value( + "pageserver_compression_image_in_bytes_total" + ) + bytes_out = pageserver.http_client().get_metric_value( + "pageserver_compression_image_out_bytes_total" + ) + assert bytes_in is not None + assert bytes_out is not None + log.info(f"Compression ratio: {bytes_out/bytes_in} ({bytes_out} in, {bytes_out} out)") + + if enabled: + # We are writing high compressible repetitive plain text, expect excellent compression + EXPECT_RATIO = 0.2 + assert bytes_out / bytes_in < EXPECT_RATIO + else: + # Nothing should be compressed if we disabled it. + assert bytes_out >= bytes_in + + # Destroy the endpoint and create a new one to resetthe caches + with env.endpoints.create_start( + "main", tenant_id=tenant_id, pageserver_id=pageserver.id + ) as endpoint: + for v in range(100): + res = endpoint.safe_psql( + f"SELECT count(*) FROM foo WHERE id={v} and val=repeat('abcde{v:0>3}', 500)" + ) + assert res[0][0] == 1 diff --git a/test_runner/regress/test_disk_usage_eviction.py b/test_runner/regress/test_disk_usage_eviction.py index fb8b7b22fa71..3c834f430b08 100644 --- a/test_runner/regress/test_disk_usage_eviction.py +++ b/test_runner/regress/test_disk_usage_eviction.py @@ -230,6 +230,9 @@ def _eviction_env( neon_env_builder.num_pageservers = num_pageservers neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) + # Disable compression support for EvictionEnv to get larger layer sizes + neon_env_builder.pageserver_config_override = "image_compression='disabled'" + # initial tenant will not be present on this pageserver env = neon_env_builder.init_configs() env.start() From 392d3524f955de375c119f1cdf99a9069843dc67 Mon Sep 17 00:00:00 2001 From: Peter Bendel Date: Fri, 19 Jul 2024 15:40:55 +0200 Subject: [PATCH 142/194] Bodobolero/fix root permissions (#8429) ## Problem My prior PR https://github.com/neondatabase/neon/pull/8422 caused leftovers in the GitHub action runner work directory with root permission. As an example see here https://github.com/neondatabase/neon/actions/runs/10001857641/job/27646237324#step:3:37 To work-around we install vanilla postgres as non-root using deb packages in /home/nonroot user directory ## Summary of changes - since we cannot use root we install the deb pkgs directly and create symbolic links for psql, pgbench and libs in expected places - continue jobs an aws even if azure jobs fail (because this region is currently unreliable) --- .github/workflows/benchmarking.yml | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index 833a4ce33c7f..c132b5b513ff 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -57,6 +57,7 @@ jobs: bench: if: ${{ github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null }} strategy: + fail-fast: false matrix: include: - DEFAULT_PG_VERSION: 16 @@ -439,6 +440,7 @@ jobs: pgbench-pgvector: strategy: + fail-fast: false matrix: include: - PLATFORM: "neon-captest-pgvector" @@ -451,13 +453,14 @@ jobs: DEFAULT_PG_VERSION: 16 TEST_OUTPUT: /tmp/test_output BUILD_TYPE: remote + LD_LIBRARY_PATH: /home/nonroot/pg/usr/lib/x86_64-linux-gnu SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }} PLATFORM: ${{ matrix.PLATFORM }} runs-on: [ self-hosted, us-east-2, x64 ] container: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:pinned - options: --init --user root + options: --init steps: - uses: actions/checkout@v4 @@ -466,12 +469,19 @@ jobs: # instead of using Neon artifacts containing pgbench - name: Install postgresql-16 where pytest expects it run: | - apt-get update && apt-get install -y postgresql-common - /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y - apt-get -y install postgresql-16 + cd /home/nonroot + wget -q https://apt.postgresql.org/pub/repos/apt/pool/main/p/postgresql-16/libpq5_16.3-1.pgdg110%2B1_amd64.deb + wget -q https://apt.postgresql.org/pub/repos/apt/pool/main/p/postgresql-16/postgresql-client-16_16.3-1.pgdg110%2B1_amd64.deb + wget -q https://apt.postgresql.org/pub/repos/apt/pool/main/p/postgresql-16/postgresql-16_16.3-1.pgdg110%2B1_amd64.deb + dpkg -x libpq5_16.3-1.pgdg110+1_amd64.deb pg + dpkg -x postgresql-client-16_16.3-1.pgdg110+1_amd64.deb pg + dpkg -x postgresql-16_16.3-1.pgdg110+1_amd64.deb pg mkdir -p /tmp/neon/pg_install/v16/bin - ln -s /usr/bin/pgbench /tmp/neon/pg_install/v16/bin/pgbench - ln -s /usr/bin/psql /tmp/neon/pg_install/v16/bin/psql + ln -s /home/nonroot/pg/usr/lib/postgresql/16/bin/pgbench /tmp/neon/pg_install/v16/bin/pgbench + ln -s /home/nonroot/pg/usr/lib/postgresql/16/bin/psql /tmp/neon/pg_install/v16/bin/psql + ln -s /home/nonroot/pg/usr/lib/x86_64-linux-gnu /tmp/neon/pg_install/v16/lib + /tmp/neon/pg_install/v16/bin/pgbench --version + /tmp/neon/pg_install/v16/bin/psql --version - name: Set up Connection String id: set-up-connstr @@ -532,7 +542,6 @@ jobs: env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} - clickbench-compare: # ClichBench DB for rds-aurora and rds-Postgres deployed to the same clusters # we use for performance testing in pgbench-compare. From 16071e57c642bddfcbe2aabc60acd9a788e2fadb Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Fri, 19 Jul 2024 18:01:02 +0200 Subject: [PATCH 143/194] pageserver: remove obsolete cached_metric_collection_interval (#8370) We're removing the usage of this long-meaningless config field in https://github.com/neondatabase/aws/pull/1599 Once that PR has been deployed to staging and prod, we can merge this PR. --- pageserver/src/bin/pageserver.rs | 1 - pageserver/src/config.rs | 24 ------------------- pageserver/src/consumption_metrics.rs | 7 ------ .../test_pageserver_metric_collection.py | 2 -- 4 files changed, 34 deletions(-) diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 9f705f0bc923..fceddfb7575c 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -622,7 +622,6 @@ fn start_pageserver( metric_collection_endpoint, &conf.metric_collection_bucket, conf.metric_collection_interval, - conf.cached_metric_collection_interval, conf.synthetic_size_calculation_interval, conf.id, local_disk_storage, diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index 5b103b551fb1..35b4e7936524 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -68,7 +68,6 @@ pub mod defaults { super::ConfigurableSemaphore::DEFAULT_INITIAL.get(); pub const DEFAULT_METRIC_COLLECTION_INTERVAL: &str = "10 min"; - pub const DEFAULT_CACHED_METRIC_COLLECTION_INTERVAL: &str = "0s"; pub const DEFAULT_METRIC_COLLECTION_ENDPOINT: Option = None; pub const DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL: &str = "10 min"; pub const DEFAULT_BACKGROUND_TASK_MAXIMUM_DELAY: &str = "10s"; @@ -123,7 +122,6 @@ pub mod defaults { #concurrent_tenant_warmup = '{DEFAULT_CONCURRENT_TENANT_WARMUP}' #metric_collection_interval = '{DEFAULT_METRIC_COLLECTION_INTERVAL}' -#cached_metric_collection_interval = '{DEFAULT_CACHED_METRIC_COLLECTION_INTERVAL}' #synthetic_size_calculation_interval = '{DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL}' #disk_usage_based_eviction = {{ max_usage_pct = .., min_avail_bytes = .., period = "10s"}} @@ -238,7 +236,6 @@ pub struct PageServerConf { // How often to collect metrics and send them to the metrics endpoint. pub metric_collection_interval: Duration, // How often to send unchanged cached metrics to the metrics endpoint. - pub cached_metric_collection_interval: Duration, pub metric_collection_endpoint: Option, pub metric_collection_bucket: Option, pub synthetic_size_calculation_interval: Duration, @@ -370,7 +367,6 @@ struct PageServerConfigBuilder { concurrent_tenant_size_logical_size_queries: BuilderValue, metric_collection_interval: BuilderValue, - cached_metric_collection_interval: BuilderValue, metric_collection_endpoint: BuilderValue>, synthetic_size_calculation_interval: BuilderValue, metric_collection_bucket: BuilderValue>, @@ -454,10 +450,6 @@ impl PageServerConfigBuilder { DEFAULT_METRIC_COLLECTION_INTERVAL, ) .expect("cannot parse default metric collection interval")), - cached_metric_collection_interval: Set(humantime::parse_duration( - DEFAULT_CACHED_METRIC_COLLECTION_INTERVAL, - ) - .expect("cannot parse default cached_metric_collection_interval")), synthetic_size_calculation_interval: Set(humantime::parse_duration( DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL, ) @@ -589,14 +581,6 @@ impl PageServerConfigBuilder { self.metric_collection_interval = BuilderValue::Set(metric_collection_interval) } - pub fn cached_metric_collection_interval( - &mut self, - cached_metric_collection_interval: Duration, - ) { - self.cached_metric_collection_interval = - BuilderValue::Set(cached_metric_collection_interval) - } - pub fn metric_collection_endpoint(&mut self, metric_collection_endpoint: Option) { self.metric_collection_endpoint = BuilderValue::Set(metric_collection_endpoint) } @@ -730,7 +714,6 @@ impl PageServerConfigBuilder { broker_keepalive_interval, log_format, metric_collection_interval, - cached_metric_collection_interval, metric_collection_endpoint, metric_collection_bucket, synthetic_size_calculation_interval, @@ -947,7 +930,6 @@ impl PageServerConf { NonZeroUsize::new(permits).context("initial semaphore permits out of range: 0, use other configuration to disable a feature")? }), "metric_collection_interval" => builder.metric_collection_interval(parse_toml_duration(key, item)?), - "cached_metric_collection_interval" => builder.cached_metric_collection_interval(parse_toml_duration(key, item)?), "metric_collection_endpoint" => { let endpoint = parse_toml_string(key, item)?.parse().context("failed to parse metric_collection_endpoint")?; builder.metric_collection_endpoint(Some(endpoint)); @@ -1080,7 +1062,6 @@ impl PageServerConf { eviction_task_immitated_concurrent_logical_size_queries: ConfigurableSemaphore::default( ), metric_collection_interval: Duration::from_secs(60), - cached_metric_collection_interval: Duration::from_secs(60 * 60), metric_collection_endpoint: defaults::DEFAULT_METRIC_COLLECTION_ENDPOINT, metric_collection_bucket: None, synthetic_size_calculation_interval: Duration::from_secs(60), @@ -1259,7 +1240,6 @@ initial_superuser_name = 'zzzz' id = 10 metric_collection_interval = '222 s' -cached_metric_collection_interval = '22200 s' metric_collection_endpoint = 'http://localhost:80/metrics' synthetic_size_calculation_interval = '333 s' @@ -1315,9 +1295,6 @@ background_task_maximum_delay = '334 s' metric_collection_interval: humantime::parse_duration( defaults::DEFAULT_METRIC_COLLECTION_INTERVAL )?, - cached_metric_collection_interval: humantime::parse_duration( - defaults::DEFAULT_CACHED_METRIC_COLLECTION_INTERVAL - )?, metric_collection_endpoint: defaults::DEFAULT_METRIC_COLLECTION_ENDPOINT, metric_collection_bucket: None, synthetic_size_calculation_interval: humantime::parse_duration( @@ -1396,7 +1373,6 @@ background_task_maximum_delay = '334 s' eviction_task_immitated_concurrent_logical_size_queries: ConfigurableSemaphore::default(), metric_collection_interval: Duration::from_secs(222), - cached_metric_collection_interval: Duration::from_secs(22200), metric_collection_endpoint: Some(Url::parse("http://localhost:80/metrics")?), metric_collection_bucket: None, synthetic_size_calculation_interval: Duration::from_secs(333), diff --git a/pageserver/src/consumption_metrics.rs b/pageserver/src/consumption_metrics.rs index 18c1a6cd9bc2..6861adad2c24 100644 --- a/pageserver/src/consumption_metrics.rs +++ b/pageserver/src/consumption_metrics.rs @@ -46,19 +46,12 @@ pub async fn collect_metrics( metric_collection_endpoint: &Url, metric_collection_bucket: &Option, metric_collection_interval: Duration, - _cached_metric_collection_interval: Duration, synthetic_size_calculation_interval: Duration, node_id: NodeId, local_disk_storage: Utf8PathBuf, cancel: CancellationToken, ctx: RequestContext, ) -> anyhow::Result<()> { - if _cached_metric_collection_interval != Duration::ZERO { - tracing::warn!( - "cached_metric_collection_interval is no longer used, please set it to zero." - ) - } - // spin up background worker that caclulates tenant sizes let worker_ctx = ctx.detached_child(TaskKind::CalculateSyntheticSize, DownloadBehavior::Download); diff --git a/test_runner/regress/test_pageserver_metric_collection.py b/test_runner/regress/test_pageserver_metric_collection.py index cea35a6acb73..24a37b04ec5f 100644 --- a/test_runner/regress/test_pageserver_metric_collection.py +++ b/test_runner/regress/test_pageserver_metric_collection.py @@ -58,7 +58,6 @@ def metrics_handler(request: Request) -> Response: metric_collection_interval="1s" metric_collection_endpoint="{metric_collection_endpoint}" metric_collection_bucket={remote_storage_to_toml_inline_table(neon_env_builder.pageserver_remote_storage)} - cached_metric_collection_interval="0s" synthetic_size_calculation_interval="3s" """ @@ -216,7 +215,6 @@ def metrics_handler(request: Request) -> Response: neon_env_builder.pageserver_config_override = f""" metric_collection_interval="1s" metric_collection_endpoint="{metric_collection_endpoint}" - cached_metric_collection_interval="0s" synthetic_size_calculation_interval="3s" """ From 44781518d044de46f6fd1d58d9aece7bf399bc40 Mon Sep 17 00:00:00 2001 From: John Spray Date: Fri, 19 Jul 2024 17:07:59 +0100 Subject: [PATCH 144/194] storage scrubber: GC ancestor shard layers (#8196) ## Problem After a shard split, the pageserver leaves the ancestor shard's content in place. It may be referenced by child shards, but eventually child shards will de-reference most ancestor layers as they write their own data and do GC. We would like to eventually clean up those ancestor layers to reclaim space. ## Summary of changes - Extend the physical GC command with `--mode=full`, which includes cleaning up unreferenced ancestor shard layers - Add test `test_scrubber_physical_gc_ancestors` - Remove colored log output: in testing this is irritating ANSI code spam in logs, and in interactive use doesn't add much. - Refactor storage controller API client code out of storcon_client into a `storage_controller/client` crate - During physical GC of ancestors, call into the storage controller to check that the latest shards seen in S3 reflect the latest state of the tenant, and there is no shard split in progress. --- Cargo.lock | 41 +- Cargo.toml | 4 +- control_plane/storcon_cli/Cargo.toml | 1 + control_plane/storcon_cli/src/main.rs | 62 +-- libs/pageserver_api/src/controller_api.rs | 4 +- libs/utils/src/auth.rs | 4 + pageserver/src/auth.rs | 16 +- safekeeper/src/auth.rs | 16 +- storage_controller/client/Cargo.toml | 23 + storage_controller/client/src/control_api.rs | 62 +++ storage_controller/client/src/lib.rs | 1 + storage_controller/src/http.rs | 2 +- storage_controller/src/main.rs | 18 +- storage_controller/src/service.rs | 2 + storage_scrubber/Cargo.toml | 1 + storage_scrubber/src/lib.rs | 33 +- storage_scrubber/src/main.rs | 46 +- .../src/pageserver_physical_gc.rs | 481 +++++++++++++++--- test_runner/fixtures/neon_fixtures.py | 24 +- .../regress/test_pageserver_generations.py | 3 +- .../regress/test_pageserver_secondary.py | 6 +- test_runner/regress/test_sharding.py | 3 +- test_runner/regress/test_storage_scrubber.py | 237 ++++++++- test_runner/regress/test_tenant_delete.py | 6 +- 24 files changed, 905 insertions(+), 191 deletions(-) create mode 100644 storage_controller/client/Cargo.toml create mode 100644 storage_controller/client/src/control_api.rs create mode 100644 storage_controller/client/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index d08da0babd36..2505d4d3ed5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3234,16 +3234,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "nu-ansi-term" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" -dependencies = [ - "overload", - "winapi", -] - [[package]] name = "num" version = "0.4.1" @@ -3539,12 +3529,6 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - [[package]] name = "p256" version = "0.11.1" @@ -5822,6 +5806,28 @@ dependencies = [ "workspace_hack", ] +[[package]] +name = "storage_controller_client" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "futures", + "pageserver_api", + "pageserver_client", + "postgres", + "reqwest 0.12.4", + "serde", + "thiserror", + "tokio", + "tokio-postgres", + "tokio-stream", + "tokio-util", + "utils", + "workspace_hack", +] + [[package]] name = "storage_scrubber" version = "0.1.0" @@ -5856,6 +5862,7 @@ dependencies = [ "serde", "serde_json", "serde_with", + "storage_controller_client", "thiserror", "tokio", "tokio-postgres", @@ -5885,6 +5892,7 @@ dependencies = [ "reqwest 0.12.4", "serde", "serde_json", + "storage_controller_client", "thiserror", "tokio", "tracing", @@ -6611,7 +6619,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" dependencies = [ "matchers", - "nu-ansi-term", "once_cell", "regex", "serde", diff --git a/Cargo.toml b/Cargo.toml index b9b4bafb4f69..615f5472ec48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ members = [ "safekeeper", "storage_broker", "storage_controller", + "storage_controller/client", "storage_scrubber", "workspace_hack", "libs/compute_api", @@ -182,7 +183,7 @@ tower-service = "0.3.2" tracing = "0.1" tracing-error = "0.2.0" tracing-opentelemetry = "0.21.0" -tracing-subscriber = { version = "0.3", default-features = false, features = ["smallvec", "fmt", "tracing-log", "std", "env-filter", "json", "ansi"] } +tracing-subscriber = { version = "0.3", default-features = false, features = ["smallvec", "fmt", "tracing-log", "std", "env-filter", "json"] } twox-hash = { version = "1.6.3", default-features = false } typed-json = "0.1" url = "2.2" @@ -221,6 +222,7 @@ remote_storage = { version = "0.1", path = "./libs/remote_storage/" } safekeeper_api = { version = "0.1", path = "./libs/safekeeper_api" } desim = { version = "0.1", path = "./libs/desim" } storage_broker = { version = "0.1", path = "./storage_broker/" } # Note: main broker code is inside the binary crate, so linking with the library shouldn't be heavy. +storage_controller_client = { path = "./storage_controller/client" } tenant_size_model = { version = "0.1", path = "./libs/tenant_size_model/" } tracing-utils = { version = "0.1", path = "./libs/tracing-utils/" } utils = { version = "0.1", path = "./libs/utils/" } diff --git a/control_plane/storcon_cli/Cargo.toml b/control_plane/storcon_cli/Cargo.toml index f96f0084b2e3..be69208d0d12 100644 --- a/control_plane/storcon_cli/Cargo.toml +++ b/control_plane/storcon_cli/Cargo.toml @@ -17,6 +17,7 @@ pageserver_client.workspace = true reqwest.workspace = true serde.workspace = true serde_json = { workspace = true, features = ["raw_value"] } +storage_controller_client.workspace = true thiserror.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/control_plane/storcon_cli/src/main.rs b/control_plane/storcon_cli/src/main.rs index 777a717a7378..5c1add070aaf 100644 --- a/control_plane/storcon_cli/src/main.rs +++ b/control_plane/storcon_cli/src/main.rs @@ -14,15 +14,15 @@ use pageserver_api::{ }, shard::{ShardStripeSize, TenantShardId}, }; -use pageserver_client::mgmt_api::{self, ResponseErrorMessageExt}; +use pageserver_client::mgmt_api::{self}; use reqwest::{Method, StatusCode, Url}; -use serde::{de::DeserializeOwned, Serialize}; use utils::id::{NodeId, TenantId}; use pageserver_api::controller_api::{ NodeConfigureRequest, NodeRegisterRequest, NodeSchedulingPolicy, PlacementPolicy, TenantShardMigrateRequest, TenantShardMigrateResponse, }; +use storage_controller_client::control_api::Client; #[derive(Subcommand, Debug)] enum Command { @@ -249,64 +249,6 @@ impl FromStr for NodeAvailabilityArg { } } -struct Client { - base_url: Url, - jwt_token: Option, - client: reqwest::Client, -} - -impl Client { - fn new(base_url: Url, jwt_token: Option) -> Self { - Self { - base_url, - jwt_token, - client: reqwest::ClientBuilder::new() - .build() - .expect("Failed to construct http client"), - } - } - - /// Simple HTTP request wrapper for calling into storage controller - async fn dispatch( - &self, - method: Method, - path: String, - body: Option, - ) -> mgmt_api::Result - where - RQ: Serialize + Sized, - RS: DeserializeOwned + Sized, - { - // The configured URL has the /upcall path prefix for pageservers to use: we will strip that out - // for general purpose API access. - let url = Url::from_str(&format!( - "http://{}:{}/{path}", - self.base_url.host_str().unwrap(), - self.base_url.port().unwrap() - )) - .unwrap(); - - let mut builder = self.client.request(method, url); - if let Some(body) = body { - builder = builder.json(&body) - } - if let Some(jwt_token) = &self.jwt_token { - builder = builder.header( - reqwest::header::AUTHORIZATION, - format!("Bearer {jwt_token}"), - ); - } - - let response = builder.send().await.map_err(mgmt_api::Error::ReceiveBody)?; - let response = response.error_from_body().await?; - - response - .json() - .await - .map_err(pageserver_client::mgmt_api::Error::ReceiveBody) - } -} - #[tokio::main] async fn main() -> anyhow::Result<()> { let cli = Cli::parse(); diff --git a/libs/pageserver_api/src/controller_api.rs b/libs/pageserver_api/src/controller_api.rs index f05c1315eafa..d0e1eb6b2894 100644 --- a/libs/pageserver_api/src/controller_api.rs +++ b/libs/pageserver_api/src/controller_api.rs @@ -87,7 +87,7 @@ pub struct TenantLocateResponse { pub shard_params: ShardParameters, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug)] pub struct TenantDescribeResponse { pub tenant_id: TenantId, pub shards: Vec, @@ -110,7 +110,7 @@ pub struct NodeDescribeResponse { pub listen_pg_port: u16, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug)] pub struct TenantDescribeResponseShard { pub tenant_shard_id: TenantShardId, diff --git a/libs/utils/src/auth.rs b/libs/utils/src/auth.rs index 03e65f74fe12..a1170a460d99 100644 --- a/libs/utils/src/auth.rs +++ b/libs/utils/src/auth.rs @@ -33,6 +33,10 @@ pub enum Scope { GenerationsApi, // Allows access to control plane managment API and some storage controller endpoints. Admin, + + /// Allows access to storage controller APIs used by the scrubber, to interrogate the state + /// of a tenant & post scrub results. + Scrubber, } /// JWT payload. See docs/authentication.md for the format diff --git a/pageserver/src/auth.rs b/pageserver/src/auth.rs index 4785c8c4c5dc..9e3dedb75a11 100644 --- a/pageserver/src/auth.rs +++ b/pageserver/src/auth.rs @@ -14,12 +14,14 @@ pub fn check_permission(claims: &Claims, tenant_id: Option) -> Result< } (Scope::PageServerApi, None) => Ok(()), // access to management api for PageServerApi scope (Scope::PageServerApi, Some(_)) => Ok(()), // access to tenant api using PageServerApi scope - (Scope::Admin | Scope::SafekeeperData | Scope::GenerationsApi, _) => Err(AuthError( - format!( - "JWT scope '{:?}' is ineligible for Pageserver auth", - claims.scope - ) - .into(), - )), + (Scope::Admin | Scope::SafekeeperData | Scope::GenerationsApi | Scope::Scrubber, _) => { + Err(AuthError( + format!( + "JWT scope '{:?}' is ineligible for Pageserver auth", + claims.scope + ) + .into(), + )) + } } } diff --git a/safekeeper/src/auth.rs b/safekeeper/src/auth.rs index dd9058c4681a..b8bc3f3e0689 100644 --- a/safekeeper/src/auth.rs +++ b/safekeeper/src/auth.rs @@ -12,13 +12,15 @@ pub fn check_permission(claims: &Claims, tenant_id: Option) -> Result< } Ok(()) } - (Scope::Admin | Scope::PageServerApi | Scope::GenerationsApi, _) => Err(AuthError( - format!( - "JWT scope '{:?}' is ineligible for Safekeeper auth", - claims.scope - ) - .into(), - )), + (Scope::Admin | Scope::PageServerApi | Scope::GenerationsApi | Scope::Scrubber, _) => { + Err(AuthError( + format!( + "JWT scope '{:?}' is ineligible for Safekeeper auth", + claims.scope + ) + .into(), + )) + } (Scope::SafekeeperData, _) => Ok(()), } } diff --git a/storage_controller/client/Cargo.toml b/storage_controller/client/Cargo.toml new file mode 100644 index 000000000000..c3bfe2bfd2f9 --- /dev/null +++ b/storage_controller/client/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "storage_controller_client" +version = "0.1.0" +edition.workspace = true +license.workspace = true + +[dependencies] +pageserver_api.workspace = true +pageserver_client.workspace = true +thiserror.workspace = true +async-trait.workspace = true +reqwest.workspace = true +utils.workspace = true +serde.workspace = true +workspace_hack = { version = "0.1", path = "../../workspace_hack" } +tokio-postgres.workspace = true +tokio-stream.workspace = true +tokio.workspace = true +futures.workspace = true +tokio-util.workspace = true +anyhow.workspace = true +postgres.workspace = true +bytes.workspace = true diff --git a/storage_controller/client/src/control_api.rs b/storage_controller/client/src/control_api.rs new file mode 100644 index 000000000000..a981b5020e69 --- /dev/null +++ b/storage_controller/client/src/control_api.rs @@ -0,0 +1,62 @@ +use pageserver_client::mgmt_api::{self, ResponseErrorMessageExt}; +use reqwest::{Method, Url}; +use serde::{de::DeserializeOwned, Serialize}; +use std::str::FromStr; + +pub struct Client { + base_url: Url, + jwt_token: Option, + client: reqwest::Client, +} + +impl Client { + pub fn new(base_url: Url, jwt_token: Option) -> Self { + Self { + base_url, + jwt_token, + client: reqwest::ClientBuilder::new() + .build() + .expect("Failed to construct http client"), + } + } + + /// Simple HTTP request wrapper for calling into storage controller + pub async fn dispatch( + &self, + method: Method, + path: String, + body: Option, + ) -> mgmt_api::Result + where + RQ: Serialize + Sized, + RS: DeserializeOwned + Sized, + { + // The configured URL has the /upcall path prefix for pageservers to use: we will strip that out + // for general purpose API access. + let url = Url::from_str(&format!( + "http://{}:{}/{path}", + self.base_url.host_str().unwrap(), + self.base_url.port().unwrap() + )) + .unwrap(); + + let mut builder = self.client.request(method, url); + if let Some(body) = body { + builder = builder.json(&body) + } + if let Some(jwt_token) = &self.jwt_token { + builder = builder.header( + reqwest::header::AUTHORIZATION, + format!("Bearer {jwt_token}"), + ); + } + + let response = builder.send().await.map_err(mgmt_api::Error::ReceiveBody)?; + let response = response.error_from_body().await?; + + response + .json() + .await + .map_err(pageserver_client::mgmt_api::Error::ReceiveBody) + } +} diff --git a/storage_controller/client/src/lib.rs b/storage_controller/client/src/lib.rs new file mode 100644 index 000000000000..6d5e20294271 --- /dev/null +++ b/storage_controller/client/src/lib.rs @@ -0,0 +1 @@ +pub mod control_api; diff --git a/storage_controller/src/http.rs b/storage_controller/src/http.rs index 9ddf98eb3bb6..8fb4be93e001 100644 --- a/storage_controller/src/http.rs +++ b/storage_controller/src/http.rs @@ -430,7 +430,7 @@ async fn handle_tenant_describe( service: Arc, req: Request, ) -> Result, ApiError> { - check_permissions(&req, Scope::Admin)?; + check_permissions(&req, Scope::Scrubber)?; let tenant_id: TenantId = parse_request_param(&req, "tenant_id")?; json_response(StatusCode::OK, service.tenant_describe(tenant_id)?) diff --git a/storage_controller/src/main.rs b/storage_controller/src/main.rs index 4bf6b528f49e..789f96beb397 100644 --- a/storage_controller/src/main.rs +++ b/storage_controller/src/main.rs @@ -5,6 +5,7 @@ use metrics::launch_timestamp::LaunchTimestamp; use metrics::BuildInfo; use std::path::PathBuf; use std::sync::Arc; +use std::time::Duration; use storage_controller::http::make_router; use storage_controller::metrics::preinitialize_metrics; use storage_controller::persistence::Persistence; @@ -310,12 +311,21 @@ async fn async_main() -> anyhow::Result<()> { tracing::info!("Terminating on signal"); // Stop HTTP server first, so that we don't have to service requests - // while shutting down Service + // while shutting down Service. server_shutdown.cancel(); - if let Err(e) = server_task.await { - tracing::error!("Error joining HTTP server task: {e}") + match tokio::time::timeout(Duration::from_secs(5), server_task).await { + Ok(Ok(_)) => { + tracing::info!("Joined HTTP server task"); + } + Ok(Err(e)) => { + tracing::error!("Error joining HTTP server task: {e}") + } + Err(_) => { + tracing::warn!("Timed out joining HTTP server task"); + // We will fall through and shut down the service anyway, any request handlers + // in flight will experience cancellation & their clients will see a torn connection. + } } - tracing::info!("Joined HTTP server task"); service.shutdown().await; tracing::info!("Service shutdown complete"); diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 3c24433c422a..a163453dca40 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -3956,6 +3956,8 @@ impl Service { "failpoint".to_string() ))); + failpoint_support::sleep_millis_async!("shard-split-post-remote-sleep", &self.cancel); + tracing::info!( "Split {} into {}", parent_id, diff --git a/storage_scrubber/Cargo.toml b/storage_scrubber/Cargo.toml index 050be66483b9..5233afbebe47 100644 --- a/storage_scrubber/Cargo.toml +++ b/storage_scrubber/Cargo.toml @@ -34,6 +34,7 @@ camino.workspace = true rustls.workspace = true rustls-native-certs.workspace = true once_cell.workspace = true +storage_controller_client.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } chrono = { workspace = true, default-features = false, features = ["clock", "serde"] } diff --git a/storage_scrubber/src/lib.rs b/storage_scrubber/src/lib.rs index 9102ad9906f2..a0b6d7ea302d 100644 --- a/storage_scrubber/src/lib.rs +++ b/storage_scrubber/src/lib.rs @@ -24,6 +24,7 @@ use camino::{Utf8Path, Utf8PathBuf}; use clap::ValueEnum; use pageserver::tenant::TENANTS_SEGMENT_NAME; use pageserver_api::shard::TenantShardId; +use remote_storage::RemotePath; use reqwest::Url; use serde::{Deserialize, Serialize}; use tokio::io::AsyncReadExt; @@ -31,7 +32,7 @@ use tracing::error; use tracing_appender::non_blocking::WorkerGuard; use tracing_subscriber::{fmt, prelude::*, EnvFilter}; use utils::fs_ext; -use utils::id::{TenantId, TimelineId}; +use utils::id::{TenantId, TenantTimelineId, TimelineId}; const MAX_RETRIES: usize = 20; const CLOUD_ADMIN_API_TOKEN_ENV_VAR: &str = "CLOUD_ADMIN_API_TOKEN"; @@ -54,7 +55,7 @@ pub struct S3Target { /// in the pageserver, as all timeline objects existing in the scope of a particular /// tenant: the scrubber is different in that it handles collections of data referring to many /// TenantShardTimelineIds in on place. -#[derive(Serialize, Deserialize, Debug, Clone, Copy, Hash, PartialEq, Eq)] +#[derive(Serialize, Deserialize, Debug, Clone, Copy, Hash, PartialEq, Eq, PartialOrd, Ord)] pub struct TenantShardTimelineId { tenant_shard_id: TenantShardId, timeline_id: TimelineId, @@ -67,6 +68,10 @@ impl TenantShardTimelineId { timeline_id, } } + + fn as_tenant_timeline_id(&self) -> TenantTimelineId { + TenantTimelineId::new(self.tenant_shard_id.tenant_id, self.timeline_id) + } } impl Display for TenantShardTimelineId { @@ -179,6 +184,22 @@ impl RootTarget { .with_sub_segment(&id.timeline_id.to_string()) } + /// Given RemotePath "tenants/foo/timelines/bar/layerxyz", prefix it to a literal + /// key in the S3 bucket. + pub fn absolute_key(&self, key: &RemotePath) -> String { + let root = match self { + Self::Pageserver(root) => root, + Self::Safekeeper(root) => root, + }; + + let prefix = &root.prefix_in_bucket; + if prefix.ends_with('/') { + format!("{prefix}{key}") + } else { + format!("{prefix}/{key}") + } + } + pub fn bucket_name(&self) -> &str { match self { Self::Pageserver(root) => &root.bucket_name, @@ -216,6 +237,14 @@ impl BucketConfig { } } +pub struct ControllerClientConfig { + /// URL to storage controller. e.g. http://127.0.0.1:1234 when using `neon_local` + pub controller_api: Url, + + /// JWT token for authenticating with storage controller. Requires scope 'scrubber' or 'admin'. + pub controller_jwt: String, +} + pub struct ConsoleConfig { pub token: String, pub base_url: Url, diff --git a/storage_scrubber/src/main.rs b/storage_scrubber/src/main.rs index d81612119263..b3ed6f645177 100644 --- a/storage_scrubber/src/main.rs +++ b/storage_scrubber/src/main.rs @@ -1,11 +1,12 @@ -use anyhow::bail; +use anyhow::{anyhow, bail}; use camino::Utf8PathBuf; use pageserver_api::shard::TenantShardId; -use storage_scrubber::find_large_objects; +use reqwest::Url; use storage_scrubber::garbage::{find_garbage, purge_garbage, PurgeMode}; use storage_scrubber::pageserver_physical_gc::GcMode; use storage_scrubber::scan_pageserver_metadata::scan_metadata; use storage_scrubber::tenant_snapshot::SnapshotDownloader; +use storage_scrubber::{find_large_objects, ControllerClientConfig}; use storage_scrubber::{ init_logging, pageserver_physical_gc::pageserver_physical_gc, scan_safekeeper_metadata::scan_safekeeper_metadata, BucketConfig, ConsoleConfig, NodeKind, @@ -24,6 +25,14 @@ struct Cli { #[arg(short, long, default_value_t = false)] delete: bool, + + #[arg(long)] + /// URL to storage controller. e.g. http://127.0.0.1:1234 when using `neon_local` + controller_api: Option, + + #[arg(long)] + /// JWT token for authenticating with storage controller. Requires scope 'scrubber' or 'admin'. + controller_jwt: Option, } #[derive(Subcommand, Debug)] @@ -204,8 +213,37 @@ async fn main() -> anyhow::Result<()> { min_age, mode, } => { - let summary = - pageserver_physical_gc(bucket_config, tenant_ids, min_age.into(), mode).await?; + let controller_client_conf = cli.controller_api.map(|controller_api| { + ControllerClientConfig { + controller_api, + // Default to no key: this is a convenience when working in a development environment + controller_jwt: cli.controller_jwt.unwrap_or("".to_owned()), + } + }); + + match (&controller_client_conf, mode) { + (Some(_), _) => { + // Any mode may run when controller API is set + } + (None, GcMode::Full) => { + // The part of physical GC where we erase ancestor layers cannot be done safely without + // confirming the most recent complete shard split with the controller. Refuse to run, rather + // than doing it unsafely. + return Err(anyhow!("Full physical GC requires `--controller-api` and `--controller-jwt` to run")); + } + (None, GcMode::DryRun | GcMode::IndicesOnly) => { + // These GcModes do not require the controller to run. + } + } + + let summary = pageserver_physical_gc( + bucket_config, + controller_client_conf, + tenant_ids, + min_age.into(), + mode, + ) + .await?; println!("{}", serde_json::to_string(&summary).unwrap()); Ok(()) } diff --git a/storage_scrubber/src/pageserver_physical_gc.rs b/storage_scrubber/src/pageserver_physical_gc.rs index fb8fbc1635ae..e977fd49f779 100644 --- a/storage_scrubber/src/pageserver_physical_gc.rs +++ b/storage_scrubber/src/pageserver_physical_gc.rs @@ -1,22 +1,50 @@ -use std::time::{Duration, UNIX_EPOCH}; +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; +use std::time::{Duration, SystemTime}; use crate::checks::{list_timeline_blobs, BlobDataParseResult}; use crate::metadata_stream::{stream_tenant_timelines, stream_tenants}; -use crate::{init_remote, BucketConfig, NodeKind, RootTarget, TenantShardTimelineId}; +use crate::{ + init_remote, BucketConfig, ControllerClientConfig, NodeKind, RootTarget, TenantShardTimelineId, +}; use aws_sdk_s3::Client; use futures_util::{StreamExt, TryStreamExt}; -use pageserver::tenant::remote_timeline_client::parse_remote_index_path; +use pageserver::tenant::remote_timeline_client::index::LayerFileMetadata; +use pageserver::tenant::remote_timeline_client::{parse_remote_index_path, remote_layer_path}; +use pageserver::tenant::storage_layer::LayerName; use pageserver::tenant::IndexPart; -use pageserver_api::shard::TenantShardId; +use pageserver_api::controller_api::TenantDescribeResponse; +use pageserver_api::shard::{ShardIndex, TenantShardId}; use remote_storage::RemotePath; +use reqwest::Method; use serde::Serialize; +use storage_controller_client::control_api; use tracing::{info_span, Instrument}; use utils::generation::Generation; +use utils::id::{TenantId, TenantTimelineId}; #[derive(Serialize, Default)] pub struct GcSummary { indices_deleted: usize, remote_storage_errors: usize, + controller_api_errors: usize, + ancestor_layers_deleted: usize, +} + +impl GcSummary { + fn merge(&mut self, other: Self) { + let Self { + indices_deleted, + remote_storage_errors, + ancestor_layers_deleted, + controller_api_errors, + } = other; + + self.indices_deleted += indices_deleted; + self.remote_storage_errors += remote_storage_errors; + self.ancestor_layers_deleted += ancestor_layers_deleted; + self.controller_api_errors += controller_api_errors; + } } #[derive(clap::ValueEnum, Debug, Clone, Copy)] @@ -26,9 +54,9 @@ pub enum GcMode { // Enable only removing old-generation indices IndicesOnly, + // Enable all forms of GC - // TODO: this will be used when shard split ancestor layer deletion is added - // All, + Full, } impl std::fmt::Display for GcMode { @@ -36,8 +64,230 @@ impl std::fmt::Display for GcMode { match self { GcMode::DryRun => write!(f, "dry-run"), GcMode::IndicesOnly => write!(f, "indices-only"), + GcMode::Full => write!(f, "full"), + } + } +} + +mod refs { + use super::*; + // Map of cross-shard layer references, giving a refcount for each layer in each shard that is referenced by some other + // shard in the same tenant. This is sparse! The vast majority of timelines will have no cross-shard refs, and those that + // do have cross shard refs should eventually drop most of them via compaction. + // + // In our inner map type, the TTID in the key is shard-agnostic, and the ShardIndex in the value refers to the _ancestor + // which is is referenced_. + #[derive(Default)] + pub(super) struct AncestorRefs( + BTreeMap>, + ); + + impl AncestorRefs { + /// Insert references for layers discovered in a particular shard-timeline that refer to an ancestral shard-timeline. + pub(super) fn update( + &mut self, + ttid: TenantShardTimelineId, + layers: Vec<(LayerName, LayerFileMetadata)>, + ) { + let ttid_refs = self.0.entry(ttid.as_tenant_timeline_id()).or_default(); + for (layer_name, layer_metadata) in layers { + // Increment refcount of this layer in the ancestor shard + *(ttid_refs + .entry((layer_metadata.shard, layer_name)) + .or_default()) += 1; + } + } + + /// For a particular TTID, return the map of all ancestor layers referenced by a descendent to their refcount + /// + /// The `ShardIndex` in the result's key is the index of the _ancestor_, not the descendent. + pub(super) fn get_ttid_refcounts( + &self, + ttid: &TenantTimelineId, + ) -> Option<&HashMap<(ShardIndex, LayerName), usize>> { + self.0.get(ttid) + } + } +} + +use refs::AncestorRefs; + +// As we see shards for a tenant, acccumulate knowledge needed for cross-shard GC: +// - Are there any ancestor shards? +// - Are there any refs to ancestor shards' layers? +#[derive(Default)] +struct TenantRefAccumulator { + shards_seen: HashMap>, + + // For each shard that has refs to an ancestor's layers, the set of ancestor layers referred to + ancestor_ref_shards: AncestorRefs, +} + +impl TenantRefAccumulator { + fn update(&mut self, ttid: TenantShardTimelineId, index_part: &IndexPart) { + let this_shard_idx = ttid.tenant_shard_id.to_index(); + (*self + .shards_seen + .entry(ttid.tenant_shard_id.tenant_id) + .or_default()) + .push(this_shard_idx); + + let mut ancestor_refs = Vec::new(); + for (layer_name, layer_metadata) in &index_part.layer_metadata { + if layer_metadata.shard != this_shard_idx { + // This is a reference from this shard to a layer in an ancestor shard: we must track this + // as a marker to not GC this layer from the parent. + ancestor_refs.push((layer_name.clone(), layer_metadata.clone())); + } + } + + if !ancestor_refs.is_empty() { + tracing::info!(%ttid, "Found {} ancestor refs", ancestor_refs.len()); + self.ancestor_ref_shards.update(ttid, ancestor_refs); + } + } + + /// Consume Self and return a vector of ancestor tenant shards that should be GC'd, and map of referenced ancestor layers to preserve + async fn into_gc_ancestors( + self, + controller_client: &control_api::Client, + summary: &mut GcSummary, + ) -> (Vec, AncestorRefs) { + let mut ancestors_to_gc = Vec::new(); + for (tenant_id, mut shard_indices) in self.shards_seen { + // Find the highest shard count + let latest_count = shard_indices + .iter() + .map(|i| i.shard_count) + .max() + .expect("Always at least one shard"); + + let (mut latest_shards, ancestor_shards) = { + let at = + itertools::partition(&mut shard_indices, |i| i.shard_count == latest_count); + (shard_indices[0..at].to_owned(), &shard_indices[at..]) + }; + // Sort shards, as we will later compare them with a sorted list from the controller + latest_shards.sort(); + + // Check that we have a complete view of the latest shard count: this should always be the case unless we happened + // to scan the S3 bucket halfway through a shard split. + if latest_shards.len() != latest_count.count() as usize { + // This should be extremely rare, so we warn on it. + tracing::warn!(%tenant_id, "Missed some shards at count {:?}", latest_count); + continue; + } + + // Check if we have any non-latest-count shards + if ancestor_shards.is_empty() { + tracing::debug!(%tenant_id, "No ancestor shards to clean up"); + continue; + } + + // Based on S3 view, this tenant looks like it might have some ancestor shard work to do. We + // must only do this work if the tenant is not currently being split: otherwise, it is not safe + // to GC ancestors, because if the split fails then the controller will try to attach ancestor + // shards again. + match controller_client + .dispatch::<(), TenantDescribeResponse>( + Method::GET, + format!("control/v1/tenant/{tenant_id}"), + None, + ) + .await + { + Err(e) => { + // We were not able to learn the latest shard split state from the controller, so we will not + // do ancestor GC on this tenant. + tracing::warn!(%tenant_id, "Failed to query storage controller, will not do ancestor GC: {e}"); + summary.controller_api_errors += 1; + continue; + } + Ok(desc) => { + // We expect to see that the latest shard count matches the one we saw in S3, and that none + // of the shards indicate splitting in progress. + + let controller_indices: Vec = desc + .shards + .iter() + .map(|s| s.tenant_shard_id.to_index()) + .collect(); + if controller_indices != latest_shards { + tracing::info!(%tenant_id, "Latest shards seen in S3 ({latest_shards:?}) don't match controller state ({controller_indices:?})"); + continue; + } + + if desc.shards.iter().any(|s| s.is_splitting) { + tracing::info!(%tenant_id, "One or more shards is currently splitting"); + continue; + } + + // This shouldn't be too noisy, because we only log this for tenants that have some ancestral refs. + tracing::info!(%tenant_id, "Validated state with controller: {desc:?}"); + } + } + + // GC ancestor shards + for ancestor_shard in ancestor_shards.iter().map(|idx| TenantShardId { + tenant_id, + shard_count: idx.shard_count, + shard_number: idx.shard_number, + }) { + ancestors_to_gc.push(ancestor_shard); + } } + + (ancestors_to_gc, self.ancestor_ref_shards) + } +} + +async fn is_old_enough( + s3_client: &Client, + bucket_config: &BucketConfig, + min_age: &Duration, + key: &str, + summary: &mut GcSummary, +) -> bool { + // Validation: we will only GC indices & layers after a time threshold (e.g. one week) so that during an incident + // it is easier to read old data for analysis, and easier to roll back shard splits without having to un-delete any objects. + let age: Duration = match s3_client + .head_object() + .bucket(&bucket_config.bucket) + .key(key) + .send() + .await + { + Ok(response) => match response.last_modified { + None => { + tracing::warn!("Missing last_modified"); + summary.remote_storage_errors += 1; + return false; + } + Some(last_modified) => match SystemTime::try_from(last_modified).map(|t| t.elapsed()) { + Ok(Ok(e)) => e, + Err(_) | Ok(Err(_)) => { + tracing::warn!("Bad last_modified time: {last_modified:?}"); + return false; + } + }, + }, + Err(e) => { + tracing::warn!("Failed to HEAD {key}: {e}"); + summary.remote_storage_errors += 1; + return false; + } + }; + let old_enough = &age > min_age; + + if !old_enough { + tracing::info!( + "Skipping young object {} < {}", + humantime::format_duration(age), + humantime::format_duration(*min_age) + ); } + + old_enough } async fn maybe_delete_index( @@ -79,45 +329,7 @@ async fn maybe_delete_index( return; } - // Validation: we will only delete indices after one week, so that during incidents we will have - // easy access to recent indices. - let age: Duration = match s3_client - .head_object() - .bucket(&bucket_config.bucket) - .key(key) - .send() - .await - { - Ok(response) => match response.last_modified { - None => { - tracing::warn!("Missing last_modified"); - summary.remote_storage_errors += 1; - return; - } - Some(last_modified) => { - let last_modified = - UNIX_EPOCH + Duration::from_secs_f64(last_modified.as_secs_f64()); - match last_modified.elapsed() { - Ok(e) => e, - Err(_) => { - tracing::warn!("Bad last_modified time: {last_modified:?}"); - return; - } - } - } - }, - Err(e) => { - tracing::warn!("Failed to HEAD {key}: {e}"); - summary.remote_storage_errors += 1; - return; - } - }; - if &age < min_age { - tracing::info!( - "Skipping young object {} < {}", - age.as_secs_f64(), - min_age.as_secs_f64() - ); + if !is_old_enough(s3_client, bucket_config, min_age, key, summary).await { return; } @@ -145,6 +357,108 @@ async fn maybe_delete_index( } } +#[allow(clippy::too_many_arguments)] +async fn gc_ancestor( + s3_client: &Client, + bucket_config: &BucketConfig, + root_target: &RootTarget, + min_age: &Duration, + ancestor: TenantShardId, + refs: &AncestorRefs, + mode: GcMode, + summary: &mut GcSummary, +) -> anyhow::Result<()> { + // Scan timelines in the ancestor + let timelines = stream_tenant_timelines(s3_client, root_target, ancestor).await?; + let mut timelines = std::pin::pin!(timelines); + + // Build a list of keys to retain + + while let Some(ttid) = timelines.next().await { + let ttid = ttid?; + + let data = list_timeline_blobs(s3_client, ttid, root_target).await?; + + let s3_layers = match data.blob_data { + BlobDataParseResult::Parsed { + index_part: _, + index_part_generation: _, + s3_layers, + } => s3_layers, + BlobDataParseResult::Relic => { + // Post-deletion tenant location: don't try and GC it. + continue; + } + BlobDataParseResult::Incorrect(reasons) => { + // Our primary purpose isn't to report on bad data, but log this rather than skipping silently + tracing::warn!( + "Skipping ancestor GC for timeline {ttid}, bad metadata: {reasons:?}" + ); + continue; + } + }; + + let ttid_refs = refs.get_ttid_refcounts(&ttid.as_tenant_timeline_id()); + let ancestor_shard_index = ttid.tenant_shard_id.to_index(); + + for (layer_name, layer_gen) in s3_layers { + let ref_count = ttid_refs + .and_then(|m| m.get(&(ancestor_shard_index, layer_name.clone()))) + .copied() + .unwrap_or(0); + + if ref_count > 0 { + tracing::debug!(%ttid, "Ancestor layer {layer_name} has {ref_count} refs"); + continue; + } + + tracing::info!(%ttid, "Ancestor layer {layer_name} is not referenced"); + + // Build the key for the layer we are considering deleting + let key = root_target.absolute_key(&remote_layer_path( + &ttid.tenant_shard_id.tenant_id, + &ttid.timeline_id, + ancestor_shard_index, + &layer_name, + layer_gen, + )); + + // We apply a time threshold to GCing objects that are un-referenced: this preserves our ability + // to roll back a shard split if we have to, by avoiding deleting ancestor layers right away + if !is_old_enough(s3_client, bucket_config, min_age, &key, summary).await { + continue; + } + + if !matches!(mode, GcMode::Full) { + tracing::info!("Dry run: would delete key {key}"); + continue; + } + + // All validations passed: erase the object + match s3_client + .delete_object() + .bucket(&bucket_config.bucket) + .key(&key) + .send() + .await + { + Ok(_) => { + tracing::info!("Successfully deleted unreferenced ancestor layer {key}"); + summary.ancestor_layers_deleted += 1; + } + Err(e) => { + tracing::warn!("Failed to delete layer {key}: {e}"); + summary.remote_storage_errors += 1; + } + } + } + + // TODO: if all the layers are gone, clean up the whole timeline dir (remove index) + } + + Ok(()) +} + /// Physical garbage collection: removing unused S3 objects. This is distinct from the garbage collection /// done inside the pageserver, which operates at a higher level (keys, layers). This type of garbage collection /// is about removing: @@ -156,22 +470,26 @@ async fn maybe_delete_index( /// make sure that object listings don't get slowed down by large numbers of garbage objects. pub async fn pageserver_physical_gc( bucket_config: BucketConfig, - tenant_ids: Vec, + controller_client_conf: Option, + tenant_shard_ids: Vec, min_age: Duration, mode: GcMode, ) -> anyhow::Result { let (s3_client, target) = init_remote(bucket_config.clone(), NodeKind::Pageserver).await?; - let tenants = if tenant_ids.is_empty() { + let tenants = if tenant_shard_ids.is_empty() { futures::future::Either::Left(stream_tenants(&s3_client, &target)) } else { - futures::future::Either::Right(futures::stream::iter(tenant_ids.into_iter().map(Ok))) + futures::future::Either::Right(futures::stream::iter(tenant_shard_ids.into_iter().map(Ok))) }; // How many tenants to process in parallel. We need to be mindful of pageservers // accessing the same per tenant prefixes, so use a lower setting than pageservers. const CONCURRENCY: usize = 32; + // Accumulate information about each tenant for cross-shard GC step we'll do at the end + let accumulator = Arc::new(std::sync::Mutex::new(TenantRefAccumulator::default())); + // Generate a stream of TenantTimelineId let timelines = tenants.map_ok(|t| stream_tenant_timelines(&s3_client, &target, t)); let timelines = timelines.try_buffered(CONCURRENCY); @@ -185,16 +503,17 @@ pub async fn pageserver_physical_gc( target: &RootTarget, mode: GcMode, ttid: TenantShardTimelineId, + accumulator: &Arc>, ) -> anyhow::Result { let mut summary = GcSummary::default(); let data = list_timeline_blobs(s3_client, ttid, target).await?; - let (latest_gen, candidates) = match &data.blob_data { + let (index_part, latest_gen, candidates) = match &data.blob_data { BlobDataParseResult::Parsed { - index_part: _index_part, + index_part, index_part_generation, s3_layers: _s3_layers, - } => (*index_part_generation, data.unused_index_keys), + } => (index_part, *index_part_generation, data.unused_index_keys), BlobDataParseResult::Relic => { // Post-deletion tenant location: don't try and GC it. return Ok(summary); @@ -206,6 +525,8 @@ pub async fn pageserver_physical_gc( } }; + accumulator.lock().unwrap().update(ttid, index_part); + for key in candidates { maybe_delete_index( s3_client, @@ -222,17 +543,61 @@ pub async fn pageserver_physical_gc( Ok(summary) } - let timelines = timelines - .map_ok(|ttid| gc_timeline(&s3_client, &bucket_config, &min_age, &target, mode, ttid)); - let mut timelines = std::pin::pin!(timelines.try_buffered(CONCURRENCY)); let mut summary = GcSummary::default(); - while let Some(i) = timelines.next().await { - let tl_summary = i?; + // Drain futures for per-shard GC, populating accumulator as a side effect + { + let timelines = timelines.map_ok(|ttid| { + gc_timeline( + &s3_client, + &bucket_config, + &min_age, + &target, + mode, + ttid, + &accumulator, + ) + }); + let mut timelines = std::pin::pin!(timelines.try_buffered(CONCURRENCY)); + + while let Some(i) = timelines.next().await { + summary.merge(i?); + } + } + + // Execute cross-shard GC, using the accumulator's full view of all the shards built in the per-shard GC + let Some(controller_client) = controller_client_conf.as_ref().map(|c| { + let ControllerClientConfig { + controller_api, + controller_jwt, + } = c; + control_api::Client::new(controller_api.clone(), Some(controller_jwt.clone())) + }) else { + tracing::info!("Skipping ancestor layer GC, because no `--controller-api` was specified"); + return Ok(summary); + }; + + let (ancestor_shards, ancestor_refs) = Arc::into_inner(accumulator) + .unwrap() + .into_inner() + .unwrap() + .into_gc_ancestors(&controller_client, &mut summary) + .await; - summary.indices_deleted += tl_summary.indices_deleted; - summary.remote_storage_errors += tl_summary.remote_storage_errors; + for ancestor_shard in ancestor_shards { + gc_ancestor( + &s3_client, + &bucket_config, + &target, + &min_age, + ancestor_shard, + &ancestor_refs, + mode, + &mut summary, + ) + .instrument(info_span!("gc_ancestor", %ancestor_shard)) + .await?; } Ok(summary) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index db7269ad4148..9e39457c066f 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -997,7 +997,7 @@ def __exit__( if self.scrub_on_exit: try: - StorageScrubber(self).scan_metadata() + self.env.storage_scrubber.scan_metadata() except Exception as e: log.error(f"Error during remote storage scrub: {e}") cleanup_error = e @@ -1225,6 +1225,9 @@ def __init__(self, config: NeonEnvBuilder): ) cfg["safekeepers"].append(sk_cfg) + # Scrubber instance for tests that use it, and for use during teardown checks + self.storage_scrubber = StorageScrubber(self, log_dir=config.test_output_dir) + log.info(f"Config: {cfg}") self.neon_cli.init( cfg, @@ -4265,9 +4268,9 @@ def paused(): class StorageScrubber: - def __init__(self, env: NeonEnvBuilder, log_dir: Optional[Path] = None): + def __init__(self, env: NeonEnv, log_dir: Path): self.env = env - self.log_dir = log_dir or env.test_output_dir + self.log_dir = log_dir def scrubber_cli(self, args: list[str], timeout) -> str: assert isinstance(self.env.pageserver_remote_storage, S3Storage) @@ -4284,11 +4287,14 @@ def scrubber_cli(self, args: list[str], timeout) -> str: if s3_storage.endpoint is not None: env.update({"AWS_ENDPOINT_URL": s3_storage.endpoint}) - base_args = [str(self.env.neon_binpath / "storage_scrubber")] + base_args = [ + str(self.env.neon_binpath / "storage_scrubber"), + f"--controller-api={self.env.storage_controller_api}", + ] args = base_args + args (output_path, stdout, status_code) = subprocess_capture( - self.env.test_output_dir, + self.log_dir, args, echo_stderr=True, echo_stdout=True, @@ -4327,7 +4333,10 @@ def tenant_snapshot(self, tenant_id: TenantId, output_path: Path): log.info(f"tenant-snapshot output: {stdout}") def pageserver_physical_gc( - self, min_age_secs: int, tenant_ids: Optional[list[TenantId]] = None + self, + min_age_secs: int, + tenant_ids: Optional[list[TenantId]] = None, + mode: Optional[str] = None, ): args = ["pageserver-physical-gc", "--min-age", f"{min_age_secs}s"] @@ -4337,6 +4346,9 @@ def pageserver_physical_gc( for tenant_id in tenant_ids: args.extend(["--tenant-id", str(tenant_id)]) + if mode is not None: + args.extend(["--mode", mode]) + stdout = self.scrubber_cli( args, timeout=30, diff --git a/test_runner/regress/test_pageserver_generations.py b/test_runner/regress/test_pageserver_generations.py index 7ce38c5c3c82..041942cda33a 100644 --- a/test_runner/regress/test_pageserver_generations.py +++ b/test_runner/regress/test_pageserver_generations.py @@ -22,7 +22,6 @@ NeonEnv, NeonEnvBuilder, PgBin, - StorageScrubber, generate_uploads_and_deletions, ) from fixtures.pageserver.common_types import parse_layer_file_name @@ -215,7 +214,7 @@ def parse_generation_suffix(key): # Having written a mixture of generation-aware and legacy index_part.json, # ensure the scrubber handles the situation as expected. - metadata_summary = StorageScrubber(neon_env_builder).scan_metadata() + metadata_summary = env.storage_scrubber.scan_metadata() assert metadata_summary["tenant_count"] == 1 # Scrubber should have seen our timeline assert metadata_summary["timeline_count"] == 1 assert metadata_summary["timeline_shard_count"] == 1 diff --git a/test_runner/regress/test_pageserver_secondary.py b/test_runner/regress/test_pageserver_secondary.py index 0416078ebc67..58d61eab0de5 100644 --- a/test_runner/regress/test_pageserver_secondary.py +++ b/test_runner/regress/test_pageserver_secondary.py @@ -7,7 +7,7 @@ import pytest from fixtures.common_types import TenantId, TimelineId from fixtures.log_helper import log -from fixtures.neon_fixtures import NeonEnvBuilder, NeonPageserver, StorageScrubber +from fixtures.neon_fixtures import NeonEnvBuilder, NeonPageserver from fixtures.pageserver.common_types import parse_layer_file_name from fixtures.pageserver.utils import ( assert_prefix_empty, @@ -234,7 +234,7 @@ def ignore_notify(request: Request): # Having done a bunch of attach/detach cycles, we will have generated some index garbage: check # that the scrubber sees it and cleans it up. We do this before the final attach+validate pass, # to also validate that the scrubber isn't breaking anything. - gc_summary = StorageScrubber(neon_env_builder).pageserver_physical_gc(min_age_secs=1) + gc_summary = env.storage_scrubber.pageserver_physical_gc(min_age_secs=1) assert gc_summary["remote_storage_errors"] == 0 assert gc_summary["indices_deleted"] > 0 @@ -555,7 +555,7 @@ def test_secondary_downloads(neon_env_builder: NeonEnvBuilder): # Scrub the remote storage # ======================== # This confirms that the scrubber isn't upset by the presence of the heatmap - StorageScrubber(neon_env_builder).scan_metadata() + env.storage_scrubber.scan_metadata() # Detach secondary and delete tenant # =================================== diff --git a/test_runner/regress/test_sharding.py b/test_runner/regress/test_sharding.py index 4471237900b8..90c6e26d012f 100644 --- a/test_runner/regress/test_sharding.py +++ b/test_runner/regress/test_sharding.py @@ -12,7 +12,6 @@ NeonEnv, NeonEnvBuilder, StorageControllerApiException, - StorageScrubber, last_flush_lsn_upload, tenant_get_shards, wait_for_last_flush_lsn, @@ -128,7 +127,7 @@ def get_sizes(): # Check the scrubber isn't confused by sharded content, then disable # it during teardown because we'll have deleted by then - StorageScrubber(neon_env_builder).scan_metadata() + env.storage_scrubber.scan_metadata() neon_env_builder.scrub_on_exit = False env.storage_controller.pageserver_api().tenant_delete(tenant_id) diff --git a/test_runner/regress/test_storage_scrubber.py b/test_runner/regress/test_storage_scrubber.py index 35ae61c380df..635690fc7fba 100644 --- a/test_runner/regress/test_storage_scrubber.py +++ b/test_runner/regress/test_storage_scrubber.py @@ -1,14 +1,19 @@ import os import shutil +import threading +import time +from concurrent.futures import ThreadPoolExecutor from typing import Optional import pytest from fixtures.common_types import TenantId, TenantShardId, TimelineId +from fixtures.log_helper import log from fixtures.neon_fixtures import ( + NeonEnv, NeonEnvBuilder, - StorageScrubber, ) from fixtures.remote_storage import S3Storage, s3_storage +from fixtures.utils import wait_until from fixtures.workload import Workload @@ -60,8 +65,7 @@ def test_scrubber_tenant_snapshot(neon_env_builder: NeonEnvBuilder, shard_count: output_path = neon_env_builder.test_output_dir / "snapshot" os.makedirs(output_path) - scrubber = StorageScrubber(neon_env_builder) - scrubber.tenant_snapshot(tenant_id, output_path) + env.storage_scrubber.tenant_snapshot(tenant_id, output_path) assert len(os.listdir(output_path)) > 0 @@ -111,6 +115,14 @@ def test_scrubber_tenant_snapshot(neon_env_builder: NeonEnvBuilder, shard_count: workload.validate() +def drop_local_state(env: NeonEnv, tenant_id: TenantId): + env.storage_controller.tenant_policy_update(tenant_id, {"placement": "Detached"}) + env.storage_controller.reconcile_until_idle() + + env.storage_controller.tenant_policy_update(tenant_id, {"placement": {"Attached": 0}}) + env.storage_controller.reconcile_until_idle() + + @pytest.mark.parametrize("shard_count", [None, 4]) def test_scrubber_physical_gc(neon_env_builder: NeonEnvBuilder, shard_count: Optional[int]): neon_env_builder.enable_pageserver_remote_storage(s3_storage()) @@ -133,28 +145,231 @@ def test_scrubber_physical_gc(neon_env_builder: NeonEnvBuilder, shard_count: Opt # For each cycle, detach and attach the tenant to bump the generation, and do some writes to generate uploads for _i in range(0, n_cycles): - env.storage_controller.tenant_policy_update(tenant_id, {"placement": "Detached"}) - env.storage_controller.reconcile_until_idle() - - env.storage_controller.tenant_policy_update(tenant_id, {"placement": {"Attached": 0}}) - env.storage_controller.reconcile_until_idle() + drop_local_state(env, tenant_id) # This write includes remote upload, will generate an index in this generation workload.write_rows(1) # With a high min_age, the scrubber should decline to delete anything - gc_summary = StorageScrubber(neon_env_builder).pageserver_physical_gc(min_age_secs=3600) + gc_summary = env.storage_scrubber.pageserver_physical_gc(min_age_secs=3600) assert gc_summary["remote_storage_errors"] == 0 assert gc_summary["indices_deleted"] == 0 # If targeting a different tenant, the scrubber shouldn't do anything - gc_summary = StorageScrubber(neon_env_builder).pageserver_physical_gc( + gc_summary = env.storage_scrubber.pageserver_physical_gc( min_age_secs=1, tenant_ids=[TenantId.generate()] ) assert gc_summary["remote_storage_errors"] == 0 assert gc_summary["indices_deleted"] == 0 # With a low min_age, the scrubber should go ahead and clean up all but the latest 2 generations - gc_summary = StorageScrubber(neon_env_builder).pageserver_physical_gc(min_age_secs=1) + gc_summary = env.storage_scrubber.pageserver_physical_gc(min_age_secs=1) assert gc_summary["remote_storage_errors"] == 0 assert gc_summary["indices_deleted"] == (expect_indices_per_shard - 2) * shard_count + + +@pytest.mark.parametrize("shard_count", [None, 2]) +def test_scrubber_physical_gc_ancestors( + neon_env_builder: NeonEnvBuilder, shard_count: Optional[int] +): + neon_env_builder.enable_pageserver_remote_storage(s3_storage()) + neon_env_builder.num_pageservers = 2 + + env = neon_env_builder.init_configs() + env.start() + + tenant_id = TenantId.generate() + timeline_id = TimelineId.generate() + env.neon_cli.create_tenant( + tenant_id, + timeline_id, + shard_count=shard_count, + conf={ + # Small layers and low compaction thresholds, so that when we split we can expect some to + # be dropped by child shards + "checkpoint_distance": f"{1024 * 1024}", + "compaction_threshold": "1", + "compaction_target_size": f"{1024 * 1024}", + "image_creation_threshold": "2", + "image_layer_creation_check_threshold": "0", + # Disable background compaction, we will do it explicitly + "compaction_period": "0s", + # No PITR, so that as soon as child shards generate an image layer, it covers ancestor deltas + # and makes them GC'able + "pitr_interval": "0s", + }, + ) + + # Make sure the original shard has some layers + workload = Workload(env, tenant_id, timeline_id) + workload.init() + workload.write_rows(100) + + new_shard_count = 4 + assert shard_count is None or new_shard_count > shard_count + shards = env.storage_controller.tenant_shard_split(tenant_id, shard_count=new_shard_count) + + # Make sure child shards have some layers + workload.write_rows(100) + + # Flush deletion queue so that we don't leave any orphan layers in the parent that will confuse subsequent checks: once + # a shard is split, any layers in its prefix that aren't referenced by a child will be considered GC'able, even + # if they were logically deleted before the shard split, just not physically deleted yet because of the queue. + for ps in env.pageservers: + ps.http_client().deletion_queue_flush(execute=True) + + # Before compacting, all the layers in the ancestor should still be referenced by the children: the scrubber + # should not erase any ancestor layers + gc_summary = env.storage_scrubber.pageserver_physical_gc(min_age_secs=1, mode="full") + assert gc_summary["remote_storage_errors"] == 0 + assert gc_summary["indices_deleted"] == 0 + assert gc_summary["ancestor_layers_deleted"] == 0 + + # Write some data and compact: compacting, some ancestor layers should no longer be needed by children + # (the compaction is part of the checkpoint that Workload does for us) + workload.churn_rows(100) + workload.churn_rows(100) + workload.churn_rows(100) + for shard in shards: + ps = env.get_tenant_pageserver(shard) + ps.http_client().timeline_compact(shard, timeline_id) + ps.http_client().timeline_gc(shard, timeline_id, 0) + + # We will use a min_age_secs=1 threshold for deletion, let it pass + time.sleep(2) + + # Our time threshold should be respected: check that with a high threshold we delete nothing + gc_summary = env.storage_scrubber.pageserver_physical_gc(min_age_secs=3600, mode="full") + assert gc_summary["remote_storage_errors"] == 0 + assert gc_summary["indices_deleted"] == 0 + assert gc_summary["ancestor_layers_deleted"] == 0 + + # Now run with a low time threshold: deletions of ancestor layers should be executed + gc_summary = env.storage_scrubber.pageserver_physical_gc(min_age_secs=1, mode="full") + assert gc_summary["remote_storage_errors"] == 0 + assert gc_summary["indices_deleted"] == 0 + assert gc_summary["ancestor_layers_deleted"] > 0 + + # We deleted some layers: now check we didn't corrupt the tenant by doing so. Detach and + # attach it, to drop any local state, then check it's still readable. + workload.stop() + drop_local_state(env, tenant_id) + + workload.validate() + + +def test_scrubber_physical_gc_ancestors_split(neon_env_builder: NeonEnvBuilder): + """ + Exercise ancestor GC while a tenant is partly split: this test ensures that if we have some child shards + which don't reference an ancestor, but some child shards that don't exist yet, then we do not incorrectly + GC any ancestor layers. + """ + neon_env_builder.enable_pageserver_remote_storage(s3_storage()) + neon_env_builder.num_pageservers = 2 + + env = neon_env_builder.init_configs() + env.start() + + tenant_id = TenantId.generate() + timeline_id = TimelineId.generate() + initial_shard_count = 2 + env.neon_cli.create_tenant( + tenant_id, + timeline_id, + shard_count=initial_shard_count, + conf={ + # Small layers and low compaction thresholds, so that when we split we can expect some to + # be dropped by child shards + "checkpoint_distance": f"{1024 * 1024}", + "compaction_threshold": "1", + "compaction_target_size": f"{1024 * 1024}", + "image_creation_threshold": "2", + "image_layer_creation_check_threshold": "0", + # Disable background compaction, we will do it explicitly + "compaction_period": "0s", + # No PITR, so that as soon as child shards generate an image layer, it covers ancestor deltas + # and makes them GC'able + "pitr_interval": "0s", + }, + ) + + unstuck = threading.Event() + + def stuck_split(): + # Pause our shard split after the first shard but before the second, such that when we run + # the scrub, the S3 bucket contains shards 0002, 0101, 0004, 0204 (but not 0104, 0304). + env.storage_controller.configure_failpoints( + ("shard-split-post-remote-sleep", "return(3600000)") + ) + try: + split_response = env.storage_controller.tenant_shard_split(tenant_id, shard_count=4) + except Exception as e: + log.info(f"Split failed with {e}") + else: + if not unstuck.is_set(): + raise RuntimeError(f"Split succeeded unexpectedly ({split_response})") + + with ThreadPoolExecutor(max_workers=1) as threads: + log.info("Starting hung shard split") + stuck_split_fut = threads.submit(stuck_split) + + # Let the controller reach the failpoint + wait_until( + 10, + 1, + lambda: env.storage_controller.assert_log_contains( + 'failpoint "shard-split-post-remote-sleep": sleeping' + ), + ) + + # Run compaction on the new child shards, so that they drop some refs to their parent + child_shards = [ + TenantShardId(tenant_id, 0, 4), + TenantShardId(tenant_id, 2, 4), + ] + log.info("Compacting first two children") + for child in child_shards: + env.get_tenant_pageserver( + TenantShardId(tenant_id, 0, initial_shard_count) + ).http_client().timeline_compact(child, timeline_id) + + # Check that the other child shards weren't created + assert env.get_tenant_pageserver(TenantShardId(tenant_id, 1, 4)) is None + assert env.get_tenant_pageserver(TenantShardId(tenant_id, 3, 4)) is None + + # Run scrubber: it should not incorrectly interpret the **04 shards' lack of refs to all + # ancestor layers as a reason to GC them, because it should realize that a split is in progress. + # (GC requires that controller does not indicate split in progress, and that if we see the highest + # shard count N, then there are N shards present with that shard count). + gc_output = env.storage_scrubber.pageserver_physical_gc(min_age_secs=0, mode="full") + log.info(f"Ran physical GC partway through split: {gc_output}") + assert gc_output["ancestor_layers_deleted"] == 0 + assert gc_output["remote_storage_errors"] == 0 + assert gc_output["controller_api_errors"] == 0 + + # Storage controller shutdown lets our split request client complete + log.info("Stopping storage controller") + unstuck.set() + env.storage_controller.allowed_errors.append(".*Timed out joining HTTP server task.*") + env.storage_controller.stop() + stuck_split_fut.result() + + # Restart the controller and retry the split with the failpoint disabled, this should + # complete successfully and result in an S3 state that allows the scrubber to proceed with removing ancestor layers + log.info("Starting & retrying split") + env.storage_controller.start() + env.storage_controller.tenant_shard_split(tenant_id, shard_count=4) + + # The other child shards exist now, we can compact them to drop refs to ancestor + log.info("Compacting second two children") + for child in [ + TenantShardId(tenant_id, 1, 4), + TenantShardId(tenant_id, 3, 4), + ]: + env.get_tenant_pageserver(child).http_client().timeline_compact(child, timeline_id) + + gc_output = env.storage_scrubber.pageserver_physical_gc(min_age_secs=0, mode="full") + log.info(f"Ran physical GC after split completed: {gc_output}") + assert gc_output["ancestor_layers_deleted"] > 0 + assert gc_output["remote_storage_errors"] == 0 + assert gc_output["controller_api_errors"] == 0 diff --git a/test_runner/regress/test_tenant_delete.py b/test_runner/regress/test_tenant_delete.py index 1d7c8b8e31f0..6d20b3d0de20 100644 --- a/test_runner/regress/test_tenant_delete.py +++ b/test_runner/regress/test_tenant_delete.py @@ -5,7 +5,6 @@ from fixtures.neon_fixtures import ( NeonEnvBuilder, PgBin, - StorageScrubber, wait_for_last_flush_lsn, ) from fixtures.pageserver.http import PageserverApiException @@ -325,7 +324,6 @@ def test_tenant_delete_scrubber(pg_bin: PgBin, neon_env_builder: NeonEnvBuilder) remote_storage_kind = RemoteStorageKind.MOCK_S3 neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) - scrubber = StorageScrubber(neon_env_builder) env = neon_env_builder.init_start(initial_tenant_conf=MANY_SMALL_LAYERS_TENANT_CONFIG) ps_http = env.pageserver.http_client() @@ -340,7 +338,7 @@ def test_tenant_delete_scrubber(pg_bin: PgBin, neon_env_builder: NeonEnvBuilder) wait_for_upload(ps_http, tenant_id, timeline_id, last_flush_lsn) env.stop() - result = scrubber.scan_metadata() + result = env.storage_scrubber.scan_metadata() assert result["with_warnings"] == [] env.start() @@ -348,5 +346,5 @@ def test_tenant_delete_scrubber(pg_bin: PgBin, neon_env_builder: NeonEnvBuilder) ps_http.tenant_delete(tenant_id) env.stop() - scrubber.scan_metadata() + env.storage_scrubber.scan_metadata() assert result["with_warnings"] == [] From 39aeb10cfc453172cd189c7a43877194ab0dc4a8 Mon Sep 17 00:00:00 2001 From: Shinya Kato <37682778+shinyaaa@users.noreply.github.com> Date: Sat, 20 Jul 2024 02:10:19 +0900 Subject: [PATCH 145/194] safekeeper: remove unused safekeeper runtimes (#8433) There are unused safekeeper runtimes `WAL_REMOVER_RUNTIME` and `METRICS_SHIFTER_RUNTIME`. `WAL_REMOVER_RUNTIME` was implemented in [#4119](https://github.com/neondatabase/neon/pull/4119) and removed in [#7887](https://github.com/neondatabase/neon/pull/7887). `METRICS_SHIFTER_RUNTIME` was also implemented in [#4119](https://github.com/neondatabase/neon/pull/4119) but has never been used. I removed unused safekeeper runtimes `WAL_REMOVER_RUNTIME` and `METRICS_SHIFTER_RUNTIME`. --- safekeeper/src/lib.rs | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/safekeeper/src/lib.rs b/safekeeper/src/lib.rs index af83feb77fac..8f2920ada39e 100644 --- a/safekeeper/src/lib.rs +++ b/safekeeper/src/lib.rs @@ -173,15 +173,6 @@ pub static BROKER_RUNTIME: Lazy = Lazy::new(|| { .expect("Failed to create broker runtime") }); -pub static WAL_REMOVER_RUNTIME: Lazy = Lazy::new(|| { - tokio::runtime::Builder::new_multi_thread() - .thread_name("WAL remover") - .worker_threads(1) - .enable_all() - .build() - .expect("Failed to create broker runtime") -}); - pub static WAL_BACKUP_RUNTIME: Lazy = Lazy::new(|| { tokio::runtime::Builder::new_multi_thread() .thread_name("WAL backup worker") @@ -189,12 +180,3 @@ pub static WAL_BACKUP_RUNTIME: Lazy = Lazy::new(|| { .build() .expect("Failed to create WAL backup runtime") }); - -pub static METRICS_SHIFTER_RUNTIME: Lazy = Lazy::new(|| { - tokio::runtime::Builder::new_multi_thread() - .thread_name("metric shifter") - .worker_threads(1) - .enable_all() - .build() - .expect("Failed to create broker runtime") -}); From a4fa250c9226ede43c7183345c19815ae6f6b61c Mon Sep 17 00:00:00 2001 From: John Spray Date: Fri, 19 Jul 2024 18:30:28 +0100 Subject: [PATCH 146/194] tests: longer timeouts in test_timeline_deletion_with_files_stuck_in_upload_queue (#8438) ## Problem This test had two locations with 2 second timeouts, which is rather low when we run on a highly contended test machine running lots of tests in parallel. It usually passes, but today I've seen both of these locations time out on separate PRs. Example failure: https://neon-github-public-dev.s3.amazonaws.com/reports/pr-8432/10007868041/index.html#suites/837740b64a53e769572c4ed7b7a7eeeb/6c6a092be083d27c ## Summary of changes - Change 2 second timeouts to 20 second timeouts --- test_runner/regress/test_remote_storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_runner/regress/test_remote_storage.py b/test_runner/regress/test_remote_storage.py index fac7fe9deef6..09f941f582a6 100644 --- a/test_runner/regress/test_remote_storage.py +++ b/test_runner/regress/test_remote_storage.py @@ -577,7 +577,7 @@ def assert_compacted_and_uploads_queued(): > 0 ) - wait_until(20, 0.1, assert_compacted_and_uploads_queued) + wait_until(200, 0.1, assert_compacted_and_uploads_queued) # Regardless, give checkpoint some time to block for good. # Not strictly necessary, but might help uncover failure modes in the future. @@ -619,7 +619,7 @@ def assert_compacted_and_uploads_queued(): ) # timeline deletion should be unblocking checkpoint ops - checkpoint_thread.join(2.0) + checkpoint_thread.join(20.0) assert not checkpoint_thread.is_alive() # Just to be sure, unblock ongoing uploads. If the previous assert was incorrect, or the prometheus metric broken, From 3fbb84d74115dadf0ba68ccf98da777c70d97400 Mon Sep 17 00:00:00 2001 From: Shinya Kato <37682778+shinyaaa@users.noreply.github.com> Date: Sat, 20 Jul 2024 03:20:57 +0900 Subject: [PATCH 147/194] Fix openapi specification (#8273) ## Problem There are some swagger errors in `pageserver/src/http/openapi_spec.yml` ``` Error 431 15000 Object includes not allowed fields Error 569 3100401 should always have a 'required' Error 569 15000 Object includes not allowed fields Error 1111 10037 properties members must be schemas ``` ## Summary of changes Fixed the above errors. --- pageserver/src/http/openapi_spec.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index ae109ec1e75f..4d243ddeb995 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -377,7 +377,7 @@ paths: schema: $ref: "#/components/schemas/ConflictError" - /v1/tenant/{tenant_id}/{timeline_id}/preserve_initdb_archive: + /v1/tenant/{tenant_id}/timeline/{timeline_id}/preserve_initdb_archive: parameters: - name: tenant_id in: path @@ -429,7 +429,9 @@ paths: schema: $ref: "#/components/schemas/SyntheticSizeResponse" text/html: - description: SVG representation of the tenant and it's timelines. + schema: + type: string + description: SVG representation of the tenant and its timelines. "401": description: Unauthorized Error content: @@ -568,7 +570,7 @@ paths: type: string - name: timeline_id in: path - Å•equired: true + required: true schema: type: string @@ -774,15 +776,13 @@ components: TenantCreateRequest: allOf: - $ref: '#/components/schemas/TenantConfig' + - $ref: '#/components/schemas/TenantLoadRequest' - type: object required: - new_tenant_id properties: new_tenant_id: type: string - generation: - type: integer - description: Attachment generation number. TenantLoadRequest: type: object properties: @@ -1106,7 +1106,7 @@ components: reparented_timelines: type: array description: Set of reparented timeline ids - properties: + items: type: string format: hex description: TimelineId From 3d582b212a8003d599f3fa2ce5d13670a3cb70e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Fri, 19 Jul 2024 21:01:59 +0200 Subject: [PATCH 148/194] Add archival_config endpoint to pageserver (#8414) This adds an archival_config endpoint to the pageserver. Currently it has no effect, and always "works", but later the intent is that it will make a timeline archived/unarchived. - [x] add yml spec - [x] add endpoint handler Part of https://github.com/neondatabase/neon/issues/8088 --- libs/pageserver_api/src/models.rs | 11 ++++++ pageserver/src/http/openapi_spec.yml | 54 ++++++++++++++++++++++++++++ pageserver/src/http/routes.rs | 44 +++++++++++++++++++++-- pageserver/src/tenant.rs | 9 +++++ 4 files changed, 115 insertions(+), 3 deletions(-) diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index 6abdcb88d0fb..231a604b475b 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -651,6 +651,17 @@ pub struct TenantDetails { pub timelines: Vec, } +#[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Copy, Debug)] +pub enum TimelineArchivalState { + Archived, + Unarchived, +} + +#[derive(Serialize, Deserialize, PartialEq, Eq, Clone)] +pub struct TimelineArchivalConfigRequest { + pub state: TimelineArchivalState, +} + /// This represents the output of the "timeline_detail" and "timeline_list" API calls. #[derive(Debug, Serialize, Deserialize, Clone)] pub struct TimelineInfo { diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index 4d243ddeb995..087d281a0c7b 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -397,6 +397,51 @@ paths: "202": description: Tenant scheduled to load successfully + /v1/tenant/{tenant_shard_id}/timeline/{timeline_id}/archival_config: + parameters: + - name: tenant_shard_id + in: path + required: true + schema: + type: string + - name: timeline_id + in: path + required: true + schema: + type: string + put: + description: | + Either archives or unarchives the given timeline. + An archived timeline may not have any non-archived children. + requestBody: + required: false + content: + application/json: + schema: + $ref: "#/components/schemas/ArchivalConfigRequest" + responses: + "200": + description: Timeline (un)archived successfully + "409": + description: | + The tenant/timeline is already being modified, perhaps by a concurrent call to this API + content: + application/json: + schema: + $ref: "#/components/schemas/ConflictError" + "500": + description: Generic operation error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "503": + description: Temporarily unavailable, please retry. + content: + application/json: + schema: + $ref: "#/components/schemas/ServiceUnavailableError" + /v1/tenant/{tenant_id}/synthetic_size: parameters: - name: tenant_id @@ -846,6 +891,15 @@ components: warm: type: boolean description: Whether to poll remote storage for layers to download. If false, secondary locations don't download anything. + ArchivalConfigRequest: + type: object + required + - state + properties: + state: + description: The archival state of a timeline + type: string + enum: ["Archived", "Unarchived"] TenantConfig: type: object properties: diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index d7ef70477f45..b8063eb5a26b 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -18,14 +18,17 @@ use hyper::StatusCode; use hyper::{Body, Request, Response, Uri}; use metrics::launch_timestamp::LaunchTimestamp; use pageserver_api::models::AuxFilePolicy; +use pageserver_api::models::DownloadRemoteLayersTaskSpawnRequest; use pageserver_api::models::IngestAuxFilesRequest; use pageserver_api::models::ListAuxFilesRequest; use pageserver_api::models::LocationConfig; use pageserver_api::models::LocationConfigListResponse; +use pageserver_api::models::LocationConfigMode; use pageserver_api::models::LsnLease; use pageserver_api::models::LsnLeaseRequest; use pageserver_api::models::ShardParameters; use pageserver_api::models::TenantDetails; +use pageserver_api::models::TenantLocationConfigRequest; use pageserver_api::models::TenantLocationConfigResponse; use pageserver_api::models::TenantScanRemoteStorageResponse; use pageserver_api::models::TenantScanRemoteStorageShard; @@ -33,12 +36,10 @@ use pageserver_api::models::TenantShardLocation; use pageserver_api::models::TenantShardSplitRequest; use pageserver_api::models::TenantShardSplitResponse; use pageserver_api::models::TenantSorting; +use pageserver_api::models::TimelineArchivalConfigRequest; use pageserver_api::models::TopTenantShardItem; use pageserver_api::models::TopTenantShardsRequest; use pageserver_api::models::TopTenantShardsResponse; -use pageserver_api::models::{ - DownloadRemoteLayersTaskSpawnRequest, LocationConfigMode, TenantLocationConfigRequest, -}; use pageserver_api::shard::ShardCount; use pageserver_api::shard::TenantShardId; use remote_storage::DownloadError; @@ -664,6 +665,39 @@ async fn timeline_preserve_initdb_handler( json_response(StatusCode::OK, ()) } +async fn timeline_archival_config_handler( + mut request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { + let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?; + let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; + + let request_data: TimelineArchivalConfigRequest = json_request(&mut request).await?; + check_permission(&request, Some(tenant_shard_id.tenant_id))?; + let state = get_state(&request); + + async { + let tenant = state + .tenant_manager + .get_attached_tenant_shard(tenant_shard_id)?; + + tenant + .apply_timeline_archival_config(timeline_id, request_data.state) + .await + .context("applying archival config") + .map_err(ApiError::InternalServerError)?; + Ok::<_, ApiError>(()) + } + .instrument(info_span!("timeline_archival_config", + tenant_id = %tenant_shard_id.tenant_id, + shard_id = %tenant_shard_id.shard_slug(), + state = ?request_data.state, + %timeline_id)) + .await?; + + json_response(StatusCode::OK, ()) +} + async fn timeline_detail_handler( request: Request, _cancel: CancellationToken, @@ -2789,6 +2823,10 @@ pub fn make_router( "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/preserve_initdb_archive", |r| api_handler(r, timeline_preserve_initdb_handler), ) + .post( + "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/archival_config", + |r| api_handler(r, timeline_archival_config_handler), + ) .get("/v1/tenant/:tenant_shard_id/timeline/:timeline_id", |r| { api_handler(r, timeline_detail_handler) }) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 637051413f16..01f7ac626bbc 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -21,6 +21,7 @@ use futures::FutureExt; use futures::StreamExt; use pageserver_api::models; use pageserver_api::models::AuxFilePolicy; +use pageserver_api::models::TimelineArchivalState; use pageserver_api::models::TimelineState; use pageserver_api::models::TopTenantShardItem; use pageserver_api::models::WalRedoManagerStatus; @@ -1228,6 +1229,14 @@ impl Tenant { Ok(timeline_preloads) } + pub async fn apply_timeline_archival_config( + &self, + _timeline_id: TimelineId, + _config: TimelineArchivalState, + ) -> anyhow::Result<()> { + Ok(()) + } + pub(crate) fn tenant_shard_id(&self) -> TenantShardId { self.tenant_shard_id } From 4e547e6274c362bf2779df90db2b0f1c445f9e13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Fri, 19 Jul 2024 21:19:30 +0200 Subject: [PATCH 149/194] Use DefaultCredentialsChain AWS authentication in remote_storage (#8440) PR #8299 has switched the storage scrubber to use `DefaultCredentialsChain`. Now we do this for `remote_storage`, as it allows us to use `remote_storage` from inside kubernetes. Most of the diff is due to `GenericRemoteStorage::from_config` becoming `async fn`. --- libs/remote_storage/src/lib.rs | 4 +- libs/remote_storage/src/s3_bucket.rs | 66 ++++------ libs/remote_storage/tests/test_real_azure.rs | 7 +- libs/remote_storage/tests/test_real_s3.rs | 7 +- pageserver/ctl/src/main.rs | 2 +- pageserver/src/bin/pageserver.rs | 6 +- pageserver/src/consumption_metrics.rs | 2 +- pageserver/src/deletion_queue.rs | 20 ++- pageserver/src/pgdatadir_mapping.rs | 2 +- pageserver/src/tenant.rs | 116 +++++++++++------- pageserver/src/tenant/mgr.rs | 4 +- .../src/tenant/remote_timeline_client.rs | 2 +- .../src/tenant/storage_layer/delta_layer.rs | 8 +- .../src/tenant/storage_layer/image_layer.rs | 4 +- .../src/tenant/storage_layer/layer/tests.rs | 25 ++-- .../tenant/storage_layer/merge_iterator.rs | 12 +- pageserver/src/tenant/timeline.rs | 5 +- .../walreceiver/connection_manager.rs | 17 +-- pageserver/src/walingest.rs | 16 ++- proxy/src/context/parquet.rs | 10 +- proxy/src/usage_metrics.rs | 14 ++- safekeeper/src/bin/safekeeper.rs | 2 +- safekeeper/src/wal_backup.rs | 26 ++-- 23 files changed, 220 insertions(+), 157 deletions(-) diff --git a/libs/remote_storage/src/lib.rs b/libs/remote_storage/src/lib.rs index d440c03a0e65..3381c4296f05 100644 --- a/libs/remote_storage/src/lib.rs +++ b/libs/remote_storage/src/lib.rs @@ -443,7 +443,7 @@ impl GenericRemoteStorage> { } impl GenericRemoteStorage { - pub fn from_config(storage_config: &RemoteStorageConfig) -> anyhow::Result { + pub async fn from_config(storage_config: &RemoteStorageConfig) -> anyhow::Result { let timeout = storage_config.timeout; Ok(match &storage_config.storage { RemoteStorageKind::LocalFs { local_path: path } => { @@ -458,7 +458,7 @@ impl GenericRemoteStorage { std::env::var("AWS_ACCESS_KEY_ID").unwrap_or_else(|_| "".into()); info!("Using s3 bucket '{}' in region '{}' as a remote storage, prefix in bucket: '{:?}', bucket endpoint: '{:?}', profile: {profile}, access_key_id: {access_key_id}", s3_config.bucket_name, s3_config.bucket_region, s3_config.prefix_in_bucket, s3_config.endpoint); - Self::AwsS3(Arc::new(S3Bucket::new(s3_config, timeout)?)) + Self::AwsS3(Arc::new(S3Bucket::new(s3_config, timeout).await?)) } RemoteStorageKind::AzureContainer(azure_config) => { let storage_account = azure_config diff --git a/libs/remote_storage/src/s3_bucket.rs b/libs/remote_storage/src/s3_bucket.rs index ef1bd2c04730..b65d8b7e9e7a 100644 --- a/libs/remote_storage/src/s3_bucket.rs +++ b/libs/remote_storage/src/s3_bucket.rs @@ -16,16 +16,10 @@ use std::{ use anyhow::{anyhow, Context as _}; use aws_config::{ - environment::credentials::EnvironmentVariableCredentialsProvider, - imds::credentials::ImdsCredentialsProvider, - meta::credentials::CredentialsProviderChain, - profile::ProfileFileCredentialsProvider, - provider_config::ProviderConfig, + default_provider::credentials::DefaultCredentialsChain, retry::{RetryConfigBuilder, RetryMode}, - web_identity_token::WebIdentityTokenCredentialsProvider, BehaviorVersion, }; -use aws_credential_types::provider::SharedCredentialsProvider; use aws_sdk_s3::{ config::{AsyncSleep, IdentityCache, Region, SharedAsyncSleep}, error::SdkError, @@ -76,40 +70,27 @@ struct GetObjectRequest { } impl S3Bucket { /// Creates the S3 storage, errors if incorrect AWS S3 configuration provided. - pub fn new(remote_storage_config: &S3Config, timeout: Duration) -> anyhow::Result { + pub async fn new(remote_storage_config: &S3Config, timeout: Duration) -> anyhow::Result { tracing::debug!( "Creating s3 remote storage for S3 bucket {}", remote_storage_config.bucket_name ); - let region = Some(Region::new(remote_storage_config.bucket_region.clone())); - - let provider_conf = ProviderConfig::without_region().with_region(region.clone()); - - let credentials_provider = { - // uses "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY" - CredentialsProviderChain::first_try( - "env", - EnvironmentVariableCredentialsProvider::new(), - ) - // uses "AWS_PROFILE" / `aws sso login --profile ` - .or_else( - "profile-sso", - ProfileFileCredentialsProvider::builder() - .configure(&provider_conf) - .build(), - ) - // uses "AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN", "AWS_ROLE_SESSION_NAME" - // needed to access remote extensions bucket - .or_else( - "token", - WebIdentityTokenCredentialsProvider::builder() - .configure(&provider_conf) - .build(), - ) - // uses imds v2 - .or_else("imds", ImdsCredentialsProvider::builder().build()) - }; + let region = Region::new(remote_storage_config.bucket_region.clone()); + let region_opt = Some(region.clone()); + + // https://docs.aws.amazon.com/sdkref/latest/guide/standardized-credentials.html + // https://docs.rs/aws-config/latest/aws_config/default_provider/credentials/struct.DefaultCredentialsChain.html + // Incomplete list of auth methods used by this: + // * "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY" + // * "AWS_PROFILE" / `aws sso login --profile ` + // * "AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN", "AWS_ROLE_SESSION_NAME" + // * http (ECS/EKS) container credentials + // * imds v2 + let credentials_provider = DefaultCredentialsChain::builder() + .region(region) + .build() + .await; // AWS SDK requires us to specify how the RetryConfig should sleep when it wants to back off let sleep_impl: Arc = Arc::new(TokioSleep::new()); @@ -118,9 +99,9 @@ impl S3Bucket { #[allow(deprecated)] /* TODO: https://github.com/neondatabase/neon/issues/7665 */ BehaviorVersion::v2023_11_09(), ) - .region(region) + .region(region_opt) .identity_cache(IdentityCache::lazy().build()) - .credentials_provider(SharedCredentialsProvider::new(credentials_provider)) + .credentials_provider(credentials_provider) .sleep_impl(SharedAsyncSleep::from(sleep_impl)); let sdk_config: aws_config::SdkConfig = std::thread::scope(|s| { @@ -1041,8 +1022,8 @@ mod tests { use crate::{RemotePath, S3Bucket, S3Config}; - #[test] - fn relative_path() { + #[tokio::test] + async fn relative_path() { let all_paths = ["", "some/path", "some/path/"]; let all_paths: Vec = all_paths .iter() @@ -1085,8 +1066,9 @@ mod tests { max_keys_per_list_response: Some(5), upload_storage_class: None, }; - let storage = - S3Bucket::new(&config, std::time::Duration::ZERO).expect("remote storage init"); + let storage = S3Bucket::new(&config, std::time::Duration::ZERO) + .await + .expect("remote storage init"); for (test_path_idx, test_path) in all_paths.iter().enumerate() { let result = storage.relative_path_to_s3_object(test_path); let expected = expected_outputs[prefix_idx][test_path_idx]; diff --git a/libs/remote_storage/tests/test_real_azure.rs b/libs/remote_storage/tests/test_real_azure.rs index 23628dfebecc..3a20649490ba 100644 --- a/libs/remote_storage/tests/test_real_azure.rs +++ b/libs/remote_storage/tests/test_real_azure.rs @@ -31,6 +31,7 @@ struct EnabledAzure { impl EnabledAzure { async fn setup(max_keys_in_list_response: Option) -> Self { let client = create_azure_client(max_keys_in_list_response) + .await .context("Azure client creation") .expect("Azure client creation failed"); @@ -187,7 +188,7 @@ impl AsyncTestContext for MaybeEnabledStorageWithSimpleTestBlobs { } } -fn create_azure_client( +async fn create_azure_client( max_keys_per_list_response: Option, ) -> anyhow::Result> { use rand::Rng; @@ -221,6 +222,8 @@ fn create_azure_client( timeout: Duration::from_secs(120), }; Ok(Arc::new( - GenericRemoteStorage::from_config(&remote_storage_config).context("remote storage init")?, + GenericRemoteStorage::from_config(&remote_storage_config) + .await + .context("remote storage init")?, )) } diff --git a/libs/remote_storage/tests/test_real_s3.rs b/libs/remote_storage/tests/test_real_s3.rs index a273abe867e1..342bc6da0bac 100644 --- a/libs/remote_storage/tests/test_real_s3.rs +++ b/libs/remote_storage/tests/test_real_s3.rs @@ -197,6 +197,7 @@ struct EnabledS3 { impl EnabledS3 { async fn setup(max_keys_in_list_response: Option) -> Self { let client = create_s3_client(max_keys_in_list_response) + .await .context("S3 client creation") .expect("S3 client creation failed"); @@ -352,7 +353,7 @@ impl AsyncTestContext for MaybeEnabledStorageWithSimpleTestBlobs { } } -fn create_s3_client( +async fn create_s3_client( max_keys_per_list_response: Option, ) -> anyhow::Result> { use rand::Rng; @@ -385,7 +386,9 @@ fn create_s3_client( timeout: RemoteStorageConfig::DEFAULT_TIMEOUT, }; Ok(Arc::new( - GenericRemoteStorage::from_config(&remote_storage_config).context("remote storage init")?, + GenericRemoteStorage::from_config(&remote_storage_config) + .await + .context("remote storage init")?, )) } diff --git a/pageserver/ctl/src/main.rs b/pageserver/ctl/src/main.rs index ea09a011e5cf..3fabf629875e 100644 --- a/pageserver/ctl/src/main.rs +++ b/pageserver/ctl/src/main.rs @@ -179,7 +179,7 @@ async fn main() -> anyhow::Result<()> { .get("remote_storage") .expect("need remote_storage"); let config = RemoteStorageConfig::from_toml(toml_item)?; - let storage = remote_storage::GenericRemoteStorage::from_config(&config); + let storage = remote_storage::GenericRemoteStorage::from_config(&config).await; let cancel = CancellationToken::new(); storage .unwrap() diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index fceddfb7575c..ec1ceb54ce0f 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -385,7 +385,7 @@ fn start_pageserver( let shutdown_pageserver = tokio_util::sync::CancellationToken::new(); // Set up remote storage client - let remote_storage = create_remote_storage_client(conf)?; + let remote_storage = BACKGROUND_RUNTIME.block_on(create_remote_storage_client(conf))?; // Set up deletion queue let (deletion_queue, deletion_workers) = DeletionQueue::new( @@ -701,7 +701,7 @@ fn start_pageserver( } } -fn create_remote_storage_client( +async fn create_remote_storage_client( conf: &'static PageServerConf, ) -> anyhow::Result { let config = if let Some(config) = &conf.remote_storage_config { @@ -711,7 +711,7 @@ fn create_remote_storage_client( }; // Create the client - let mut remote_storage = GenericRemoteStorage::from_config(config)?; + let mut remote_storage = GenericRemoteStorage::from_config(config).await?; // If `test_remote_failures` is non-zero, wrap the client with a // wrapper that simulates failures. diff --git a/pageserver/src/consumption_metrics.rs b/pageserver/src/consumption_metrics.rs index 6861adad2c24..9104da60729c 100644 --- a/pageserver/src/consumption_metrics.rs +++ b/pageserver/src/consumption_metrics.rs @@ -96,7 +96,7 @@ pub async fn collect_metrics( .expect("Failed to create http client with timeout"); let bucket_client = if let Some(bucket_config) = metric_collection_bucket { - match GenericRemoteStorage::from_config(bucket_config) { + match GenericRemoteStorage::from_config(bucket_config).await { Ok(client) => Some(client), Err(e) => { // Non-fatal error: if we were given an invalid config, we will proceed diff --git a/pageserver/src/deletion_queue.rs b/pageserver/src/deletion_queue.rs index 3e48552ace44..22f7d5b8242d 100644 --- a/pageserver/src/deletion_queue.rs +++ b/pageserver/src/deletion_queue.rs @@ -828,9 +828,9 @@ mod test { } } - fn setup(test_name: &str) -> anyhow::Result { + async fn setup(test_name: &str) -> anyhow::Result { let test_name = Box::leak(Box::new(format!("deletion_queue__{test_name}"))); - let harness = TenantHarness::create(test_name)?; + let harness = TenantHarness::create(test_name).await?; // We do not load() the harness: we only need its config and remote_storage @@ -844,7 +844,9 @@ mod test { }, timeout: RemoteStorageConfig::DEFAULT_TIMEOUT, }; - let storage = GenericRemoteStorage::from_config(&storage_config).unwrap(); + let storage = GenericRemoteStorage::from_config(&storage_config) + .await + .unwrap(); let mock_control_plane = MockControlPlane::new(); @@ -922,7 +924,9 @@ mod test { #[tokio::test] async fn deletion_queue_smoke() -> anyhow::Result<()> { // Basic test that the deletion queue processes the deletions we pass into it - let ctx = setup("deletion_queue_smoke").expect("Failed test setup"); + let ctx = setup("deletion_queue_smoke") + .await + .expect("Failed test setup"); let client = ctx.deletion_queue.new_client(); client.recover(HashMap::new())?; @@ -992,7 +996,9 @@ mod test { #[tokio::test] async fn deletion_queue_validation() -> anyhow::Result<()> { - let ctx = setup("deletion_queue_validation").expect("Failed test setup"); + let ctx = setup("deletion_queue_validation") + .await + .expect("Failed test setup"); let client = ctx.deletion_queue.new_client(); client.recover(HashMap::new())?; @@ -1051,7 +1057,9 @@ mod test { #[tokio::test] async fn deletion_queue_recovery() -> anyhow::Result<()> { // Basic test that the deletion queue processes the deletions we pass into it - let mut ctx = setup("deletion_queue_recovery").expect("Failed test setup"); + let mut ctx = setup("deletion_queue_recovery") + .await + .expect("Failed test setup"); let client = ctx.deletion_queue.new_client(); client.recover(HashMap::new())?; diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index a821b824d0c3..3bbd084ab498 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -2031,7 +2031,7 @@ mod tests { #[tokio::test] async fn aux_files_round_trip() -> anyhow::Result<()> { let name = "aux_files_round_trip"; - let harness = TenantHarness::create(name)?; + let harness = TenantHarness::create(name).await?; pub const TIMELINE_ID: TimelineId = TimelineId::from_array(hex!("11223344556677881122334455667788")); diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 01f7ac626bbc..6d597526068f 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -3797,7 +3797,7 @@ pub(crate) mod harness { } impl TenantHarness { - pub fn create_custom( + pub async fn create_custom( test_name: &'static str, tenant_conf: TenantConf, tenant_id: TenantId, @@ -3833,7 +3833,7 @@ pub(crate) mod harness { }, timeout: RemoteStorageConfig::DEFAULT_TIMEOUT, }; - let remote_storage = GenericRemoteStorage::from_config(&config).unwrap(); + let remote_storage = GenericRemoteStorage::from_config(&config).await.unwrap(); let deletion_queue = MockDeletionQueue::new(Some(remote_storage.clone())); Ok(Self { @@ -3848,7 +3848,7 @@ pub(crate) mod harness { }) } - pub fn create(test_name: &'static str) -> anyhow::Result { + pub async fn create(test_name: &'static str) -> anyhow::Result { // Disable automatic GC and compaction to make the unit tests more deterministic. // The tests perform them manually if needed. let tenant_conf = TenantConf { @@ -3865,6 +3865,7 @@ pub(crate) mod harness { shard, Generation::new(0xdeadbeef), ) + .await } pub fn span(&self) -> tracing::Span { @@ -4001,7 +4002,7 @@ mod tests { #[tokio::test] async fn test_basic() -> anyhow::Result<()> { - let (tenant, ctx) = TenantHarness::create("test_basic")?.load().await; + let (tenant, ctx) = TenantHarness::create("test_basic").await?.load().await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(0x08), DEFAULT_PG_VERSION, &ctx) .await?; @@ -4048,7 +4049,8 @@ mod tests { #[tokio::test] async fn no_duplicate_timelines() -> anyhow::Result<()> { - let (tenant, ctx) = TenantHarness::create("no_duplicate_timelines")? + let (tenant, ctx) = TenantHarness::create("no_duplicate_timelines") + .await? .load() .await; let _ = tenant @@ -4080,7 +4082,7 @@ mod tests { async fn test_branch() -> anyhow::Result<()> { use std::str::from_utf8; - let (tenant, ctx) = TenantHarness::create("test_branch")?.load().await; + let (tenant, ctx) = TenantHarness::create("test_branch").await?.load().await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) .await?; @@ -4202,7 +4204,8 @@ mod tests { #[tokio::test] async fn test_prohibit_branch_creation_on_garbage_collected_data() -> anyhow::Result<()> { let (tenant, ctx) = - TenantHarness::create("test_prohibit_branch_creation_on_garbage_collected_data")? + TenantHarness::create("test_prohibit_branch_creation_on_garbage_collected_data") + .await? .load() .await; let tline = tenant @@ -4249,7 +4252,8 @@ mod tests { #[tokio::test] async fn test_prohibit_branch_creation_on_pre_initdb_lsn() -> anyhow::Result<()> { let (tenant, ctx) = - TenantHarness::create("test_prohibit_branch_creation_on_pre_initdb_lsn")? + TenantHarness::create("test_prohibit_branch_creation_on_pre_initdb_lsn") + .await? .load() .await; @@ -4304,7 +4308,8 @@ mod tests { #[tokio::test] async fn test_get_branchpoints_from_an_inactive_timeline() -> anyhow::Result<()> { let (tenant, ctx) = - TenantHarness::create("test_get_branchpoints_from_an_inactive_timeline")? + TenantHarness::create("test_get_branchpoints_from_an_inactive_timeline") + .await? .load() .await; let tline = tenant @@ -4361,7 +4366,8 @@ mod tests { #[tokio::test] async fn test_retain_data_in_parent_which_is_needed_for_child() -> anyhow::Result<()> { let (tenant, ctx) = - TenantHarness::create("test_retain_data_in_parent_which_is_needed_for_child")? + TenantHarness::create("test_retain_data_in_parent_which_is_needed_for_child") + .await? .load() .await; let tline = tenant @@ -4391,10 +4397,10 @@ mod tests { } #[tokio::test] async fn test_parent_keeps_data_forever_after_branching() -> anyhow::Result<()> { - let (tenant, ctx) = - TenantHarness::create("test_parent_keeps_data_forever_after_branching")? - .load() - .await; + let (tenant, ctx) = TenantHarness::create("test_parent_keeps_data_forever_after_branching") + .await? + .load() + .await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) .await?; @@ -4432,7 +4438,7 @@ mod tests { #[tokio::test] async fn timeline_load() -> anyhow::Result<()> { const TEST_NAME: &str = "timeline_load"; - let harness = TenantHarness::create(TEST_NAME)?; + let harness = TenantHarness::create(TEST_NAME).await?; { let (tenant, ctx) = harness.load().await; let tline = tenant @@ -4459,7 +4465,7 @@ mod tests { #[tokio::test] async fn timeline_load_with_ancestor() -> anyhow::Result<()> { const TEST_NAME: &str = "timeline_load_with_ancestor"; - let harness = TenantHarness::create(TEST_NAME)?; + let harness = TenantHarness::create(TEST_NAME).await?; // create two timelines { let (tenant, ctx) = harness.load().await; @@ -4507,7 +4513,10 @@ mod tests { #[tokio::test] async fn delta_layer_dumping() -> anyhow::Result<()> { use storage_layer::AsLayerDesc; - let (tenant, ctx) = TenantHarness::create("test_layer_dumping")?.load().await; + let (tenant, ctx) = TenantHarness::create("test_layer_dumping") + .await? + .load() + .await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) .await?; @@ -4534,7 +4543,7 @@ mod tests { #[tokio::test] async fn test_images() -> anyhow::Result<()> { - let (tenant, ctx) = TenantHarness::create("test_images")?.load().await; + let (tenant, ctx) = TenantHarness::create("test_images").await?.load().await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(0x08), DEFAULT_PG_VERSION, &ctx) .await?; @@ -4705,7 +4714,7 @@ mod tests { // #[tokio::test] async fn test_bulk_insert() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_bulk_insert")?; + let harness = TenantHarness::create("test_bulk_insert").await?; let (tenant, ctx) = harness.load().await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(0x08), DEFAULT_PG_VERSION, &ctx) @@ -4736,7 +4745,7 @@ mod tests { // so the search can stop at the first delta layer and doesn't traverse any deeper. #[tokio::test] async fn test_get_vectored() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_get_vectored")?; + let harness = TenantHarness::create("test_get_vectored").await?; let (tenant, ctx) = harness.load().await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(0x08), DEFAULT_PG_VERSION, &ctx) @@ -4814,7 +4823,7 @@ mod tests { #[tokio::test] async fn test_get_vectored_aux_files() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_get_vectored_aux_files")?; + let harness = TenantHarness::create("test_get_vectored_aux_files").await?; let (tenant, ctx) = harness.load().await; let tline = tenant @@ -4900,7 +4909,8 @@ mod tests { TenantId::generate(), ShardIdentity::unsharded(), Generation::new(0xdeadbeef), - )?; + ) + .await?; let (tenant, ctx) = harness.load().await; let mut current_key = Key::from_hex("010000000033333333444444445500000000").unwrap(); @@ -5043,7 +5053,7 @@ mod tests { // ``` #[tokio::test] async fn test_get_vectored_ancestor_descent() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_get_vectored_on_lsn_axis")?; + let harness = TenantHarness::create("test_get_vectored_on_lsn_axis").await?; let (tenant, ctx) = harness.load().await; let start_key = Key::from_hex("010000000033333333444444445500000000").unwrap(); @@ -5192,7 +5202,7 @@ mod tests { name: &'static str, compaction_algorithm: CompactionAlgorithm, ) -> anyhow::Result<()> { - let mut harness = TenantHarness::create(name)?; + let mut harness = TenantHarness::create(name).await?; harness.tenant_conf.compaction_algorithm = CompactionAlgorithmSettings { kind: compaction_algorithm, }; @@ -5276,7 +5286,8 @@ mod tests { #[tokio::test] async fn test_traverse_branches() -> anyhow::Result<()> { - let (tenant, ctx) = TenantHarness::create("test_traverse_branches")? + let (tenant, ctx) = TenantHarness::create("test_traverse_branches") + .await? .load() .await; let mut tline = tenant @@ -5366,7 +5377,8 @@ mod tests { #[tokio::test] async fn test_traverse_ancestors() -> anyhow::Result<()> { - let (tenant, ctx) = TenantHarness::create("test_traverse_ancestors")? + let (tenant, ctx) = TenantHarness::create("test_traverse_ancestors") + .await? .load() .await; let mut tline = tenant @@ -5432,7 +5444,8 @@ mod tests { #[tokio::test] async fn test_write_at_initdb_lsn_takes_optimization_code_path() -> anyhow::Result<()> { - let (tenant, ctx) = TenantHarness::create("test_empty_test_timeline_is_usable")? + let (tenant, ctx) = TenantHarness::create("test_empty_test_timeline_is_usable") + .await? .load() .await; @@ -5501,7 +5514,7 @@ mod tests { #[tokio::test] async fn test_create_guard_crash() -> anyhow::Result<()> { let name = "test_create_guard_crash"; - let harness = TenantHarness::create(name)?; + let harness = TenantHarness::create(name).await?; { let (tenant, ctx) = harness.load().await; let tline = tenant @@ -5554,7 +5567,7 @@ mod tests { name: &'static str, compaction_algorithm: CompactionAlgorithm, ) -> anyhow::Result<()> { - let mut harness = TenantHarness::create(name)?; + let mut harness = TenantHarness::create(name).await?; harness.tenant_conf.compaction_algorithm = CompactionAlgorithmSettings { kind: compaction_algorithm, }; @@ -5578,7 +5591,7 @@ mod tests { #[tokio::test] async fn test_metadata_scan() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_metadata_scan")?; + let harness = TenantHarness::create("test_metadata_scan").await?; let (tenant, ctx) = harness.load().await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) @@ -5697,7 +5710,7 @@ mod tests { #[tokio::test] async fn test_metadata_compaction_trigger() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_metadata_compaction_trigger")?; + let harness = TenantHarness::create("test_metadata_compaction_trigger").await?; let (tenant, ctx) = harness.load().await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) @@ -5756,7 +5769,9 @@ mod tests { #[tokio::test] async fn test_branch_copies_dirty_aux_file_flag() { - let harness = TenantHarness::create("test_branch_copies_dirty_aux_file_flag").unwrap(); + let harness = TenantHarness::create("test_branch_copies_dirty_aux_file_flag") + .await + .unwrap(); // the default aux file policy to switch is v1 if not set by the admins assert_eq!( @@ -5858,7 +5873,9 @@ mod tests { #[tokio::test] async fn aux_file_policy_switch() { - let mut harness = TenantHarness::create("aux_file_policy_switch").unwrap(); + let mut harness = TenantHarness::create("aux_file_policy_switch") + .await + .unwrap(); harness.tenant_conf.switch_aux_file_policy = AuxFilePolicy::CrossValidation; // set to cross-validation mode let (tenant, ctx) = harness.load().await; @@ -6032,7 +6049,9 @@ mod tests { #[tokio::test] async fn aux_file_policy_force_switch() { - let mut harness = TenantHarness::create("aux_file_policy_force_switch").unwrap(); + let mut harness = TenantHarness::create("aux_file_policy_force_switch") + .await + .unwrap(); harness.tenant_conf.switch_aux_file_policy = AuxFilePolicy::V1; let (tenant, ctx) = harness.load().await; @@ -6093,7 +6112,9 @@ mod tests { #[tokio::test] async fn aux_file_policy_auto_detect() { - let mut harness = TenantHarness::create("aux_file_policy_auto_detect").unwrap(); + let mut harness = TenantHarness::create("aux_file_policy_auto_detect") + .await + .unwrap(); harness.tenant_conf.switch_aux_file_policy = AuxFilePolicy::V2; // set to cross-validation mode let (tenant, ctx) = harness.load().await; @@ -6156,7 +6177,7 @@ mod tests { #[tokio::test] async fn test_metadata_image_creation() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_metadata_image_creation")?; + let harness = TenantHarness::create("test_metadata_image_creation").await?; let (tenant, ctx) = harness.load().await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) @@ -6255,7 +6276,7 @@ mod tests { #[tokio::test] async fn test_vectored_missing_data_key_reads() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_vectored_missing_data_key_reads")?; + let harness = TenantHarness::create("test_vectored_missing_data_key_reads").await?; let (tenant, ctx) = harness.load().await; let base_key = Key::from_hex("000000000033333333444444445500000000").unwrap(); @@ -6327,7 +6348,7 @@ mod tests { #[tokio::test] async fn test_vectored_missing_metadata_key_reads() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_vectored_missing_metadata_key_reads")?; + let harness = TenantHarness::create("test_vectored_missing_metadata_key_reads").await?; let (tenant, ctx) = harness.load().await; let base_key = Key::from_hex("620000000033333333444444445500000000").unwrap(); @@ -6419,7 +6440,7 @@ mod tests { #[tokio::test] async fn test_metadata_tombstone_reads() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_metadata_tombstone_reads")?; + let harness = TenantHarness::create("test_metadata_tombstone_reads").await?; let (tenant, ctx) = harness.load().await; let key0 = Key::from_hex("620000000033333333444444445500000000").unwrap(); let key1 = Key::from_hex("620000000033333333444444445500000001").unwrap(); @@ -6499,7 +6520,9 @@ mod tests { #[tokio::test] async fn test_metadata_tombstone_image_creation() { - let harness = TenantHarness::create("test_metadata_tombstone_image_creation").unwrap(); + let harness = TenantHarness::create("test_metadata_tombstone_image_creation") + .await + .unwrap(); let (tenant, ctx) = harness.load().await; let key0 = Key::from_hex("620000000033333333444444445500000000").unwrap(); @@ -6571,8 +6594,9 @@ mod tests { #[tokio::test] async fn test_metadata_tombstone_empty_image_creation() { - let harness = - TenantHarness::create("test_metadata_tombstone_empty_image_creation").unwrap(); + let harness = TenantHarness::create("test_metadata_tombstone_empty_image_creation") + .await + .unwrap(); let (tenant, ctx) = harness.load().await; let key1 = Key::from_hex("620000000033333333444444445500000001").unwrap(); @@ -6635,7 +6659,7 @@ mod tests { #[tokio::test] async fn test_simple_bottom_most_compaction_images() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_simple_bottom_most_compaction_images")?; + let harness = TenantHarness::create("test_simple_bottom_most_compaction_images").await?; let (tenant, ctx) = harness.load().await; fn get_key(id: u32) -> Key { @@ -6843,7 +6867,7 @@ mod tests { #[tokio::test] async fn test_neon_test_record() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_neon_test_record")?; + let harness = TenantHarness::create("test_neon_test_record").await?; let (tenant, ctx) = harness.load().await; fn get_key(id: u32) -> Key { @@ -6924,7 +6948,7 @@ mod tests { #[tokio::test] async fn test_lsn_lease() -> anyhow::Result<()> { - let (tenant, ctx) = TenantHarness::create("test_lsn_lease")?.load().await; + let (tenant, ctx) = TenantHarness::create("test_lsn_lease").await?.load().await; let key = Key::from_hex("010000000033333333444444445500000000").unwrap(); let end_lsn = Lsn(0x100); @@ -7013,7 +7037,7 @@ mod tests { #[tokio::test] async fn test_simple_bottom_most_compaction_deltas() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_simple_bottom_most_compaction_deltas")?; + let harness = TenantHarness::create("test_simple_bottom_most_compaction_deltas").await?; let (tenant, ctx) = harness.load().await; fn get_key(id: u32) -> Key { diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index b0159e22bfc0..49126086772b 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -2698,7 +2698,9 @@ mod tests { // Test that if an InProgress tenant is in the map during shutdown, the shutdown will gracefully // wait for it to complete before proceeding. - let h = TenantHarness::create("shutdown_awaits_in_progress_tenant").unwrap(); + let h = TenantHarness::create("shutdown_awaits_in_progress_tenant") + .await + .unwrap(); let (t, _ctx) = h.load().await; // harness loads it to active, which is forced and nothing is running on the tenant diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index 66b759c8e0d8..bb42fbeebf78 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -2128,7 +2128,7 @@ mod tests { impl TestSetup { async fn new(test_name: &str) -> anyhow::Result { let test_name = Box::leak(Box::new(format!("remote_timeline_client__{test_name}"))); - let harness = TenantHarness::create(test_name)?; + let harness = TenantHarness::create(test_name).await?; let (tenant, ctx) = harness.load().await; let timeline = tenant diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index c34923320aee..512e9e86fac1 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -1934,7 +1934,7 @@ pub(crate) mod test { #[tokio::test] async fn test_delta_layer_vectored_read_end_to_end() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_delta_layer_oversized_vectored_read")?; + let harness = TenantHarness::create("test_delta_layer_oversized_vectored_read").await?; let (tenant, ctx) = harness.load().await; let timeline_id = TimelineId::generate(); @@ -2034,7 +2034,9 @@ pub(crate) mod test { use crate::walrecord::NeonWalRecord; use bytes::Bytes; - let h = crate::tenant::harness::TenantHarness::create("truncate_delta_smoke").unwrap(); + let h = crate::tenant::harness::TenantHarness::create("truncate_delta_smoke") + .await + .unwrap(); let (tenant, ctx) = h.load().await; let ctx = &ctx; let timeline = tenant @@ -2312,7 +2314,7 @@ pub(crate) mod test { #[tokio::test] async fn delta_layer_iterator() { - let harness = TenantHarness::create("delta_layer_iterator").unwrap(); + let harness = TenantHarness::create("delta_layer_iterator").await.unwrap(); let (tenant, ctx) = harness.load().await; let tline = tenant diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index 45b47bb62b0c..19e4e9e2e9ca 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -1111,6 +1111,7 @@ mod test { ShardIdentity::unsharded(), get_next_gen(), ) + .await .unwrap(); let (tenant, ctx) = harness.load().await; let timeline = tenant @@ -1177,6 +1178,7 @@ mod test { // But here, all we care about is that the gen number is unique. get_next_gen(), ) + .await .unwrap(); let (tenant, ctx) = harness.load().await; let timeline = tenant @@ -1308,7 +1310,7 @@ mod test { #[tokio::test] async fn image_layer_iterator() { - let harness = TenantHarness::create("image_layer_iterator").unwrap(); + let harness = TenantHarness::create("image_layer_iterator").await.unwrap(); let (tenant, ctx) = harness.load().await; let tline = tenant diff --git a/pageserver/src/tenant/storage_layer/layer/tests.rs b/pageserver/src/tenant/storage_layer/layer/tests.rs index 3a7aca7a6cc4..8a3737f8a760 100644 --- a/pageserver/src/tenant/storage_layer/layer/tests.rs +++ b/pageserver/src/tenant/storage_layer/layer/tests.rs @@ -22,7 +22,7 @@ const FOREVER: std::time::Duration = std::time::Duration::from_secs(ADVANCE.as_s async fn smoke_test() { let handle = tokio::runtime::Handle::current(); - let h = TenantHarness::create("smoke_test").unwrap(); + let h = TenantHarness::create("smoke_test").await.unwrap(); let span = h.span(); let download_span = span.in_scope(|| tracing::info_span!("downloading", timeline_id = 1)); let (tenant, _) = h.load().await; @@ -176,7 +176,9 @@ async fn evict_and_wait_on_wanted_deleted() { // this is the runtime on which Layer spawns the blocking tasks on let handle = tokio::runtime::Handle::current(); - let h = TenantHarness::create("evict_and_wait_on_wanted_deleted").unwrap(); + let h = TenantHarness::create("evict_and_wait_on_wanted_deleted") + .await + .unwrap(); utils::logging::replace_panic_hook_with_tracing_panic_hook().forget(); let (tenant, ctx) = h.load().await; @@ -258,7 +260,9 @@ fn read_wins_pending_eviction() { rt.block_on(async move { // this is the runtime on which Layer spawns the blocking tasks on let handle = tokio::runtime::Handle::current(); - let h = TenantHarness::create("read_wins_pending_eviction").unwrap(); + let h = TenantHarness::create("read_wins_pending_eviction") + .await + .unwrap(); let (tenant, ctx) = h.load().await; let span = h.span(); let download_span = span.in_scope(|| tracing::info_span!("downloading", timeline_id = 1)); @@ -390,7 +394,7 @@ fn multiple_pending_evictions_scenario(name: &'static str, in_order: bool) { rt.block_on(async move { // this is the runtime on which Layer spawns the blocking tasks on let handle = tokio::runtime::Handle::current(); - let h = TenantHarness::create(name).unwrap(); + let h = TenantHarness::create(name).await.unwrap(); let (tenant, ctx) = h.load().await; let span = h.span(); let download_span = span.in_scope(|| tracing::info_span!("downloading", timeline_id = 1)); @@ -559,8 +563,9 @@ fn multiple_pending_evictions_scenario(name: &'static str, in_order: bool) { #[tokio::test(start_paused = true)] async fn cancelled_get_or_maybe_download_does_not_cancel_eviction() { let handle = tokio::runtime::Handle::current(); - let h = - TenantHarness::create("cancelled_get_or_maybe_download_does_not_cancel_eviction").unwrap(); + let h = TenantHarness::create("cancelled_get_or_maybe_download_does_not_cancel_eviction") + .await + .unwrap(); let (tenant, ctx) = h.load().await; let timeline = tenant @@ -636,7 +641,9 @@ async fn cancelled_get_or_maybe_download_does_not_cancel_eviction() { #[tokio::test(start_paused = true)] async fn evict_and_wait_does_not_wait_for_download() { // let handle = tokio::runtime::Handle::current(); - let h = TenantHarness::create("evict_and_wait_does_not_wait_for_download").unwrap(); + let h = TenantHarness::create("evict_and_wait_does_not_wait_for_download") + .await + .unwrap(); let (tenant, ctx) = h.load().await; let span = h.span(); let download_span = span.in_scope(|| tracing::info_span!("downloading", timeline_id = 1)); @@ -733,7 +740,9 @@ async fn eviction_cancellation_on_drop() { // this is the runtime on which Layer spawns the blocking tasks on let handle = tokio::runtime::Handle::current(); - let h = TenantHarness::create("eviction_cancellation_on_drop").unwrap(); + let h = TenantHarness::create("eviction_cancellation_on_drop") + .await + .unwrap(); utils::logging::replace_panic_hook_with_tracing_panic_hook().forget(); let (tenant, ctx) = h.load().await; diff --git a/pageserver/src/tenant/storage_layer/merge_iterator.rs b/pageserver/src/tenant/storage_layer/merge_iterator.rs index 6f59b2fd7765..eb4a1f28a11c 100644 --- a/pageserver/src/tenant/storage_layer/merge_iterator.rs +++ b/pageserver/src/tenant/storage_layer/merge_iterator.rs @@ -293,7 +293,9 @@ mod tests { use crate::repository::Value; use bytes::Bytes; - let harness = TenantHarness::create("merge_iterator_merge_in_between").unwrap(); + let harness = TenantHarness::create("merge_iterator_merge_in_between") + .await + .unwrap(); let (tenant, ctx) = harness.load().await; let tline = tenant @@ -356,7 +358,9 @@ mod tests { use crate::repository::Value; use bytes::Bytes; - let harness = TenantHarness::create("merge_iterator_delta_merge").unwrap(); + let harness = TenantHarness::create("merge_iterator_delta_merge") + .await + .unwrap(); let (tenant, ctx) = harness.load().await; let tline = tenant @@ -430,7 +434,9 @@ mod tests { use crate::repository::Value; use bytes::Bytes; - let harness = TenantHarness::create("merge_iterator_delta_image_mixed_merge").unwrap(); + let harness = TenantHarness::create("merge_iterator_delta_image_mixed_merge") + .await + .unwrap(); let (tenant, ctx) = harness.load().await; let tline = tenant diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 3d3d3ac34de1..19b13969811c 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -6046,8 +6046,9 @@ mod tests { #[tokio::test] async fn two_layer_eviction_attempts_at_the_same_time() { - let harness = - TenantHarness::create("two_layer_eviction_attempts_at_the_same_time").unwrap(); + let harness = TenantHarness::create("two_layer_eviction_attempts_at_the_same_time") + .await + .unwrap(); let (tenant, ctx) = harness.load().await; let timeline = tenant diff --git a/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs b/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs index 1d2ffec08fb5..de50f217d80e 100644 --- a/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs +++ b/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs @@ -1118,7 +1118,7 @@ mod tests { #[tokio::test] async fn no_connection_no_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::create("no_connection_no_candidate")?; + let harness = TenantHarness::create("no_connection_no_candidate").await?; let mut state = dummy_state(&harness).await; let now = Utc::now().naive_utc(); @@ -1151,7 +1151,7 @@ mod tests { #[tokio::test] async fn connection_no_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::create("connection_no_candidate")?; + let harness = TenantHarness::create("connection_no_candidate").await?; let mut state = dummy_state(&harness).await; let now = Utc::now().naive_utc(); @@ -1216,7 +1216,7 @@ mod tests { #[tokio::test] async fn no_connection_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::create("no_connection_candidate")?; + let harness = TenantHarness::create("no_connection_candidate").await?; let mut state = dummy_state(&harness).await; let now = Utc::now().naive_utc(); @@ -1279,7 +1279,7 @@ mod tests { #[tokio::test] async fn candidate_with_many_connection_failures() -> anyhow::Result<()> { - let harness = TenantHarness::create("candidate_with_many_connection_failures")?; + let harness = TenantHarness::create("candidate_with_many_connection_failures").await?; let mut state = dummy_state(&harness).await; let now = Utc::now().naive_utc(); @@ -1319,7 +1319,7 @@ mod tests { #[tokio::test] async fn lsn_wal_over_threshold_current_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::create("lsn_wal_over_threshcurrent_candidate")?; + let harness = TenantHarness::create("lsn_wal_over_threshcurrent_candidate").await?; let mut state = dummy_state(&harness).await; let current_lsn = Lsn(100_000).align(); let now = Utc::now().naive_utc(); @@ -1385,7 +1385,8 @@ mod tests { #[tokio::test] async fn timeout_connection_threshold_current_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::create("timeout_connection_threshold_current_candidate")?; + let harness = + TenantHarness::create("timeout_connection_threshold_current_candidate").await?; let mut state = dummy_state(&harness).await; let current_lsn = Lsn(100_000).align(); let now = Utc::now().naive_utc(); @@ -1448,7 +1449,7 @@ mod tests { #[tokio::test] async fn timeout_wal_over_threshold_current_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::create("timeout_wal_over_threshold_current_candidate")?; + let harness = TenantHarness::create("timeout_wal_over_threshold_current_candidate").await?; let mut state = dummy_state(&harness).await; let current_lsn = Lsn(100_000).align(); let new_lsn = Lsn(100_100).align(); @@ -1550,7 +1551,7 @@ mod tests { // and pageserver should prefer to connect to it. let test_az = Some("test_az".to_owned()); - let harness = TenantHarness::create("switch_to_same_availability_zone")?; + let harness = TenantHarness::create("switch_to_same_availability_zone").await?; let mut state = dummy_state(&harness).await; state.conf.availability_zone.clone_from(&test_az); let current_lsn = Lsn(100_000).align(); diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index 07c90385e654..dff3a8f52da4 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -1754,7 +1754,7 @@ mod tests { #[tokio::test] async fn test_relsize() -> Result<()> { - let (tenant, ctx) = TenantHarness::create("test_relsize")?.load().await; + let (tenant, ctx) = TenantHarness::create("test_relsize").await?.load().await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(8), DEFAULT_PG_VERSION, &ctx) .await?; @@ -1975,7 +1975,10 @@ mod tests { // and then created it again within the same layer. #[tokio::test] async fn test_drop_extend() -> Result<()> { - let (tenant, ctx) = TenantHarness::create("test_drop_extend")?.load().await; + let (tenant, ctx) = TenantHarness::create("test_drop_extend") + .await? + .load() + .await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(8), DEFAULT_PG_VERSION, &ctx) .await?; @@ -2046,7 +2049,10 @@ mod tests { // and then extended it again within the same layer. #[tokio::test] async fn test_truncate_extend() -> Result<()> { - let (tenant, ctx) = TenantHarness::create("test_truncate_extend")?.load().await; + let (tenant, ctx) = TenantHarness::create("test_truncate_extend") + .await? + .load() + .await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(8), DEFAULT_PG_VERSION, &ctx) .await?; @@ -2188,7 +2194,7 @@ mod tests { /// split into multiple 1 GB segments in Postgres. #[tokio::test] async fn test_large_rel() -> Result<()> { - let (tenant, ctx) = TenantHarness::create("test_large_rel")?.load().await; + let (tenant, ctx) = TenantHarness::create("test_large_rel").await?.load().await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(8), DEFAULT_PG_VERSION, &ctx) .await?; @@ -2296,7 +2302,7 @@ mod tests { let startpoint = Lsn::from_hex("14AEC08").unwrap(); let _endpoint = Lsn::from_hex("1FFFF98").unwrap(); - let harness = TenantHarness::create("test_ingest_real_wal").unwrap(); + let harness = TenantHarness::create("test_ingest_real_wal").await.unwrap(); let (tenant, ctx) = harness.load().await; let remote_initdb_path = diff --git a/proxy/src/context/parquet.rs b/proxy/src/context/parquet.rs index cfc1f8e89e3f..543a45827400 100644 --- a/proxy/src/context/parquet.rs +++ b/proxy/src/context/parquet.rs @@ -181,8 +181,9 @@ pub async fn worker( let rx = futures::stream::poll_fn(move |cx| rx.poll_recv(cx)); let rx = rx.map(RequestData::from); - let storage = - GenericRemoteStorage::from_config(&remote_storage_config).context("remote storage init")?; + let storage = GenericRemoteStorage::from_config(&remote_storage_config) + .await + .context("remote storage init")?; let properties = WriterProperties::builder() .set_data_page_size_limit(config.parquet_upload_page_size) @@ -217,6 +218,7 @@ pub async fn worker( let storage_disconnect = GenericRemoteStorage::from_config(&disconnect_events_storage_config) + .await .context("remote storage for disconnect events init")?; let parquet_config_disconnect = parquet_config.clone(); tokio::try_join!( @@ -545,7 +547,9 @@ mod tests { }, timeout: std::time::Duration::from_secs(120), }; - let storage = GenericRemoteStorage::from_config(&remote_storage_config).unwrap(); + let storage = GenericRemoteStorage::from_config(&remote_storage_config) + .await + .unwrap(); worker_inner(storage, rx, config).await.unwrap(); diff --git a/proxy/src/usage_metrics.rs b/proxy/src/usage_metrics.rs index 56ed2145dc25..a8735fe0bbda 100644 --- a/proxy/src/usage_metrics.rs +++ b/proxy/src/usage_metrics.rs @@ -357,11 +357,15 @@ pub async fn task_backup( info!("metrics backup has shut down"); } // Even if the remote storage is not configured, we still want to clear the metrics. - let storage = backup_config - .remote_storage_config - .as_ref() - .map(|config| GenericRemoteStorage::from_config(config).context("remote storage init")) - .transpose()?; + let storage = if let Some(config) = backup_config.remote_storage_config.as_ref() { + Some( + GenericRemoteStorage::from_config(config) + .await + .context("remote storage init")?, + ) + } else { + None + }; let mut ticker = tokio::time::interval(backup_config.interval); let mut prev = Utc::now(); let hostname = hostname::get()?.as_os_str().to_string_lossy().into_owned(); diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index 9eb6546d6bae..2365fd05871f 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -418,7 +418,7 @@ async fn start_safekeeper(conf: SafeKeeperConf) -> Result<()> { let timeline_collector = safekeeper::metrics::TimelineCollector::new(); metrics::register_internal(Box::new(timeline_collector))?; - wal_backup::init_remote_storage(&conf); + wal_backup::init_remote_storage(&conf).await; // Keep handles to main tasks to die if any of them disappears. let mut tasks_handles: FuturesUnordered> = diff --git a/safekeeper/src/wal_backup.rs b/safekeeper/src/wal_backup.rs index 5a590689c374..7ecee178f3b4 100644 --- a/safekeeper/src/wal_backup.rs +++ b/safekeeper/src/wal_backup.rs @@ -22,7 +22,7 @@ use tokio::fs::File; use tokio::select; use tokio::sync::mpsc::{self, Receiver, Sender}; -use tokio::sync::watch; +use tokio::sync::{watch, OnceCell}; use tokio::time::sleep; use tracing::*; @@ -33,8 +33,6 @@ use crate::timeline::{PeerInfo, WalResidentTimeline}; use crate::timeline_manager::{Manager, StateSnapshot}; use crate::{SafeKeeperConf, WAL_BACKUP_RUNTIME}; -use once_cell::sync::OnceCell; - const UPLOAD_FAILURE_RETRY_MIN_MS: u64 = 10; const UPLOAD_FAILURE_RETRY_MAX_MS: u64 = 5000; @@ -167,7 +165,7 @@ fn determine_offloader( } } -static REMOTE_STORAGE: OnceCell> = OnceCell::new(); +static REMOTE_STORAGE: OnceCell> = OnceCell::const_new(); // Storage must be configured and initialized when this is called. fn get_configured_remote_storage() -> &'static GenericRemoteStorage { @@ -178,14 +176,22 @@ fn get_configured_remote_storage() -> &'static GenericRemoteStorage { .unwrap() } -pub fn init_remote_storage(conf: &SafeKeeperConf) { +pub async fn init_remote_storage(conf: &SafeKeeperConf) { // TODO: refactor REMOTE_STORAGE to avoid using global variables, and provide // dependencies to all tasks instead. - REMOTE_STORAGE.get_or_init(|| { - conf.remote_storage - .as_ref() - .map(|c| GenericRemoteStorage::from_config(c).expect("failed to create remote storage")) - }); + REMOTE_STORAGE + .get_or_init(|| async { + if let Some(conf) = conf.remote_storage.as_ref() { + Some( + GenericRemoteStorage::from_config(conf) + .await + .expect("failed to create remote storage"), + ) + } else { + None + } + }) + .await; } struct WalBackupTask { From 1637a6ee054608887d8ea04c8f0252fae5036acc Mon Sep 17 00:00:00 2001 From: Luca Bruno Date: Mon, 15 Jul 2024 13:38:52 +0200 Subject: [PATCH 150/194] proxy/http: switch to typed_json (#8377) ## Summary of changes This switches JSON rendering logic to `typed_json` in order to reduce the number of allocations in the HTTP responder path. Followup from https://github.com/neondatabase/neon/pull/8319#issuecomment-2216991760. --------- Co-authored-by: Conrad Ludgate --- Cargo.lock | 11 +++ Cargo.toml | 1 + proxy/Cargo.toml | 1 + proxy/src/serverless/sql_over_http.rs | 97 +++++++++++++-------------- 4 files changed, 59 insertions(+), 51 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bab0b4dd1f7a..88973647017a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4404,6 +4404,7 @@ dependencies = [ "tracing-opentelemetry", "tracing-subscriber", "tracing-utils", + "typed-json", "url", "urlencoding", "utils", @@ -6665,6 +6666,16 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "typed-json" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6024a8d0025400b3f6b189366e9aa92012cf9c4fe1cd2620848dd61425c49eed" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "typenum" version = "1.16.0" diff --git a/Cargo.toml b/Cargo.toml index 670e3241d51d..4f42203683d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -184,6 +184,7 @@ tracing-error = "0.2.0" tracing-opentelemetry = "0.21.0" tracing-subscriber = { version = "0.3", default-features = false, features = ["smallvec", "fmt", "tracing-log", "std", "env-filter", "json", "ansi"] } twox-hash = { version = "1.6.3", default-features = false } +typed-json = "0.1" url = "2.2" urlencoding = "2.1" uuid = { version = "1.6.1", features = ["v4", "v7", "serde"] } diff --git a/proxy/Cargo.toml b/proxy/Cargo.toml index 288f7769fef3..2f18b5fbc6cf 100644 --- a/proxy/Cargo.toml +++ b/proxy/Cargo.toml @@ -92,6 +92,7 @@ tracing-opentelemetry.workspace = true tracing-subscriber.workspace = true tracing-utils.workspace = true tracing.workspace = true +typed-json.workspace = true url.workspace = true urlencoding.workspace = true utils.workspace = true diff --git a/proxy/src/serverless/sql_over_http.rs b/proxy/src/serverless/sql_over_http.rs index 8118ae5ea89d..6400e4ac7b3a 100644 --- a/proxy/src/serverless/sql_over_http.rs +++ b/proxy/src/serverless/sql_over_http.rs @@ -18,7 +18,7 @@ use hyper1::Response; use hyper1::StatusCode; use hyper1::{HeaderMap, Request}; use pq_proto::StartupMessageParamsBuilder; -use serde_json::json; +use serde::Serialize; use serde_json::Value; use tokio::time; use tokio_postgres::error::DbError; @@ -32,6 +32,7 @@ use tokio_postgres::Transaction; use tokio_util::sync::CancellationToken; use tracing::error; use tracing::info; +use typed_json::json; use url::Url; use utils::http::error::ApiError; @@ -263,13 +264,8 @@ pub async fn handle( | SqlOverHttpError::Postgres(e) => e.as_db_error(), _ => None, }; - fn get<'a, T: serde::Serialize>( - db: Option<&'a DbError>, - x: impl FnOnce(&'a DbError) -> T, - ) -> Value { - db.map(x) - .and_then(|t| serde_json::to_value(t).ok()) - .unwrap_or_default() + fn get<'a, T: Default>(db: Option<&'a DbError>, x: impl FnOnce(&'a DbError) -> T) -> T { + db.map(x).unwrap_or_default() } if let Some(db_error) = db_error { @@ -278,17 +274,11 @@ pub async fn handle( let position = db_error.and_then(|db| db.position()); let (position, internal_position, internal_query) = match position { - Some(ErrorPosition::Original(position)) => ( - Value::String(position.to_string()), - Value::Null, - Value::Null, - ), - Some(ErrorPosition::Internal { position, query }) => ( - Value::Null, - Value::String(position.to_string()), - Value::String(query.clone()), - ), - None => (Value::Null, Value::Null, Value::Null), + Some(ErrorPosition::Original(position)) => (Some(position.to_string()), None, None), + Some(ErrorPosition::Internal { position, query }) => { + (None, Some(position.to_string()), Some(query.clone())) + } + None => (None, None, None), }; let code = get(db_error, |db| db.code().code()); @@ -578,10 +568,8 @@ async fn handle_inner( .status(StatusCode::OK) .header(header::CONTENT_TYPE, "application/json"); - // - // Now execute the query and return the result - // - let result = match payload { + // Now execute the query and return the result. + let json_output = match payload { Payload::Single(stmt) => stmt.process(cancel, &mut client, parsed_headers).await?, Payload::Batch(statements) => { if parsed_headers.txn_read_only { @@ -605,11 +593,9 @@ async fn handle_inner( let metrics = client.metrics(); - // how could this possibly fail - let body = serde_json::to_string(&result).expect("json serialization should not fail"); - let len = body.len(); + let len = json_output.len(); let response = response - .body(Full::new(Bytes::from(body))) + .body(Full::new(Bytes::from(json_output))) // only fails if invalid status code or invalid header/values are given. // these are not user configurable so it cannot fail dynamically .expect("building response payload should not fail"); @@ -631,7 +617,7 @@ impl QueryData { cancel: CancellationToken, client: &mut Client, parsed_headers: HttpHeaders, - ) -> Result { + ) -> Result { let (inner, mut discard) = client.inner(); let cancel_token = inner.cancel_token(); @@ -644,7 +630,10 @@ impl QueryData { // The query successfully completed. Either::Left((Ok((status, results)), __not_yet_cancelled)) => { discard.check_idle(status); - Ok(results) + + let json_output = + serde_json::to_string(&results).expect("json serialization should not fail"); + Ok(json_output) } // The query failed with an error Either::Left((Err(e), __not_yet_cancelled)) => { @@ -662,7 +651,10 @@ impl QueryData { // query successed before it was cancelled. Ok(Ok((status, results))) => { discard.check_idle(status); - Ok(results) + + let json_output = serde_json::to_string(&results) + .expect("json serialization should not fail"); + Ok(json_output) } // query failed or was cancelled. Ok(Err(error)) => { @@ -696,7 +688,7 @@ impl BatchQueryData { cancel: CancellationToken, client: &mut Client, parsed_headers: HttpHeaders, - ) -> Result { + ) -> Result { info!("starting transaction"); let (inner, mut discard) = client.inner(); let cancel_token = inner.cancel_token(); @@ -718,9 +710,9 @@ impl BatchQueryData { e })?; - let results = + let json_output = match query_batch(cancel.child_token(), &transaction, self, parsed_headers).await { - Ok(results) => { + Ok(json_output) => { info!("commit"); let status = transaction.commit().await.map_err(|e| { // if we cannot commit - for now don't return connection to pool @@ -729,7 +721,7 @@ impl BatchQueryData { e })?; discard.check_idle(status); - results + json_output } Err(SqlOverHttpError::Cancelled(_)) => { if let Err(err) = cancel_token.cancel_query(NoTls).await { @@ -753,7 +745,7 @@ impl BatchQueryData { } }; - Ok(json!({ "results": results })) + Ok(json_output) } } @@ -762,7 +754,7 @@ async fn query_batch( transaction: &Transaction<'_>, queries: BatchQueryData, parsed_headers: HttpHeaders, -) -> Result, SqlOverHttpError> { +) -> Result { let mut results = Vec::with_capacity(queries.queries.len()); let mut current_size = 0; for stmt in queries.queries { @@ -787,7 +779,11 @@ async fn query_batch( } } } - Ok(results) + + let results = json!({ "results": results }); + let json_output = serde_json::to_string(&results).expect("json serialization should not fail"); + + Ok(json_output) } async fn query_to_json( @@ -795,7 +791,7 @@ async fn query_to_json( data: QueryData, current_size: &mut usize, parsed_headers: HttpHeaders, -) -> Result<(ReadyForQueryStatus, Value), SqlOverHttpError> { +) -> Result<(ReadyForQueryStatus, impl Serialize), SqlOverHttpError> { info!("executing query"); let query_params = data.params; let mut row_stream = std::pin::pin!(client.query_raw_txt(&data.query, query_params).await?); @@ -844,8 +840,8 @@ async fn query_to_json( for c in row_stream.columns() { fields.push(json!({ - "name": Value::String(c.name().to_owned()), - "dataTypeID": Value::Number(c.type_().oid().into()), + "name": c.name().to_owned(), + "dataTypeID": c.type_().oid(), "tableID": c.table_oid(), "columnID": c.column_id(), "dataTypeSize": c.type_size(), @@ -863,15 +859,14 @@ async fn query_to_json( .map(|row| pg_text_row_to_json(row, &columns, parsed_headers.raw_output, array_mode)) .collect::, _>>()?; - // resulting JSON format is based on the format of node-postgres result - Ok(( - ready, - json!({ - "command": command_tag_name, - "rowCount": command_tag_count, - "rows": rows, - "fields": fields, - "rowAsArray": array_mode, - }), - )) + // Resulting JSON format is based on the format of node-postgres result. + let results = json!({ + "command": command_tag_name.to_string(), + "rowCount": command_tag_count, + "rows": rows, + "fields": fields, + "rowAsArray": array_mode, + }); + + Ok((ready, results)) } From 537ecf45f87819fe95bf43b6ded7ef5a2c15d80f Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Thu, 11 Jul 2024 16:35:31 +0300 Subject: [PATCH 151/194] Fix test_timeline_copy flakiness. fixes https://github.com/neondatabase/neon/issues/8355 --- safekeeper/src/copy_timeline.rs | 10 ++++++++-- test_runner/regress/test_wal_acceptor.py | 5 +++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/safekeeper/src/copy_timeline.rs b/safekeeper/src/copy_timeline.rs index 14bd3c03b810..220988c3ce14 100644 --- a/safekeeper/src/copy_timeline.rs +++ b/safekeeper/src/copy_timeline.rs @@ -74,10 +74,16 @@ pub async fn handle_request(request: Request) -> Result<()> { assert!(flush_lsn >= start_lsn); if request.until_lsn > flush_lsn { - bail!("requested LSN is beyond the end of the timeline"); + bail!(format!( + "requested LSN {} is beyond the end of the timeline {}", + request.until_lsn, flush_lsn + )); } if request.until_lsn < start_lsn { - bail!("requested LSN is before the start of the timeline"); + bail!(format!( + "requested LSN {} is before the start of the timeline {}", + request.until_lsn, start_lsn + )); } if request.until_lsn > commit_lsn { diff --git a/test_runner/regress/test_wal_acceptor.py b/test_runner/regress/test_wal_acceptor.py index e0ad4fdd5cf9..2e906e616051 100644 --- a/test_runner/regress/test_wal_acceptor.py +++ b/test_runner/regress/test_wal_acceptor.py @@ -2065,6 +2065,11 @@ def remember_lsn(): log.info(f"Original digest: {orig_digest}") for sk in env.safekeepers: + wait( + partial(is_flush_lsn_caught_up, sk, tenant_id, timeline_id, lsn), + f"sk_id={sk.id} to flush {lsn}", + ) + sk.http_client().copy_timeline( tenant_id, timeline_id, From 72c2d0812ee860354f1554ac76533ca02fe58237 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Mon, 15 Jul 2024 16:33:56 +0200 Subject: [PATCH 152/194] remove page_service `show ` (#8372) This operation isn't used in practice, so let's remove it. Context: in https://github.com/neondatabase/neon/pull/8339 --- pageserver/src/metrics.rs | 1 - pageserver/src/page_service.rs | 60 ---------------- test_runner/regress/test_auth.py | 2 +- test_runner/regress/test_tenant_conf.py | 96 ++----------------------- 4 files changed, 5 insertions(+), 154 deletions(-) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 9b3bb481b9ae..abad4b44b802 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -1490,7 +1490,6 @@ pub(crate) enum ComputeCommandKind { Basebackup, Fullbackup, LeaseLsn, - Show, } pub(crate) struct ComputeCommandCounters { diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index f94b0d335e8e..00147a8ca6cd 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -1479,66 +1479,6 @@ where ))? } }; - } else if let Some(params) = parts.strip_prefix(&["show"]) { - // show - if params.len() != 1 { - return Err(QueryError::Other(anyhow::anyhow!( - "invalid param number for config command" - ))); - } - let tenant_id = TenantId::from_str(params[0]) - .with_context(|| format!("Failed to parse tenant id from {}", params[0]))?; - - tracing::Span::current().record("tenant_id", field::display(tenant_id)); - - self.check_permission(Some(tenant_id))?; - - COMPUTE_COMMANDS_COUNTERS - .for_command(ComputeCommandKind::Show) - .inc(); - - let tenant = self - .get_active_tenant_with_timeout( - tenant_id, - ShardSelector::Zero, - ACTIVE_TENANT_TIMEOUT, - ) - .await?; - pgb.write_message_noflush(&BeMessage::RowDescription(&[ - RowDescriptor::int8_col(b"checkpoint_distance"), - RowDescriptor::int8_col(b"checkpoint_timeout"), - RowDescriptor::int8_col(b"compaction_target_size"), - RowDescriptor::int8_col(b"compaction_period"), - RowDescriptor::int8_col(b"compaction_threshold"), - RowDescriptor::int8_col(b"gc_horizon"), - RowDescriptor::int8_col(b"gc_period"), - RowDescriptor::int8_col(b"image_creation_threshold"), - RowDescriptor::int8_col(b"pitr_interval"), - ]))? - .write_message_noflush(&BeMessage::DataRow(&[ - Some(tenant.get_checkpoint_distance().to_string().as_bytes()), - Some( - tenant - .get_checkpoint_timeout() - .as_secs() - .to_string() - .as_bytes(), - ), - Some(tenant.get_compaction_target_size().to_string().as_bytes()), - Some( - tenant - .get_compaction_period() - .as_secs() - .to_string() - .as_bytes(), - ), - Some(tenant.get_compaction_threshold().to_string().as_bytes()), - Some(tenant.get_gc_horizon().to_string().as_bytes()), - Some(tenant.get_gc_period().as_secs().to_string().as_bytes()), - Some(tenant.get_image_creation_threshold().to_string().as_bytes()), - Some(tenant.get_pitr_interval().as_secs().to_string().as_bytes()), - ]))? - .write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?; } else { return Err(QueryError::Other(anyhow::anyhow!( "unknown command {query_string}" diff --git a/test_runner/regress/test_auth.py b/test_runner/regress/test_auth.py index 922a21a99929..7cb85e3dd1b2 100644 --- a/test_runner/regress/test_auth.py +++ b/test_runner/regress/test_auth.py @@ -211,7 +211,7 @@ def op(): def check_pageserver(expect_success: bool, **conn_kwargs): check_connection( env.pageserver, - f"show {env.initial_tenant}", + f"pagestream {env.initial_tenant} {env.initial_timeline}", expect_success, **conn_kwargs, ) diff --git a/test_runner/regress/test_tenant_conf.py b/test_runner/regress/test_tenant_conf.py index 80fb2b55b8b2..1a8bc3b98363 100644 --- a/test_runner/regress/test_tenant_conf.py +++ b/test_runner/regress/test_tenant_conf.py @@ -1,10 +1,7 @@ import json -from contextlib import closing from typing import Any, Dict -import psycopg2.extras from fixtures.common_types import Lsn -from fixtures.log_helper import log from fixtures.neon_fixtures import ( NeonEnvBuilder, ) @@ -63,25 +60,6 @@ def set_some_nondefault_global_config(ps_cfg: Dict[str, Any]): # check the configuration of the default tenant # it should match global configuration - with closing(env.pageserver.connect()) as psconn: - with psconn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as pscur: - log.info(f"show {env.initial_tenant}") - pscur.execute(f"show {env.initial_tenant}") - res = pscur.fetchone() - assert res is not None - assert all( - i in res.items() - for i in { - "checkpoint_distance": 10000, - "compaction_target_size": 1048576, - "compaction_period": 20, - "compaction_threshold": 10, - "gc_horizon": 67108864, - "gc_period": 60 * 60, - "image_creation_threshold": 3, - "pitr_interval": 604800, # 7 days - }.items() - ), f"Unexpected res: {res}" default_tenant_config = http_client.tenant_config(tenant_id=env.initial_tenant) assert ( not default_tenant_config.tenant_specific_overrides @@ -103,25 +81,6 @@ def set_some_nondefault_global_config(ps_cfg: Dict[str, Any]): } # check the configuration of the new tenant - with closing(env.pageserver.connect()) as psconn: - with psconn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as pscur: - pscur.execute(f"show {tenant}") - res = pscur.fetchone() - log.info(f"res: {res}") - assert res is not None - assert all( - i in res.items() - for i in { - "checkpoint_distance": 20000, - "compaction_target_size": 1048576, - "compaction_period": 20, - "compaction_threshold": 10, - "gc_horizon": 67108864, - "gc_period": 30, - "image_creation_threshold": 3, - "pitr_interval": 604800, - }.items() - ), f"Unexpected res: {res}" new_tenant_config = http_client.tenant_config(tenant_id=tenant) new_specific_config = new_tenant_config.tenant_specific_overrides assert new_specific_config["checkpoint_distance"] == 20000 @@ -166,25 +125,6 @@ def set_some_nondefault_global_config(ps_cfg: Dict[str, Any]): conf=conf_update, ) - with closing(env.pageserver.connect()) as psconn: - with psconn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as pscur: - pscur.execute(f"show {tenant}") - res = pscur.fetchone() - log.info(f"after config res: {res}") - assert res is not None - assert all( - i in res.items() - for i in { - "checkpoint_distance": 15000, - "compaction_target_size": 1048576, - "compaction_period": 80, - "compaction_threshold": 10, - "gc_horizon": 67108864, - "gc_period": 80, - "image_creation_threshold": 2, - "pitr_interval": 604800, - }.items() - ), f"Unexpected res: {res}" updated_tenant_config = http_client.tenant_config(tenant_id=tenant) updated_specific_config = updated_tenant_config.tenant_specific_overrides assert updated_specific_config["checkpoint_distance"] == 15000 @@ -222,25 +162,6 @@ def set_some_nondefault_global_config(ps_cfg: Dict[str, Any]): env.pageserver.stop() env.pageserver.start() - with closing(env.pageserver.connect()) as psconn: - with psconn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as pscur: - pscur.execute(f"show {tenant}") - res = pscur.fetchone() - log.info(f"after restart res: {res}") - assert res is not None - assert all( - i in res.items() - for i in { - "checkpoint_distance": 15000, - "compaction_target_size": 1048576, - "compaction_period": 80, - "compaction_threshold": 10, - "gc_horizon": 67108864, - "gc_period": 80, - "image_creation_threshold": 2, - "pitr_interval": 604800, - }.items() - ), f"Unexpected res: {res}" restarted_tenant_config = http_client.tenant_config(tenant_id=tenant) assert ( restarted_tenant_config == updated_tenant_config @@ -283,19 +204,10 @@ def set_some_nondefault_global_config(ps_cfg: Dict[str, Any]): env.pageserver.stop() env.pageserver.start() - with closing(env.pageserver.connect()) as psconn: - with psconn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as pscur: - pscur.execute(f"show {tenant}") - res = pscur.fetchone() - log.info(f"after restart res: {res}") - assert res is not None - assert all( - i in res.items() - for i in { - "compaction_period": 20, - "pitr_interval": 60, - }.items() - ), f"Unexpected res: {res}" + restarted_final_tenant_config = http_client.tenant_config(tenant_id=tenant) + assert ( + restarted_final_tenant_config == final_tenant_config + ), "Updated config should not change after the restart" def test_creating_tenant_conf_after_attach(neon_env_builder: NeonEnvBuilder): From cfaf30f5e84583907678e07d2df65c68dec47930 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Mon, 15 Jul 2024 18:08:24 +0300 Subject: [PATCH 153/194] feat(storcon): timeline detach ancestor passthrough (#8353) Currently storage controller does not support forwarding timeline detach ancestor requests to pageservers. Add support for forwarding `PUT .../:tenant_id/timelines/:timeline_id/detach_ancestor`. Implement the support mostly as is, because the timeline detach ancestor will be made (mostly) idempotent in future PR. Cc: #6994 --- .../src/models/detach_ancestor.rs | 2 +- pageserver/client/src/mgmt_api.rs | 18 +++ storage_controller/src/http.rs | 26 ++++ storage_controller/src/pageserver_client.rs | 22 ++- storage_controller/src/service.rs | 140 ++++++++++++++++-- test_runner/fixtures/neon_fixtures.py | 2 +- .../regress/test_timeline_detach_ancestor.py | 97 +++++++++++- 7 files changed, 281 insertions(+), 26 deletions(-) diff --git a/libs/pageserver_api/src/models/detach_ancestor.rs b/libs/pageserver_api/src/models/detach_ancestor.rs index fc1f10e7345f..ae5a21bab91c 100644 --- a/libs/pageserver_api/src/models/detach_ancestor.rs +++ b/libs/pageserver_api/src/models/detach_ancestor.rs @@ -1,6 +1,6 @@ use utils::id::TimelineId; -#[derive(Default, serde::Serialize)] +#[derive(Debug, Default, PartialEq, serde::Serialize, serde::Deserialize)] pub struct AncestorDetached { pub reparented_timelines: Vec, } diff --git a/pageserver/client/src/mgmt_api.rs b/pageserver/client/src/mgmt_api.rs index e3ddb446fa2c..ac3ff1bb896a 100644 --- a/pageserver/client/src/mgmt_api.rs +++ b/pageserver/client/src/mgmt_api.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use bytes::Bytes; +use detach_ancestor::AncestorDetached; use pageserver_api::{models::*, shard::TenantShardId}; use reqwest::{IntoUrl, Method, StatusCode}; use utils::{ @@ -418,6 +419,23 @@ impl Client { } } + pub async fn timeline_detach_ancestor( + &self, + tenant_shard_id: TenantShardId, + timeline_id: TimelineId, + ) -> Result { + let uri = format!( + "{}/v1/tenant/{tenant_shard_id}/timeline/{timeline_id}/detach_ancestor", + self.mgmt_api_endpoint + ); + + self.request(Method::PUT, &uri, ()) + .await? + .json() + .await + .map_err(Error::ReceiveBody) + } + pub async fn tenant_reset(&self, tenant_shard_id: TenantShardId) -> Result<()> { let uri = format!( "{}/v1/tenant/{}/reset", diff --git a/storage_controller/src/http.rs b/storage_controller/src/http.rs index 3a62c0dd4ffb..9ddf98eb3bb6 100644 --- a/storage_controller/src/http.rs +++ b/storage_controller/src/http.rs @@ -330,6 +330,22 @@ async fn handle_tenant_timeline_delete( .await } +async fn handle_tenant_timeline_detach_ancestor( + service: Arc, + req: Request, +) -> Result, ApiError> { + let tenant_id: TenantId = parse_request_param(&req, "tenant_id")?; + check_permissions(&req, Scope::PageServerApi)?; + + let timeline_id: TimelineId = parse_request_param(&req, "timeline_id")?; + + let res = service + .tenant_timeline_detach_ancestor(tenant_id, timeline_id) + .await?; + + json_response(StatusCode::OK, res) +} + async fn handle_tenant_timeline_passthrough( service: Arc, req: Request, @@ -1006,6 +1022,16 @@ pub fn make_router( RequestName("v1_tenant_timeline"), ) }) + .put( + "/v1/tenant/:tenant_id/timeline/:timeline_id/detach_ancestor", + |r| { + tenant_service_handler( + r, + handle_tenant_timeline_detach_ancestor, + RequestName("v1_tenant_timeline_detach_ancestor"), + ) + }, + ) // Tenant detail GET passthrough to shard zero: .get("/v1/tenant/:tenant_id", |r| { tenant_service_handler( diff --git a/storage_controller/src/pageserver_client.rs b/storage_controller/src/pageserver_client.rs index 769aba80cad1..8d64201cd939 100644 --- a/storage_controller/src/pageserver_client.rs +++ b/storage_controller/src/pageserver_client.rs @@ -1,8 +1,9 @@ use pageserver_api::{ models::{ - LocationConfig, LocationConfigListResponse, PageserverUtilization, SecondaryProgress, - TenantScanRemoteStorageResponse, TenantShardSplitRequest, TenantShardSplitResponse, - TimelineCreateRequest, TimelineInfo, TopTenantShardsRequest, TopTenantShardsResponse, + detach_ancestor::AncestorDetached, LocationConfig, LocationConfigListResponse, + PageserverUtilization, SecondaryProgress, TenantScanRemoteStorageResponse, + TenantShardSplitRequest, TenantShardSplitResponse, TimelineCreateRequest, TimelineInfo, + TopTenantShardsRequest, TopTenantShardsResponse, }, shard::TenantShardId, }; @@ -226,6 +227,21 @@ impl PageserverClient { ) } + pub(crate) async fn timeline_detach_ancestor( + &self, + tenant_shard_id: TenantShardId, + timeline_id: TimelineId, + ) -> Result { + measured_request!( + "timeline_detach_ancestor", + crate::metrics::Method::Put, + &self.node_id_label, + self.inner + .timeline_detach_ancestor(tenant_shard_id, timeline_id) + .await + ) + } + pub(crate) async fn get_utilization(&self) -> Result { measured_request!( "utilization", diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index deaac83ea526..95522525cb6e 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -117,6 +117,7 @@ enum TenantOperations { TimelineCreate, TimelineDelete, AttachHook, + TimelineDetachAncestor, } #[derive(Clone, strum_macros::Display)] @@ -2376,18 +2377,18 @@ impl Service { tracing::info!("Doing time travel recovery for shard {tenant_shard_id}",); client - .tenant_time_travel_remote_storage( - tenant_shard_id, - ×tamp, - &done_if_after, - ) - .await - .map_err(|e| { - ApiError::InternalServerError(anyhow::anyhow!( - "Error doing time travel recovery for shard {tenant_shard_id} on node {}: {e}", - node - )) - })?; + .tenant_time_travel_remote_storage( + tenant_shard_id, + ×tamp, + &done_if_after, + ) + .await + .map_err(|e| { + ApiError::InternalServerError(anyhow::anyhow!( + "Error doing time travel recovery for shard {tenant_shard_id} on node {}: {e}", + node + )) + })?; } } Ok(()) @@ -2757,7 +2758,7 @@ impl Service { // Create timeline on remaining shards with number >0 if !targets.is_empty() { // If we had multiple shards, issue requests for the remainder now. - let jwt = self.config.jwt_token.clone(); + let jwt = &self.config.jwt_token; self.tenant_for_shards(targets, |tenant_shard_id: TenantShardId, node: Node| { let create_req = create_req.clone(); Box::pin(create_one(tenant_shard_id, node, jwt.clone(), create_req)) @@ -2768,6 +2769,115 @@ impl Service { Ok(timeline_info) } + pub(crate) async fn tenant_timeline_detach_ancestor( + &self, + tenant_id: TenantId, + timeline_id: TimelineId, + ) -> Result { + tracing::info!("Detaching timeline {tenant_id}/{timeline_id}",); + + let _tenant_lock = trace_shared_lock( + &self.tenant_op_locks, + tenant_id, + TenantOperations::TimelineDetachAncestor, + ) + .await; + + self.ensure_attached_wait(tenant_id).await?; + + let targets = { + let locked = self.inner.read().unwrap(); + let mut targets = Vec::new(); + + for (tenant_shard_id, shard) in + locked.tenants.range(TenantShardId::tenant_range(tenant_id)) + { + let node_id = shard.intent.get_attached().ok_or_else(|| { + ApiError::InternalServerError(anyhow::anyhow!("Shard not scheduled")) + })?; + let node = locked + .nodes + .get(&node_id) + .expect("Pageservers may not be deleted while referenced"); + + targets.push((*tenant_shard_id, node.clone())); + } + targets + }; + + if targets.is_empty() { + return Err(ApiError::NotFound( + anyhow::anyhow!("Tenant not found").into(), + )); + } + + async fn detach_one( + tenant_shard_id: TenantShardId, + timeline_id: TimelineId, + node: Node, + jwt: Option, + ) -> Result<(ShardNumber, models::detach_ancestor::AncestorDetached), ApiError> { + tracing::info!( + "Detaching timeline on shard {tenant_shard_id}/{timeline_id}, attached to node {node}", + ); + + let client = PageserverClient::new(node.get_id(), node.base_url(), jwt.as_deref()); + client + .timeline_detach_ancestor(tenant_shard_id, timeline_id) + .await + .map_err(|e| { + use mgmt_api::Error; + + match e { + // no ancestor (ever) + Error::ApiError(StatusCode::CONFLICT, msg) => { + ApiError::Conflict(format!("{node}: {msg}")) + } + // too many ancestors + Error::ApiError(StatusCode::BAD_REQUEST, msg) => { + ApiError::BadRequest(anyhow::anyhow!("{node}: {msg}")) + } + // rest can be mapped + other => passthrough_api_error(&node, other), + } + }) + .map(|res| (tenant_shard_id.shard_number, res)) + } + + // no shard needs to go first/last; the operation should be idempotent + // TODO: it would be great to ensure that all shards return the same error + let mut results = self + .tenant_for_shards(targets, |tenant_shard_id, node| { + futures::FutureExt::boxed(detach_one( + tenant_shard_id, + timeline_id, + node, + self.config.jwt_token.clone(), + )) + }) + .await?; + + let any = results.pop().expect("we must have at least one response"); + + // FIXME: the ordering is not stable yet on pageserver, should be (ancestor_lsn, + // TimelineId) + let mismatching = results + .iter() + .filter(|(_, res)| res != &any.1) + .collect::>(); + if !mismatching.is_empty() { + let matching = results.len() - mismatching.len(); + tracing::error!( + matching, + compared_against=?any, + ?mismatching, + "shards returned different results" + ); + } + + Ok(any.1) + } + /// Helper for concurrently calling a pageserver API on a number of shards, such as timeline creation. /// /// On success, the returned vector contains exactly the same number of elements as the input `locations`. @@ -2894,8 +3004,8 @@ impl Service { .await .map_err(|e| { ApiError::InternalServerError(anyhow::anyhow!( - "Error deleting timeline {timeline_id} on {tenant_shard_id} on node {node}: {e}", - )) + "Error deleting timeline {timeline_id} on {tenant_shard_id} on node {node}: {e}", + )) }) } diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 463e4a3b012e..90ed838e1db3 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2400,7 +2400,7 @@ def tenant_create( def locate(self, tenant_id: TenantId) -> list[dict[str, Any]]: """ - :return: list of {"shard_id": "", "node_id": int, "listen_pg_addr": str, "listen_pg_port": int, "listen_http_addr: str, "listen_http_port: int} + :return: list of {"shard_id": "", "node_id": int, "listen_pg_addr": str, "listen_pg_port": int, "listen_http_addr": str, "listen_http_port": int} """ response = self.request( "GET", diff --git a/test_runner/regress/test_timeline_detach_ancestor.py b/test_runner/regress/test_timeline_detach_ancestor.py index 606ce203cdc6..803fcac58357 100644 --- a/test_runner/regress/test_timeline_detach_ancestor.py +++ b/test_runner/regress/test_timeline_detach_ancestor.py @@ -11,11 +11,12 @@ from fixtures.neon_fixtures import ( NeonEnvBuilder, PgBin, + flush_ep_to_pageserver, wait_for_last_flush_lsn, ) from fixtures.pageserver.http import HistoricLayerInfo, PageserverApiException -from fixtures.pageserver.utils import wait_timeline_detail_404 -from fixtures.remote_storage import LocalFsStorage +from fixtures.pageserver.utils import wait_for_last_record_lsn, wait_timeline_detail_404 +from fixtures.remote_storage import LocalFsStorage, RemoteStorageKind from fixtures.utils import assert_pageserver_backups_equal @@ -559,11 +560,24 @@ def delta_layers(timeline_id: TimelineId): assert_pageserver_backups_equal(fullbackup_before, fullbackup_after, set()) -def test_timeline_ancestor_errors(neon_env_builder: NeonEnvBuilder): - env = neon_env_builder.init_start() - env.pageserver.allowed_errors.extend(SHUTDOWN_ALLOWED_ERRORS) +@pytest.mark.parametrize("sharded", [True, False]) +def test_timeline_ancestor_detach_errors(neon_env_builder: NeonEnvBuilder, sharded: bool): + shards = 2 if sharded else 1 - client = env.pageserver.http_client() + neon_env_builder.num_pageservers = shards + env = neon_env_builder.init_start(initial_tenant_shard_count=shards if sharded else None) + + pageservers = dict((int(p.id), p) for p in env.pageservers) + + for ps in pageservers.values(): + ps.allowed_errors.extend(SHUTDOWN_ALLOWED_ERRORS) + + if sharded: + # FIXME: should this be in the neon_env_builder.init_start? + env.storage_controller.reconcile_until_idle() + client = env.storage_controller.pageserver_api() + else: + client = env.pageserver.http_client() with pytest.raises(PageserverApiException, match=".* no ancestors") as info: client.detach_ancestor(env.initial_tenant, env.initial_timeline) @@ -577,6 +591,17 @@ def test_timeline_ancestor_errors(neon_env_builder: NeonEnvBuilder): client.detach_ancestor(env.initial_tenant, second_branch) assert info.value.status_code == 400 + client.detach_ancestor(env.initial_tenant, first_branch) + + # FIXME: this should be done by the http req handler + for ps in pageservers.values(): + ps.quiesce_tenants() + + with pytest.raises(PageserverApiException, match=".* no ancestors") as info: + client.detach_ancestor(env.initial_tenant, first_branch) + # FIXME: this should be 200 OK because we've already completed it + assert info.value.status_code == 409 + client.tenant_delete(env.initial_tenant) with pytest.raises(PageserverApiException) as e: @@ -584,6 +609,58 @@ def test_timeline_ancestor_errors(neon_env_builder: NeonEnvBuilder): assert e.value.status_code == 404 +def test_sharded_timeline_detach_ancestor(neon_env_builder: NeonEnvBuilder): + branch_name = "soon_detached" + shard_count = 4 + neon_env_builder.num_pageservers = shard_count + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.MOCK_S3) + + env = neon_env_builder.init_start(initial_tenant_shard_count=shard_count) + for ps in env.pageservers: + ps.allowed_errors.extend(SHUTDOWN_ALLOWED_ERRORS) + + # FIXME: should this be in the neon_env_builder.init_start? + env.storage_controller.reconcile_until_idle() + shards = env.storage_controller.locate(env.initial_tenant) + + branch_timeline_id = env.neon_cli.create_branch(branch_name, tenant_id=env.initial_tenant) + + with env.endpoints.create_start(branch_name, tenant_id=env.initial_tenant) as ep: + ep.safe_psql( + "create table foo as select 1::bigint, i::bigint from generate_series(1, 10000) v(i)" + ) + lsn = flush_ep_to_pageserver(env, ep, env.initial_tenant, branch_timeline_id) + + pageservers = dict((int(p.id), p) for p in env.pageservers) + + for shard_info in shards: + node_id = int(shard_info["node_id"]) + shard_id = shard_info["shard_id"] + detail = pageservers[node_id].http_client().timeline_detail(shard_id, branch_timeline_id) + + assert Lsn(detail["last_record_lsn"]) >= lsn + assert Lsn(detail["initdb_lsn"]) < lsn + assert TimelineId(detail["ancestor_timeline_id"]) == env.initial_timeline + + env.storage_controller.pageserver_api().detach_ancestor(env.initial_tenant, branch_timeline_id) + + for shard_info in shards: + node_id = int(shard_info["node_id"]) + shard_id = shard_info["shard_id"] + + # TODO: ensure quescing is done on pageserver? + pageservers[node_id].quiesce_tenants() + detail = pageservers[node_id].http_client().timeline_detail(shard_id, branch_timeline_id) + wait_for_last_record_lsn( + pageservers[node_id].http_client(), shard_id, branch_timeline_id, lsn + ) + assert detail.get("ancestor_timeline_id") is None + + with env.endpoints.create_start(branch_name, tenant_id=env.initial_tenant) as ep: + count = int(ep.safe_psql("select count(*) from foo")[0][0]) + assert count == 10000 + + # TODO: # - after starting the operation, tenant is deleted # - after starting the operation, pageserver is shutdown, restarted @@ -591,3 +668,11 @@ def test_timeline_ancestor_errors(neon_env_builder: NeonEnvBuilder): # - deletion of reparented while reparenting should fail once, then succeed (?) # - branch near existing L1 boundary, image layers? # - investigate: why are layers started at uneven lsn? not just after branching, but in general. +# +# TEST: 1. tad which partially succeeds, one returns 500 +# 2. create branch below timeline? or delete timeline below +# 3. on retry all should report the same reparented timelines +# +# TEST: 1. tad is started, one node stalls, other restarts +# 2. client timeout before stall over +# 3. on retry with stalled and other being able to proceed From 2a3a1364746e8cd5bfab82308bba5fe84feb7ff3 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 15 Jul 2024 17:43:05 +0100 Subject: [PATCH 154/194] pageserver: use PITR GC cutoffs as authoritative (#8365) ## Problem Pageserver GC uses a size-based condition (GC "horizon" in addition to time-based "PITR"). Eventually we plan to retire the size-based condition: https://github.com/neondatabase/neon/issues/6374 Currently, we always apply the more conservative of the two, meaning that tenants always retain at least 64MB of history (default horizon), even after a very long time has passed. This is particularly acute in cases where someone has dropped tables/databases, and then leaves a database idle: the horizon can prevent GCing very large quantities of historical data (we already account for this in synthetic size by ignoring gc horizon). We're not entirely removing GC horizon right now because we don't want to 100% rely on standby_horizon for robustness of physical replication, but we can tweak our logic to avoid retaining that 64MB LSN length indefinitely. ## Summary of changes - Rework `Timeline::find_gc_cutoffs`, with new logic: - If there is no PITR set, then use `DEFAULT_PITR_INTERVAL` (1 week) to calculate a time threshold. Retain either the horizon or up to that thresholds, whichever requires less data. - When there is a PITR set, and we have unambiguously resolved the timestamp to an LSN, then ignore the GC horizon entirely. For typical PITRs (1 day, 1 week), this will still easily retain enough data to avoid stressing read only replicas. The key property we end up with, whether a PITR is set or not, is that after enough time has passed, our GC cutoff on an idle timeline will catch up with the last_record_lsn. Using `DEFAULT_PITR_INTERVAL` is a bit of an arbitrary hack, but this feels like it isn't really worth the noise of exposing in TenantConfig. We could just make it a different named constant though. The end-end state will be that there is no gc_horizon at all, and that tenants with pitr_interval=0 would truly retain no history, so this constant would go away. --- pageserver/src/tenant/timeline.rs | 144 +++++++++++++--------- test_runner/regress/test_branch_and_gc.py | 4 +- 2 files changed, 88 insertions(+), 60 deletions(-) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index a3ddb3a1d190..0996616a670e 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -69,6 +69,7 @@ use std::{ use crate::{ aux_file::AuxFileSizeEstimator, tenant::{ + config::defaults::DEFAULT_PITR_INTERVAL, layer_map::{LayerMap, SearchResult}, metadata::TimelineMetadata, storage_layer::PersistentLayerDesc, @@ -4945,20 +4946,17 @@ impl Timeline { } /// Find the Lsns above which layer files need to be retained on - /// garbage collection. This is separate from actually performing the GC, - /// and is updated more frequently, so that compaction can remove obsolete - /// page versions more aggressively. + /// garbage collection. /// - /// TODO: that's wishful thinking, compaction doesn't actually do that - /// currently. + /// We calculate two cutoffs, one based on time and one based on WAL size. `pitr` + /// controls the time cutoff (or ZERO to disable time-based retention), and `cutoff_horizon` controls + /// the space-based retention. /// - /// The 'cutoff_horizon' point is used to retain recent versions that might still be - /// needed by read-only nodes. (As of this writing, the caller just passes - /// the latest LSN subtracted by a constant, and doesn't do anything smart - /// to figure out what read-only nodes might actually need.) - /// - /// The 'pitr' duration is used to calculate a 'pitr_cutoff', which can be used to determine - /// whether a record is needed for PITR. + /// This function doesn't simply to calculate time & space based retention: it treats time-based + /// retention as authoritative if enabled, and falls back to space-based retention if calculating + /// the LSN for a time point isn't possible. Therefore the GcCutoffs::horizon in the response might + /// be different to the `cutoff_horizon` input. Callers should treat the min() of the two cutoffs + /// in the response as the GC cutoff point for the timeline. #[instrument(skip_all, fields(timeline_id=%self.timeline_id))] pub(super) async fn find_gc_cutoffs( &self, @@ -4975,58 +4973,88 @@ impl Timeline { pausable_failpoint!("Timeline::find_gc_cutoffs-pausable"); - // First, calculate pitr_cutoff_timestamp and then convert it to LSN. - // - // Some unit tests depend on garbage-collection working even when - // CLOG data is missing, so that find_lsn_for_timestamp() doesn't - // work, so avoid calling it altogether if time-based retention is not - // configured. It would be pointless anyway. - let pitr_cutoff = if pitr != Duration::ZERO { + if cfg!(test) { + // Unit tests which specify zero PITR interval expect to avoid doing any I/O for timestamp lookup + if pitr == Duration::ZERO { + return Ok(GcCutoffs { + pitr: self.get_last_record_lsn(), + horizon: cutoff_horizon, + }); + } + } + + // Calculate a time-based limit on how much to retain: + // - if PITR interval is set, then this is our cutoff. + // - if PITR interval is not set, then we do a lookup + // based on DEFAULT_PITR_INTERVAL, so that size-based retention (horizon) + // does not result in keeping history around permanently on idle databases. + let time_cutoff = { let now = SystemTime::now(); - if let Some(pitr_cutoff_timestamp) = now.checked_sub(pitr) { - let pitr_timestamp = to_pg_timestamp(pitr_cutoff_timestamp); + let time_range = if pitr == Duration::ZERO { + humantime::parse_duration(DEFAULT_PITR_INTERVAL).expect("constant is invalid") + } else { + pitr + }; - match self - .find_lsn_for_timestamp(pitr_timestamp, cancel, ctx) - .await? - { - LsnForTimestamp::Present(lsn) => lsn, - LsnForTimestamp::Future(lsn) => { - // The timestamp is in the future. That sounds impossible, - // but what it really means is that there hasn't been - // any commits since the cutoff timestamp. - // - // In this case we should use the LSN of the most recent commit, - // which is implicitly the last LSN in the log. - debug!("future({})", lsn); - self.get_last_record_lsn() - } - LsnForTimestamp::Past(lsn) => { - debug!("past({})", lsn); - // conservative, safe default is to remove nothing, when we - // have no commit timestamp data available - *self.get_latest_gc_cutoff_lsn() - } - LsnForTimestamp::NoData(lsn) => { - debug!("nodata({})", lsn); - // conservative, safe default is to remove nothing, when we - // have no commit timestamp data available - *self.get_latest_gc_cutoff_lsn() - } + // If PITR is so large or `now` is so small that this underflows, we will retain no history (highly unexpected case) + let time_cutoff = now.checked_sub(time_range).unwrap_or(now); + let timestamp = to_pg_timestamp(time_cutoff); + + match self.find_lsn_for_timestamp(timestamp, cancel, ctx).await? { + LsnForTimestamp::Present(lsn) => Some(lsn), + LsnForTimestamp::Future(lsn) => { + // The timestamp is in the future. That sounds impossible, + // but what it really means is that there hasn't been + // any commits since the cutoff timestamp. + // + // In this case we should use the LSN of the most recent commit, + // which is implicitly the last LSN in the log. + debug!("future({})", lsn); + Some(self.get_last_record_lsn()) + } + LsnForTimestamp::Past(lsn) => { + debug!("past({})", lsn); + None + } + LsnForTimestamp::NoData(lsn) => { + debug!("nodata({})", lsn); + None } - } else { - // If we don't have enough data to convert to LSN, - // play safe and don't remove any layers. - *self.get_latest_gc_cutoff_lsn() } - } else { - // No time-based retention was configured. Interpret this as "keep no history". - self.get_last_record_lsn() }; - Ok(GcCutoffs { - horizon: cutoff_horizon, - pitr: pitr_cutoff, + Ok(match (pitr, time_cutoff) { + (Duration::ZERO, Some(time_cutoff)) => { + // PITR is not set. Retain the size-based limit, or the default time retention, + // whichever requires less data. + GcCutoffs { + pitr: std::cmp::max(time_cutoff, cutoff_horizon), + horizon: std::cmp::max(time_cutoff, cutoff_horizon), + } + } + (Duration::ZERO, None) => { + // PITR is not set, and time lookup failed + GcCutoffs { + pitr: self.get_last_record_lsn(), + horizon: cutoff_horizon, + } + } + (_, None) => { + // PITR interval is set & we didn't look up a timestamp successfully. Conservatively assume PITR + // cannot advance beyond what was already GC'd, and respect space-based retention + GcCutoffs { + pitr: *self.get_latest_gc_cutoff_lsn(), + horizon: cutoff_horizon, + } + } + (_, Some(time_cutoff)) => { + // PITR interval is set and we looked up timestamp successfully. Ignore + // size based retention and make time cutoff authoritative + GcCutoffs { + pitr: time_cutoff, + horizon: time_cutoff, + } + } }) } diff --git a/test_runner/regress/test_branch_and_gc.py b/test_runner/regress/test_branch_and_gc.py index eb503ddbfa0d..f2e3855c123e 100644 --- a/test_runner/regress/test_branch_and_gc.py +++ b/test_runner/regress/test_branch_and_gc.py @@ -65,8 +65,8 @@ def test_branch_and_gc(neon_simple_env: NeonEnv, build_type: str): "compaction_period": "1 s", "compaction_threshold": "2", "image_creation_threshold": "1", - # set PITR interval to be small, so we can do GC - "pitr_interval": "1 s", + # Disable PITR, this test will set an explicit space-based GC limit + "pitr_interval": "0 s", } ) From 957f99cad5b3137cd5d754044902e3837658b568 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Mon, 15 Jul 2024 20:47:53 +0300 Subject: [PATCH 155/194] feat(timeline_detach_ancestor): success idempotency (#8354) Right now timeline detach ancestor reports an error (409, "no ancestor") on a new attempt after successful completion. This makes it troublesome for storage controller retries. Fix it to respond with `200 OK` as if the operation had just completed quickly. Additionally, the returned timeline identifiers in the 200 OK response are now ordered so that responses between different nodes for error comparison are done by the storage controller added in #8353. Design-wise, this PR introduces a new strategy for accessing the latest uploaded IndexPart: `RemoteTimelineClient::initialized_upload_queue(&self) -> Result, NotInitialized>`. It should be a more scalable way to query the latest uploaded `IndexPart` than to add a query method for each question directly on `RemoteTimelineClient`. GC blocking will need to be introduced to make the operation fully idempotent. However, it is idempotent for the cases demonstrated by tests. Cc: #6994 --- pageserver/src/http/routes.rs | 47 +- .../src/tenant/remote_timeline_client.rs | 27 +- .../tenant/remote_timeline_client/index.rs | 26 ++ pageserver/src/tenant/timeline.rs | 8 +- .../src/tenant/timeline/detach_ancestor.rs | 130 +++++- pageserver/src/tenant/upload_queue.rs | 10 +- storage_controller/src/service.rs | 9 +- test_runner/fixtures/pageserver/http.py | 21 +- .../regress/test_timeline_detach_ancestor.py | 430 ++++++++++++++++-- 9 files changed, 633 insertions(+), 75 deletions(-) diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 6f8f3e6389d5..d7ef70477f45 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -1721,7 +1721,9 @@ async fn timeline_detach_ancestor_handler( request: Request, _cancel: CancellationToken, ) -> Result, ApiError> { - use crate::tenant::timeline::detach_ancestor::Options; + use crate::tenant::timeline::detach_ancestor; + use pageserver_api::models::detach_ancestor::AncestorDetached; + let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?; check_permission(&request, Some(tenant_shard_id.tenant_id))?; let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; @@ -1729,7 +1731,7 @@ async fn timeline_detach_ancestor_handler( let span = tracing::info_span!("detach_ancestor", tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), %timeline_id); async move { - let mut options = Options::default(); + let mut options = detach_ancestor::Options::default(); let rewrite_concurrency = parse_query_param::<_, std::num::NonZeroUsize>(&request, "rewrite_concurrency")?; @@ -1757,27 +1759,36 @@ async fn timeline_detach_ancestor_handler( let timeline = tenant.get_timeline(timeline_id, true)?; - let (_guard, prepared) = timeline + let progress = timeline .prepare_to_detach_from_ancestor(&tenant, options, ctx) .await?; - let res = state - .tenant_manager - .complete_detaching_timeline_ancestor(tenant_shard_id, timeline_id, prepared, ctx) - .await; - - match res { - Ok(reparented_timelines) => { - let resp = pageserver_api::models::detach_ancestor::AncestorDetached { + // uncomment to allow early as possible Tenant::drop + // drop(tenant); + + let resp = match progress { + detach_ancestor::Progress::Prepared(_guard, prepared) => { + // it would be great to tag the guard on to the tenant activation future + let reparented_timelines = state + .tenant_manager + .complete_detaching_timeline_ancestor( + tenant_shard_id, + timeline_id, + prepared, + ctx, + ) + .await + .context("timeline detach ancestor completion") + .map_err(ApiError::InternalServerError)?; + + AncestorDetached { reparented_timelines, - }; - - json_response(StatusCode::OK, resp) + } } - Err(e) => Err(ApiError::InternalServerError( - e.context("timeline detach completion"), - )), - } + detach_ancestor::Progress::Done(resp) => resp, + }; + + json_response(StatusCode::OK, resp) } .instrument(span) .await diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index bc9364de61d4..66b759c8e0d8 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -241,7 +241,7 @@ use self::index::IndexPart; use super::metadata::MetadataUpdate; use super::storage_layer::{Layer, LayerName, ResidentLayer}; -use super::upload_queue::SetDeletedFlagProgress; +use super::upload_queue::{NotInitialized, SetDeletedFlagProgress}; use super::Generation; pub(crate) use download::{ @@ -1930,6 +1930,31 @@ impl RemoteTimelineClient { } } } + + /// Returns an accessor which will hold the UploadQueue mutex for accessing the upload queue + /// externally to RemoteTimelineClient. + pub(crate) fn initialized_upload_queue( + &self, + ) -> Result, NotInitialized> { + let mut inner = self.upload_queue.lock().unwrap(); + inner.initialized_mut()?; + Ok(UploadQueueAccessor { inner }) + } +} + +pub(crate) struct UploadQueueAccessor<'a> { + inner: std::sync::MutexGuard<'a, UploadQueue>, +} + +impl<'a> UploadQueueAccessor<'a> { + pub(crate) fn latest_uploaded_index_part(&self) -> &IndexPart { + match &*self.inner { + UploadQueue::Initialized(x) => &x.clean.0, + UploadQueue::Uninitialized | UploadQueue::Stopped(_) => { + unreachable!("checked before constructing") + } + } + } } pub fn remote_tenant_path(tenant_shard_id: &TenantShardId) -> RemotePath { diff --git a/pageserver/src/tenant/remote_timeline_client/index.rs b/pageserver/src/tenant/remote_timeline_client/index.rs index 6233a3477e4d..b439df8edb10 100644 --- a/pageserver/src/tenant/remote_timeline_client/index.rs +++ b/pageserver/src/tenant/remote_timeline_client/index.rs @@ -176,6 +176,24 @@ pub(crate) struct Lineage { /// /// If you are adding support for detaching from a hierarchy, consider changing the ancestry /// into a `Vec<(TimelineId, Lsn)>` to be a path instead. + // FIXME: this is insufficient even for path of two timelines for future wal recovery + // purposes: + // + // assuming a "old main" which has received most of the WAL, and has a branch "new main", + // starting a bit before "old main" last_record_lsn. the current version works fine, + // because we will know to replay wal and branch at the recorded Lsn to do wal recovery. + // + // then assuming "new main" would similarly receive a branch right before its last_record_lsn, + // "new new main". the current implementation would just store ("new main", ancestor_lsn, _) + // here. however, we cannot recover from WAL using only that information, we would need the + // whole ancestry here: + // + // ```json + // [ + // ["old main", ancestor_lsn("new main"), _], + // ["new main", ancestor_lsn("new new main"), _] + // ] + // ``` #[serde(skip_serializing_if = "Option::is_none", default)] original_ancestor: Option<(TimelineId, Lsn, NaiveDateTime)>, } @@ -217,6 +235,14 @@ impl Lineage { self.original_ancestor .is_some_and(|(_, ancestor_lsn, _)| ancestor_lsn == lsn) } + + pub(crate) fn is_detached_from_original_ancestor(&self) -> bool { + self.original_ancestor.is_some() + } + + pub(crate) fn is_reparented(&self) -> bool { + !self.reparenting_history.is_empty() + } } #[cfg(test)] diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 0996616a670e..239dce878640 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -4733,13 +4733,7 @@ impl Timeline { tenant: &crate::tenant::Tenant, options: detach_ancestor::Options, ctx: &RequestContext, - ) -> Result< - ( - completion::Completion, - detach_ancestor::PreparedTimelineDetach, - ), - detach_ancestor::Error, - > { + ) -> Result { detach_ancestor::prepare(self, tenant, options, ctx).await } diff --git a/pageserver/src/tenant/timeline/detach_ancestor.rs b/pageserver/src/tenant/timeline/detach_ancestor.rs index 4fc89330ba42..49ce3db3e63d 100644 --- a/pageserver/src/tenant/timeline/detach_ancestor.rs +++ b/pageserver/src/tenant/timeline/detach_ancestor.rs @@ -10,6 +10,7 @@ use crate::{ }, virtual_file::{MaybeFatalIo, VirtualFile}, }; +use pageserver_api::models::detach_ancestor::AncestorDetached; use tokio_util::sync::CancellationToken; use tracing::Instrument; use utils::{completion, generation::Generation, http::error::ApiError, id::TimelineId, lsn::Lsn}; @@ -39,6 +40,9 @@ pub(crate) enum Error { #[error("unexpected error")] Unexpected(#[source] anyhow::Error), + + #[error("failpoint: {}", .0)] + Failpoint(&'static str), } impl From for ApiError { @@ -57,11 +61,41 @@ impl From for ApiError { | e @ Error::CopyDeltaPrefix(_) | e @ Error::UploadRewritten(_) | e @ Error::CopyFailed(_) - | e @ Error::Unexpected(_) => ApiError::InternalServerError(e.into()), + | e @ Error::Unexpected(_) + | e @ Error::Failpoint(_) => ApiError::InternalServerError(e.into()), + } + } +} + +impl From for Error { + fn from(_: crate::tenant::upload_queue::NotInitialized) -> Self { + // treat all as shutting down signals, even though that is not entirely correct + // (uninitialized state) + Error::ShuttingDown + } +} + +impl From for Error { + fn from(value: FlushLayerError) -> Self { + match value { + FlushLayerError::Cancelled => Error::ShuttingDown, + FlushLayerError::NotRunning(_) => { + // FIXME(#6424): technically statically unreachable right now, given how we never + // drop the sender + Error::ShuttingDown + } + FlushLayerError::CreateImageLayersError(_) | FlushLayerError::Other(_) => { + Error::FlushAncestor(value) + } } } } +pub(crate) enum Progress { + Prepared(completion::Completion, PreparedTimelineDetach), + Done(AncestorDetached), +} + pub(crate) struct PreparedTimelineDetach { layers: Vec, } @@ -88,7 +122,7 @@ pub(super) async fn prepare( tenant: &Tenant, options: Options, ctx: &RequestContext, -) -> Result<(completion::Completion, PreparedTimelineDetach), Error> { +) -> Result { use Error::*; let Some((ancestor, ancestor_lsn)) = detached @@ -96,15 +130,67 @@ pub(super) async fn prepare( .as_ref() .map(|tl| (tl.clone(), detached.ancestor_lsn)) else { - // TODO: check if we have already been detached; for this we need to read the stored data - // on remote client, for that we need a follow-up which makes uploads cheaper and maintains - // a projection of the commited data. + { + let accessor = detached.remote_client.initialized_upload_queue()?; + + // we are safe to inspect the latest uploaded, because we can only witness this after + // restart is complete and ancestor is no more. + let latest = accessor.latest_uploaded_index_part(); + if !latest.lineage.is_detached_from_original_ancestor() { + return Err(NoAncestor); + } + } + + // detached has previously been detached; let's inspect each of the current timelines and + // report back the timelines which have been reparented by our detach + let mut all_direct_children = tenant + .timelines + .lock() + .unwrap() + .values() + .filter(|tl| matches!(tl.ancestor_timeline.as_ref(), Some(ancestor) if Arc::ptr_eq(ancestor, detached))) + .map(|tl| (tl.ancestor_lsn, tl.clone())) + .collect::>(); + + let mut any_shutdown = false; + + all_direct_children.retain( + |(_, tl)| match tl.remote_client.initialized_upload_queue() { + Ok(accessor) => accessor + .latest_uploaded_index_part() + .lineage + .is_reparented(), + Err(_shutdownalike) => { + // not 100% a shutdown, but let's bail early not to give inconsistent results in + // sharded enviroment. + any_shutdown = true; + true + } + }, + ); + + if any_shutdown { + // it could be one or many being deleted; have client retry + return Err(Error::ShuttingDown); + } + + let mut reparented = all_direct_children; + // why this instead of hashset? there is a reason, but I've forgotten it many times. // - // the error is wrong per openapi - return Err(NoAncestor); + // maybe if this was a hashset we would not be able to distinguish some race condition. + reparented.sort_unstable_by_key(|(lsn, tl)| (*lsn, tl.timeline_id)); + + return Ok(Progress::Done(AncestorDetached { + reparented_timelines: reparented + .into_iter() + .map(|(_, tl)| tl.timeline_id) + .collect(), + })); }; if !ancestor_lsn.is_valid() { + // rare case, probably wouldn't even load + tracing::error!("ancestor is set, but ancestor_lsn is invalid, this timeline needs fixing"); return Err(NoAncestor); } @@ -131,6 +217,15 @@ pub(super) async fn prepare( let _gate_entered = detached.gate.enter().map_err(|_| ShuttingDown)?; + utils::pausable_failpoint!("timeline-detach-ancestor::before_starting_after_locking_pausable"); + + fail::fail_point!( + "timeline-detach-ancestor::before_starting_after_locking", + |_| Err(Error::Failpoint( + "timeline-detach-ancestor::before_starting_after_locking" + )) + ); + if ancestor_lsn >= ancestor.get_disk_consistent_lsn() { let span = tracing::info_span!("freeze_and_flush", ancestor_timeline_id=%ancestor.timeline_id); @@ -151,7 +246,7 @@ pub(super) async fn prepare( } }; - res.map_err(FlushAncestor)?; + res?; // we do not need to wait for uploads to complete but we do need `struct Layer`, // copying delta prefix is unsupported currently for `InMemoryLayer`. @@ -159,7 +254,7 @@ pub(super) async fn prepare( elapsed_ms = started_at.elapsed().as_millis(), "froze and flushed the ancestor" ); - Ok(()) + Ok::<_, Error>(()) } .instrument(span) .await?; @@ -283,7 +378,7 @@ pub(super) async fn prepare( let prepared = PreparedTimelineDetach { layers: new_layers }; - Ok((guard, prepared)) + Ok(Progress::Prepared(guard, prepared)) } fn partition_work( @@ -350,7 +445,11 @@ async fn copy_lsn_prefix( target_timeline: &Arc, ctx: &RequestContext, ) -> Result, Error> { - use Error::{CopyDeltaPrefix, RewrittenDeltaDownloadFailed}; + use Error::{CopyDeltaPrefix, RewrittenDeltaDownloadFailed, ShuttingDown}; + + if target_timeline.cancel.is_cancelled() { + return Err(ShuttingDown); + } tracing::debug!(%layer, %end_lsn, "copying lsn prefix"); @@ -529,7 +628,7 @@ pub(super) async fn complete( match res { Ok(Some(timeline)) => { tracing::info!(reparented=%timeline.timeline_id, "reparenting done"); - reparented.push(timeline.timeline_id); + reparented.push((timeline.ancestor_lsn, timeline.timeline_id)); } Ok(None) => { // lets just ignore this for now. one or all reparented timelines could had @@ -551,5 +650,12 @@ pub(super) async fn complete( tracing::info!("failed to reparent some candidates"); } + reparented.sort_unstable(); + + let reparented = reparented + .into_iter() + .map(|(_, timeline_id)| timeline_id) + .collect(); + Ok(reparented) } diff --git a/pageserver/src/tenant/upload_queue.rs b/pageserver/src/tenant/upload_queue.rs index 50c977a950fe..f7440ecdae12 100644 --- a/pageserver/src/tenant/upload_queue.rs +++ b/pageserver/src/tenant/upload_queue.rs @@ -228,18 +228,20 @@ impl UploadQueue { Ok(self.initialized_mut().expect("we just set it")) } - pub(crate) fn initialized_mut(&mut self) -> anyhow::Result<&mut UploadQueueInitialized> { + pub(crate) fn initialized_mut( + &mut self, + ) -> Result<&mut UploadQueueInitialized, NotInitialized> { use UploadQueue::*; match self { - Uninitialized => Err(NotInitialized::Uninitialized.into()), + Uninitialized => Err(NotInitialized::Uninitialized), Initialized(x) => { if x.shutting_down { - Err(NotInitialized::ShuttingDown.into()) + Err(NotInitialized::ShuttingDown) } else { Ok(x) } } - Stopped(_) => Err(NotInitialized::Stopped.into()), + Stopped(_) => Err(NotInitialized::Stopped), } } diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 95522525cb6e..3c24433c422a 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -2830,9 +2830,10 @@ impl Service { match e { // no ancestor (ever) - Error::ApiError(StatusCode::CONFLICT, msg) => { - ApiError::Conflict(format!("{node}: {msg}")) - } + Error::ApiError(StatusCode::CONFLICT, msg) => ApiError::Conflict(format!( + "{node}: {}", + msg.strip_prefix("Conflict: ").unwrap_or(&msg) + )), // too many ancestors Error::ApiError(StatusCode::BAD_REQUEST, msg) => { ApiError::BadRequest(anyhow::anyhow!("{node}: {msg}")) @@ -2859,8 +2860,6 @@ impl Service { let any = results.pop().expect("we must have at least one response"); - // FIXME: the ordering is not stable yet on pageserver, should be (ancestor_lsn, - // TimelineId) let mismatching = results .iter() .filter(|(_, res)| res != &any.1) diff --git a/test_runner/fixtures/pageserver/http.py b/test_runner/fixtures/pageserver/http.py index 03aee9e5c597..d66b94948a8e 100644 --- a/test_runner/fixtures/pageserver/http.py +++ b/test_runner/fixtures/pageserver/http.py @@ -172,6 +172,21 @@ def __init__( if auth_token is not None: self.headers["Authorization"] = f"Bearer {auth_token}" + def without_status_retrying(self) -> PageserverHttpClient: + retries = Retry( + status=0, + connect=5, + read=False, + backoff_factor=0.2, + status_forcelist=[], + allowed_methods=None, + remove_headers_on_redirect=[], + ) + + return PageserverHttpClient( + self.port, self.is_testing_enabled_or_skip, self.auth_token, retries + ) + @property def base_url(self) -> str: return f"http://localhost:{self.port}" @@ -814,17 +829,19 @@ def detach_ancestor( tenant_id: Union[TenantId, TenantShardId], timeline_id: TimelineId, batch_size: int | None = None, - ) -> Set[TimelineId]: + **kwargs, + ) -> List[TimelineId]: params = {} if batch_size is not None: params["batch_size"] = batch_size res = self.put( f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/detach_ancestor", params=params, + **kwargs, ) self.verbose_error(res) json = res.json() - return set(map(TimelineId, json["reparented_timelines"])) + return list(map(TimelineId, json["reparented_timelines"])) def evict_layer( self, tenant_id: Union[TenantId, TenantShardId], timeline_id: TimelineId, layer_name: str diff --git a/test_runner/regress/test_timeline_detach_ancestor.py b/test_runner/regress/test_timeline_detach_ancestor.py index 803fcac58357..d75ab4c0604f 100644 --- a/test_runner/regress/test_timeline_detach_ancestor.py +++ b/test_runner/regress/test_timeline_detach_ancestor.py @@ -1,5 +1,7 @@ import datetime import enum +import threading +import time from concurrent.futures import ThreadPoolExecutor from queue import Empty, Queue from threading import Barrier @@ -9,6 +11,7 @@ from fixtures.common_types import Lsn, TimelineId from fixtures.log_helper import log from fixtures.neon_fixtures import ( + LogCursor, NeonEnvBuilder, PgBin, flush_ep_to_pageserver, @@ -17,7 +20,8 @@ from fixtures.pageserver.http import HistoricLayerInfo, PageserverApiException from fixtures.pageserver.utils import wait_for_last_record_lsn, wait_timeline_detail_404 from fixtures.remote_storage import LocalFsStorage, RemoteStorageKind -from fixtures.utils import assert_pageserver_backups_equal +from fixtures.utils import assert_pageserver_backups_equal, wait_until +from requests import ReadTimeout def by_end_lsn(info: HistoricLayerInfo) -> Lsn: @@ -161,7 +165,7 @@ def test_ancestor_detach_branched_from( ) all_reparented = client.detach_ancestor(env.initial_tenant, timeline_id) - assert all_reparented == set() + assert all_reparented == [] if restart_after: env.pageserver.stop() @@ -270,7 +274,7 @@ def test_ancestor_detach_reparents_earlier(neon_env_builder: NeonEnvBuilder): after = env.neon_cli.create_branch("after", "main", env.initial_tenant, ancestor_start_lsn=None) all_reparented = client.detach_ancestor(env.initial_tenant, timeline_id) - assert all_reparented == {reparented, same_branchpoint} + assert set(all_reparented) == {reparented, same_branchpoint} env.pageserver.quiesce_tenants() @@ -530,7 +534,7 @@ def delta_layers(timeline_id: TimelineId): for _, timeline_id in skip_main: reparented = client.detach_ancestor(env.initial_tenant, timeline_id) - assert reparented == set(), "we have no earlier branches at any level" + assert reparented == [], "we have no earlier branches at any level" post_detach_l0s = list(filter(lambda x: x.l0, delta_layers(branch_timeline_id))) assert len(post_detach_l0s) == 5, "should had inherited 4 L0s, have 5 in total" @@ -561,7 +565,9 @@ def delta_layers(timeline_id: TimelineId): @pytest.mark.parametrize("sharded", [True, False]) -def test_timeline_ancestor_detach_errors(neon_env_builder: NeonEnvBuilder, sharded: bool): +def test_timeline_ancestor_detach_idempotent_success( + neon_env_builder: NeonEnvBuilder, sharded: bool +): shards = 2 if sharded else 1 neon_env_builder.num_pageservers = shards @@ -579,28 +585,28 @@ def test_timeline_ancestor_detach_errors(neon_env_builder: NeonEnvBuilder, shard else: client = env.pageserver.http_client() - with pytest.raises(PageserverApiException, match=".* no ancestors") as info: - client.detach_ancestor(env.initial_tenant, env.initial_timeline) - assert info.value.status_code == 409 - first_branch = env.neon_cli.create_branch("first_branch") - second_branch = env.neon_cli.create_branch("second_branch", ancestor_branch_name="first_branch") - # funnily enough this does not have a prefix - with pytest.raises(PageserverApiException, match="too many ancestors") as info: - client.detach_ancestor(env.initial_tenant, second_branch) - assert info.value.status_code == 400 + _ = env.neon_cli.create_branch("second_branch", ancestor_branch_name="first_branch") + + # these two will be reparented, and they should be returned in stable order + # from pageservers OR otherwise there will be an `error!` logging from + # storage controller + reparented1 = env.neon_cli.create_branch("first_reparented", ancestor_branch_name="main") + reparented2 = env.neon_cli.create_branch("second_reparented", ancestor_branch_name="main") - client.detach_ancestor(env.initial_tenant, first_branch) + first_reparenting_response = client.detach_ancestor(env.initial_tenant, first_branch) + assert set(first_reparenting_response) == {reparented1, reparented2} # FIXME: this should be done by the http req handler for ps in pageservers.values(): ps.quiesce_tenants() - with pytest.raises(PageserverApiException, match=".* no ancestors") as info: - client.detach_ancestor(env.initial_tenant, first_branch) - # FIXME: this should be 200 OK because we've already completed it - assert info.value.status_code == 409 + for _ in range(5): + # once completed, we can retry this how many times + assert ( + client.detach_ancestor(env.initial_tenant, first_branch) == first_reparenting_response + ) client.tenant_delete(env.initial_tenant) @@ -609,7 +615,50 @@ def test_timeline_ancestor_detach_errors(neon_env_builder: NeonEnvBuilder, shard assert e.value.status_code == 404 +@pytest.mark.parametrize("sharded", [True, False]) +def test_timeline_ancestor_detach_errors(neon_env_builder: NeonEnvBuilder, sharded: bool): + # the test is split from test_timeline_ancestor_detach_idempotent_success as only these error cases should create "request was dropped before completing", + # given the current first error handling + shards = 2 if sharded else 1 + + neon_env_builder.num_pageservers = shards + env = neon_env_builder.init_start(initial_tenant_shard_count=shards if sharded else None) + + pageservers = dict((int(p.id), p) for p in env.pageservers) + + for ps in pageservers.values(): + ps.allowed_errors.extend(SHUTDOWN_ALLOWED_ERRORS) + ps.allowed_errors.append( + ".* WARN .* path=/v1/tenant/.*/timeline/.*/detach_ancestor request_id=.*: request was dropped before completing" + ) + + client = ( + env.pageserver.http_client() if not sharded else env.storage_controller.pageserver_api() + ) + + with pytest.raises(PageserverApiException, match=".* no ancestors") as info: + client.detach_ancestor(env.initial_tenant, env.initial_timeline) + assert info.value.status_code == 409 + + _ = env.neon_cli.create_branch("first_branch") + + second_branch = env.neon_cli.create_branch("second_branch", ancestor_branch_name="first_branch") + + # funnily enough this does not have a prefix + with pytest.raises(PageserverApiException, match="too many ancestors") as info: + client.detach_ancestor(env.initial_tenant, second_branch) + assert info.value.status_code == 400 + + def test_sharded_timeline_detach_ancestor(neon_env_builder: NeonEnvBuilder): + """ + Sharded timeline detach ancestor; 4 nodes: 1 stuck, 1 restarted, 2 normal. + + Stuck node gets stuck on a pause failpoint for first storage controller request. + Restarted node remains stuck until explicit restart from test code. + + We retry the request until storage controller gets 200 OK from all nodes. + """ branch_name = "soon_detached" shard_count = 4 neon_env_builder.num_pageservers = shard_count @@ -621,8 +670,15 @@ def test_sharded_timeline_detach_ancestor(neon_env_builder: NeonEnvBuilder): # FIXME: should this be in the neon_env_builder.init_start? env.storage_controller.reconcile_until_idle() + # as we will stop a node, make sure there is no clever rebalancing + env.storage_controller.tenant_policy_update(env.initial_tenant, body={"scheduling": "Stop"}) + env.storage_controller.allowed_errors.append(".*: Scheduling is disabled by policy Stop .*") + shards = env.storage_controller.locate(env.initial_tenant) + utilized_pageservers = {x["node_id"] for x in shards} + assert len(utilized_pageservers) > 1, "all shards got placed on single pageserver?" + branch_timeline_id = env.neon_cli.create_branch(branch_name, tenant_id=env.initial_tenant) with env.endpoints.create_start(branch_name, tenant_id=env.initial_tenant) as ep: @@ -642,7 +698,79 @@ def test_sharded_timeline_detach_ancestor(neon_env_builder: NeonEnvBuilder): assert Lsn(detail["initdb_lsn"]) < lsn assert TimelineId(detail["ancestor_timeline_id"]) == env.initial_timeline - env.storage_controller.pageserver_api().detach_ancestor(env.initial_tenant, branch_timeline_id) + # make one of the nodes get stuck, but continue the initial operation + # make another of the nodes get stuck, then restart + + stuck = pageservers[int(shards[0]["node_id"])] + stuck.allowed_errors.append(".*: request was dropped before completing") + env.storage_controller.allowed_errors.append(".*: request was dropped before completing") + stuck_http = stuck.http_client() + stuck_http.configure_failpoints( + ("timeline-detach-ancestor::before_starting_after_locking_pausable", "pause") + ) + + restarted = pageservers[int(shards[1]["node_id"])] + restarted.allowed_errors.extend( + [ + ".*: request was dropped before completing", + ".*: Cancelled request finished with an error: ShuttingDown", + ] + ) + assert restarted.id != stuck.id + restarted_http = restarted.http_client() + restarted_http.configure_failpoints( + [ + ("timeline-detach-ancestor::before_starting_after_locking_pausable", "pause"), + ] + ) + + target = env.storage_controller.pageserver_api() + + with pytest.raises(ReadTimeout): + target.detach_ancestor(env.initial_tenant, branch_timeline_id, timeout=1) + + stuck_http.configure_failpoints( + ("timeline-detach-ancestor::before_starting_after_locking_pausable", "off") + ) + + barrier = threading.Barrier(2) + + def restart_restarted(): + barrier.wait() + # graceful shutdown should just work, because simultaneously unpaused + restarted.stop() + # this does not happen always, depends how fast we exit after unpausing + # restarted.assert_log_contains("Cancelled request finished with an error: ShuttingDown") + restarted.start() + + with ThreadPoolExecutor(max_workers=1) as pool: + fut = pool.submit(restart_restarted) + barrier.wait() + # we have 10s, lets use 1/2 of that to help the shutdown start + time.sleep(5) + restarted_http.configure_failpoints( + ("timeline-detach-ancestor::before_starting_after_locking_pausable", "off") + ) + fut.result() + + # detach ancestor request handling is not sensitive to http cancellation. + # this means that the "stuck" is on its way to complete the detach, but the restarted is off + # now it can either be complete on all nodes, or still in progress with + # one. + without_retrying = target.without_status_retrying() + + # this retry loop will be long enough that the tenant can always activate + reparented = None + for _ in range(10): + try: + reparented = without_retrying.detach_ancestor(env.initial_tenant, branch_timeline_id) + except PageserverApiException as info: + assert info.status_code == 503 + time.sleep(2) + else: + break + + assert reparented == [], "too many retries (None) or unexpected reparentings" for shard_info in shards: node_id = int(shard_info["node_id"]) @@ -661,8 +789,262 @@ def test_sharded_timeline_detach_ancestor(neon_env_builder: NeonEnvBuilder): assert count == 10000 +@pytest.mark.parametrize("mode", ["delete_timeline", "delete_tenant"]) +@pytest.mark.parametrize("sharded", [False, True]) +def test_timeline_detach_ancestor_interrupted_by_deletion( + neon_env_builder: NeonEnvBuilder, mode: str, sharded: bool +): + """ + Timeline ancestor detach interrupted by deleting either: + - the detached timeline + - the whole tenant + + after starting the detach. + + What remains not tested by this: + - shutdown winning over complete + + Shutdown winning over complete needs gc blocking and reparenting any left-overs on retry. + """ + + if sharded and mode == "delete_tenant": + # the shared/exclusive lock for tenant is blocking this: + # timeline detach ancestor takes shared, delete tenant takes exclusive + pytest.skip( + "tenant deletion while timeline ancestor detach is underway is not supported yet" + ) + + shard_count = 2 if sharded else 1 + + neon_env_builder.num_pageservers = shard_count + + env = neon_env_builder.init_start(initial_tenant_shard_count=shard_count if sharded else None) + + for ps in env.pageservers: + ps.allowed_errors.extend(SHUTDOWN_ALLOWED_ERRORS) + + pageservers = dict((int(p.id), p) for p in env.pageservers) + + detached_timeline = env.neon_cli.create_branch("detached soon", "main") + + failpoint = "timeline-detach-ancestor::before_starting_after_locking_pausable" + + env.storage_controller.reconcile_until_idle() + shards = env.storage_controller.locate(env.initial_tenant) + + assert len(set(info["node_id"] for info in shards)) == shard_count + + target = env.storage_controller.pageserver_api() if sharded else env.pageserver.http_client() + target = target.without_status_retrying() + + victim = pageservers[int(shards[-1]["node_id"])] + victim_http = victim.http_client() + victim_http.configure_failpoints((failpoint, "pause")) + + def detach_ancestor(): + target.detach_ancestor(env.initial_tenant, detached_timeline) + + def at_failpoint() -> Tuple[str, LogCursor]: + return victim.assert_log_contains(f"at failpoint {failpoint}") + + def start_delete(): + if mode == "delete_timeline": + target.timeline_delete(env.initial_tenant, detached_timeline) + elif mode == "delete_tenant": + target.tenant_delete(env.initial_tenant) + else: + raise RuntimeError(f"unimplemented mode {mode}") + + def at_waiting_on_gate_close(start_offset: LogCursor) -> LogCursor: + _, offset = victim.assert_log_contains( + "closing is taking longer than expected", offset=start_offset + ) + return offset + + def is_deleted(): + try: + if mode == "delete_timeline": + target.timeline_detail(env.initial_tenant, detached_timeline) + elif mode == "delete_tenant": + target.tenant_status(env.initial_tenant) + else: + return False + except PageserverApiException as e: + assert e.status_code == 404 + return True + else: + raise RuntimeError("waiting for 404") + + with ThreadPoolExecutor(max_workers=2) as pool: + try: + fut = pool.submit(detach_ancestor) + _, offset = wait_until(10, 1.0, at_failpoint) + + delete = pool.submit(start_delete) + + wait_until(10, 1.0, lambda: at_waiting_on_gate_close(offset)) + + victim_http.configure_failpoints((failpoint, "off")) + + delete.result() + + assert wait_until(10, 1.0, is_deleted), f"unimplemented mode {mode}" + + with pytest.raises(PageserverApiException) as exc: + fut.result() + assert exc.value.status_code == 503 + finally: + victim_http.configure_failpoints((failpoint, "off")) + + +@pytest.mark.parametrize("mode", ["delete_reparentable_timeline"]) +def test_sharded_tad_interleaved_after_partial_success(neon_env_builder: NeonEnvBuilder, mode: str): + """ + Technically possible storage controller concurrent interleaving timeline + deletion with timeline detach. + + Deletion is fine, as any sharded pageservers reach the same end state, but + creating reparentable timeline would create an issue as the two nodes would + never agree. There is a solution though: the created reparentable timeline + must be detached. + """ + + assert ( + mode == "delete_reparentable_timeline" + ), "only one now, but we could have the create just as well, need gc blocking" + + shard_count = 2 + neon_env_builder.num_pageservers = shard_count + env = neon_env_builder.init_start(initial_tenant_shard_count=shard_count) + + for ps in env.pageservers: + ps.allowed_errors.extend(SHUTDOWN_ALLOWED_ERRORS) + + pageservers = dict((int(p.id), p) for p in env.pageservers) + + env.storage_controller.reconcile_until_idle() + shards = env.storage_controller.locate(env.initial_tenant) + assert len(set(x["node_id"] for x in shards)) == shard_count + + with env.endpoints.create_start("main") as ep: + ep.safe_psql("create table foo as select i::bigint from generate_series(1, 1000) t(i)") + + # as the interleaved operation, we will delete this timeline, which was reparenting candidate + first_branch_lsn = wait_for_last_flush_lsn( + env, ep, env.initial_tenant, env.initial_timeline + ) + for ps, shard_id in [(pageservers[int(x["node_id"])], x["shard_id"]) for x in shards]: + ps.http_client().timeline_checkpoint(shard_id, env.initial_timeline) + + ep.safe_psql("create table bar as select i::bigint from generate_series(1, 2000) t(i)") + detached_branch_lsn = flush_ep_to_pageserver( + env, ep, env.initial_tenant, env.initial_timeline + ) + + for ps, shard_id in [(pageservers[int(x["node_id"])], x["shard_id"]) for x in shards]: + ps.http_client().timeline_checkpoint(shard_id, env.initial_timeline) + + first_branch = env.neon_cli.create_branch( + "first_branch", ancestor_branch_name="main", ancestor_start_lsn=first_branch_lsn + ) + detached_branch = env.neon_cli.create_branch( + "detached_branch", ancestor_branch_name="main", ancestor_start_lsn=detached_branch_lsn + ) + + pausepoint = "timeline-detach-ancestor::before_starting_after_locking_pausable" + + stuck = pageservers[int(shards[0]["node_id"])] + stuck_http = stuck.http_client().without_status_retrying() + stuck_http.configure_failpoints((pausepoint, "pause")) + + victim = pageservers[int(shards[-1]["node_id"])] + victim_http = victim.http_client().without_status_retrying() + victim_http.configure_failpoints( + (pausepoint, "pause"), + ) + + # noticed a surprising 409 if the other one would fail instead + # victim_http.configure_failpoints([ + # (pausepoint, "pause"), + # ("timeline-detach-ancestor::before_starting_after_locking", "return"), + # ]) + + # interleaving a create_timeline which could be reparented will produce two + # permanently different reparentings: one node has reparented, other has + # not + # + # with deletion there is no such problem + def detach_timeline(): + env.storage_controller.pageserver_api().detach_ancestor(env.initial_tenant, detached_branch) + + def paused_at_failpoint(): + stuck.assert_log_contains(f"at failpoint {pausepoint}") + victim.assert_log_contains(f"at failpoint {pausepoint}") + + def first_completed(): + detail = stuck_http.timeline_detail(shards[0]["shard_id"], detached_branch) + log.info(detail) + assert detail.get("ancestor_lsn") is None + + def first_branch_gone(): + try: + env.storage_controller.pageserver_api().timeline_detail( + env.initial_tenant, first_branch + ) + except PageserverApiException as e: + log.info(f"error {e}") + assert e.status_code == 404 + else: + log.info("still ok") + raise RuntimeError("not done yet") + + with ThreadPoolExecutor(max_workers=1) as pool: + try: + fut = pool.submit(detach_timeline) + wait_until(10, 1.0, paused_at_failpoint) + + # let stuck complete + stuck_http.configure_failpoints((pausepoint, "off")) + wait_until(10, 1.0, first_completed) + + # if we would let victim fail, for some reason there'd be a 409 response instead of 500 + # victim_http.configure_failpoints((pausepoint, "off")) + # with pytest.raises(PageserverApiException, match=".* 500 Internal Server Error failpoint: timeline-detach-ancestor::before_starting_after_locking") as exc: + # fut.result() + # assert exc.value.status_code == 409 + + env.storage_controller.pageserver_api().timeline_delete( + env.initial_tenant, first_branch + ) + victim_http.configure_failpoints((pausepoint, "off")) + wait_until(10, 1.0, first_branch_gone) + + # it now passes, and we should get an error messages about mixed reparenting as the stuck still had something to reparent + fut.result() + + msg, offset = env.storage_controller.assert_log_contains( + ".*/timeline/\\S+/detach_ancestor.*: shards returned different results matching=0 .*" + ) + log.info(f"expected error message: {msg}") + env.storage_controller.allowed_errors.append( + ".*: shards returned different results matching=0 .*" + ) + + detach_timeline() + + # FIXME: perhaps the above should be automatically retried, if we get mixed results? + not_found = env.storage_controller.log_contains( + ".*/timeline/\\S+/detach_ancestor.*: shards returned different results matching=0 .*", + offset=offset, + ) + + assert not_found is None + finally: + stuck_http.configure_failpoints((pausepoint, "off")) + victim_http.configure_failpoints((pausepoint, "off")) + + # TODO: -# - after starting the operation, tenant is deleted # - after starting the operation, pageserver is shutdown, restarted # - after starting the operation, bottom-most timeline is deleted, pageserver is restarted, gc is inhibited # - deletion of reparented while reparenting should fail once, then succeed (?) @@ -670,9 +1052,5 @@ def test_sharded_timeline_detach_ancestor(neon_env_builder: NeonEnvBuilder): # - investigate: why are layers started at uneven lsn? not just after branching, but in general. # # TEST: 1. tad which partially succeeds, one returns 500 -# 2. create branch below timeline? or delete timeline below +# 2. create branch below timeline? ~or delete reparented timeline~ (done) # 3. on retry all should report the same reparented timelines -# -# TEST: 1. tad is started, one node stalls, other restarts -# 2. client timeout before stall over -# 3. on retry with stalled and other being able to proceed From 349373cb11d5f40f69bd9c17f3fdeccadc321141 Mon Sep 17 00:00:00 2001 From: Sasha Krassovsky Date: Mon, 15 Jul 2024 14:55:57 -0700 Subject: [PATCH 156/194] Allow reusing projects between runs of logical replication benchmarks (#8393) --- test_runner/fixtures/neon_api.py | 44 +++ test_runner/fixtures/neon_fixtures.py | 14 +- .../performance/test_logical_replication.py | 341 +++++++----------- 3 files changed, 182 insertions(+), 217 deletions(-) diff --git a/test_runner/fixtures/neon_api.py b/test_runner/fixtures/neon_api.py index 39baf5fab69f..658ed119a175 100644 --- a/test_runner/fixtures/neon_api.py +++ b/test_runner/fixtures/neon_api.py @@ -261,3 +261,47 @@ def wait_for_operation_to_finish(self, project_id: str): if op["status"] in {"scheduling", "running", "cancelling"}: has_running = True time.sleep(0.5) + + +class NeonApiEndpoint: + def __init__(self, neon_api: NeonAPI, pg_version: PgVersion, project_id: Optional[str]): + self.neon_api = neon_api + if project_id is None: + project = neon_api.create_project(pg_version) + neon_api.wait_for_operation_to_finish(project["project"]["id"]) + self.project_id = project["project"]["id"] + self.endpoint_id = project["endpoints"][0]["id"] + self.connstr = project["connection_uris"][0]["connection_uri"] + self.pgbench_env = connection_parameters_to_env( + project["connection_uris"][0]["connection_parameters"] + ) + self.is_new = True + else: + project = neon_api.get_project_details(project_id) + if int(project["project"]["pg_version"]) != int(pg_version): + raise Exception( + f"A project with the provided ID exists, but it's not of the specified version (expected {pg_version}, got {project['project']['pg_version']})" + ) + self.project_id = project_id + eps = neon_api.get_endpoints(project_id)["endpoints"] + self.endpoint_id = eps[0]["id"] + self.connstr = neon_api.get_connection_uri(project_id, endpoint_id=self.endpoint_id)[ + "uri" + ] + pw = self.connstr.split("@")[0].split(":")[-1] + self.pgbench_env = { + "PGHOST": eps[0]["host"], + "PGDATABASE": "neondb", + "PGUSER": "neondb_owner", + "PGPASSWORD": pw, + } + self.is_new = False + + def restart(self): + self.neon_api.restart_endpoint(self.project_id, self.endpoint_id) + self.neon_api.wait_for_operation_to_finish(self.project_id) + + def get_synthetic_storage_size(self) -> int: + return int( + self.neon_api.get_project_details(self.project_id)["project"]["synthetic_storage_size"] + ) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 90ed838e1db3..fe4a33445834 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -87,7 +87,7 @@ ) from fixtures.utils import AuxFileStore as AuxFileStore # reexport -from .neon_api import NeonAPI +from .neon_api import NeonAPI, NeonApiEndpoint """ This file contains pytest fixtures. A fixture is a test resource that can be @@ -3158,6 +3158,18 @@ def __exit__( pass +@pytest.fixture(scope="function") +def benchmark_project_pub(neon_api: NeonAPI, pg_version: PgVersion) -> NeonApiEndpoint: + project_id = os.getenv("BENCHMARK_PROJECT_ID_PUB") + return NeonApiEndpoint(neon_api, pg_version, project_id) + + +@pytest.fixture(scope="function") +def benchmark_project_sub(neon_api: NeonAPI, pg_version: PgVersion) -> NeonApiEndpoint: + project_id = os.getenv("BENCHMARK_PROJECT_ID_SUB") + return NeonApiEndpoint(neon_api, pg_version, project_id) + + @pytest.fixture(scope="function") def remote_pg( test_output_dir: Path, pg_distrib_dir: Path, pg_version: PgVersion diff --git a/test_runner/performance/test_logical_replication.py b/test_runner/performance/test_logical_replication.py index 5ab83dd31d0b..53bb29a65908 100644 --- a/test_runner/performance/test_logical_replication.py +++ b/test_runner/performance/test_logical_replication.py @@ -1,7 +1,6 @@ from __future__ import annotations import time -import traceback from typing import TYPE_CHECKING import psycopg2 @@ -10,15 +9,12 @@ from fixtures.benchmark_fixture import MetricReport from fixtures.common_types import Lsn from fixtures.log_helper import log -from fixtures.neon_api import connection_parameters_to_env from fixtures.neon_fixtures import AuxFileStore, logical_replication_sync -from fixtures.pg_version import PgVersion if TYPE_CHECKING: from fixtures.benchmark_fixture import NeonBenchmarker - from fixtures.neon_api import NeonAPI + from fixtures.neon_api import NeonApiEndpoint from fixtures.neon_fixtures import NeonEnv, PgBin - from fixtures.pg_version import PgVersion @pytest.mark.parametrize("pageserver_aux_file_policy", [AuxFileStore.V2]) @@ -86,8 +82,8 @@ def measure_logical_replication_lag(sub_cur, pub_cur, timeout_sec=600): @pytest.mark.timeout(2 * 60 * 60) def test_subscriber_lag( pg_bin: PgBin, - neon_api: NeonAPI, - pg_version: PgVersion, + benchmark_project_pub: NeonApiEndpoint, + benchmark_project_sub: NeonApiEndpoint, zenbenchmark: NeonBenchmarker, ): """ @@ -99,125 +95,82 @@ def test_subscriber_lag( sync_interval_min = 5 pgbench_duration = f"-T{test_duration_min * 60 * 2}" - pub_project = neon_api.create_project(pg_version) - pub_project_id = pub_project["project"]["id"] - neon_api.wait_for_operation_to_finish(pub_project_id) - error_occurred = False - try: - sub_project = neon_api.create_project(pg_version) - sub_project_id = sub_project["project"]["id"] - sub_endpoint_id = sub_project["endpoints"][0]["id"] - neon_api.wait_for_operation_to_finish(sub_project_id) - try: - pub_env = connection_parameters_to_env( - pub_project["connection_uris"][0]["connection_parameters"] - ) - sub_env = connection_parameters_to_env( - sub_project["connection_uris"][0]["connection_parameters"] - ) - pub_connstr = pub_project["connection_uris"][0]["connection_uri"] - sub_connstr = sub_project["connection_uris"][0]["connection_uri"] - - pg_bin.run_capture(["pgbench", "-i", "-s100"], env=pub_env) - pg_bin.run_capture(["pgbench", "-i", "-s100"], env=sub_env) - - pub_conn = psycopg2.connect(pub_connstr) - sub_conn = psycopg2.connect(sub_connstr) - pub_conn.autocommit = True - sub_conn.autocommit = True - with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: - sub_cur.execute("truncate table pgbench_accounts") - sub_cur.execute("truncate table pgbench_history") - - pub_cur.execute( - "create publication pub1 for table pgbench_accounts, pgbench_history" - ) - sub_cur.execute( - f"create subscription sub1 connection '{pub_connstr}' publication pub1" - ) + pub_env = benchmark_project_pub.pgbench_env + sub_env = benchmark_project_sub.pgbench_env + pub_connstr = benchmark_project_pub.connstr + sub_connstr = benchmark_project_sub.connstr + + pg_bin.run_capture(["pgbench", "-i", "-s100"], env=pub_env) + pg_bin.run_capture(["pgbench", "-i", "-s100"], env=sub_env) + + pub_conn = psycopg2.connect(pub_connstr) + sub_conn = psycopg2.connect(sub_connstr) + pub_conn.autocommit = True + sub_conn.autocommit = True + with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: + if benchmark_project_pub.is_new: + pub_cur.execute("create publication pub1 for table pgbench_accounts, pgbench_history") + + if benchmark_project_sub.is_new: + sub_cur.execute("truncate table pgbench_accounts") + sub_cur.execute("truncate table pgbench_history") + + sub_cur.execute(f"create subscription sub1 connection '{pub_connstr}' publication pub1") - initial_sync_lag = measure_logical_replication_lag(sub_cur, pub_cur) - pub_conn.close() - sub_conn.close() + initial_sync_lag = measure_logical_replication_lag(sub_cur, pub_cur) + pub_conn.close() + sub_conn.close() - zenbenchmark.record( - "initial_sync_lag", initial_sync_lag, "s", MetricReport.LOWER_IS_BETTER - ) + zenbenchmark.record("initial_sync_lag", initial_sync_lag, "s", MetricReport.LOWER_IS_BETTER) + + pub_workload = pg_bin.run_nonblocking( + ["pgbench", "-c10", pgbench_duration, "-Mprepared"], env=pub_env + ) + try: + sub_workload = pg_bin.run_nonblocking( + ["pgbench", "-c10", pgbench_duration, "-S"], + env=sub_env, + ) + try: + start = time.time() + while time.time() - start < test_duration_min * 60: + time.sleep(sync_interval_min * 60) + check_pgbench_still_running(pub_workload, "pub") + check_pgbench_still_running(sub_workload, "sub") + + with psycopg2.connect(pub_connstr) as pub_conn, psycopg2.connect( + sub_connstr + ) as sub_conn: + with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: + lag = measure_logical_replication_lag(sub_cur, pub_cur) + + log.info(f"Replica lagged behind master by {lag} seconds") + zenbenchmark.record("replica_lag", lag, "s", MetricReport.LOWER_IS_BETTER) + sub_workload.terminate() + benchmark_project_sub.restart() - pub_workload = pg_bin.run_nonblocking( - ["pgbench", "-c10", pgbench_duration, "-Mprepared"], env=pub_env - ) - try: sub_workload = pg_bin.run_nonblocking( ["pgbench", "-c10", pgbench_duration, "-S"], env=sub_env, ) - try: - start = time.time() - while time.time() - start < test_duration_min * 60: - time.sleep(sync_interval_min * 60) - check_pgbench_still_running(pub_workload, "pub") - check_pgbench_still_running(sub_workload, "sub") - - with psycopg2.connect(pub_connstr) as pub_conn, psycopg2.connect( - sub_connstr - ) as sub_conn: - with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: - lag = measure_logical_replication_lag(sub_cur, pub_cur) - - log.info(f"Replica lagged behind master by {lag} seconds") - zenbenchmark.record("replica_lag", lag, "s", MetricReport.LOWER_IS_BETTER) - sub_workload.terminate() - neon_api.restart_endpoint( - sub_project_id, - sub_endpoint_id, - ) - neon_api.wait_for_operation_to_finish(sub_project_id) - sub_workload = pg_bin.run_nonblocking( - ["pgbench", "-c10", pgbench_duration, "-S"], - env=sub_env, - ) - - # Measure storage to make sure replication information isn't bloating storage - sub_storage = neon_api.get_project_details(sub_project_id)["project"][ - "synthetic_storage_size" - ] - pub_storage = neon_api.get_project_details(pub_project_id)["project"][ - "synthetic_storage_size" - ] - zenbenchmark.record( - "sub_storage", sub_storage, "B", MetricReport.LOWER_IS_BETTER - ) - zenbenchmark.record( - "pub_storage", pub_storage, "B", MetricReport.LOWER_IS_BETTER - ) - - finally: - sub_workload.terminate() - finally: - pub_workload.terminate() - except Exception as e: - error_occurred = True - log.error(f"Caught exception {e}") - log.error(traceback.format_exc()) + + # Measure storage to make sure replication information isn't bloating storage + sub_storage = benchmark_project_sub.get_synthetic_storage_size() + pub_storage = benchmark_project_pub.get_synthetic_storage_size() + zenbenchmark.record("sub_storage", sub_storage, "B", MetricReport.LOWER_IS_BETTER) + zenbenchmark.record("pub_storage", pub_storage, "B", MetricReport.LOWER_IS_BETTER) finally: - if not error_occurred: - neon_api.delete_project(sub_project_id) - except Exception as e: - error_occurred = True - log.error(f"Caught exception {e}") - log.error(traceback.format_exc()) + sub_workload.terminate() finally: - assert not error_occurred - neon_api.delete_project(pub_project_id) + pub_workload.terminate() @pytest.mark.remote_cluster @pytest.mark.timeout(2 * 60 * 60) def test_publisher_restart( pg_bin: PgBin, - neon_api: NeonAPI, - pg_version: PgVersion, + benchmark_project_pub: NeonApiEndpoint, + benchmark_project_sub: NeonApiEndpoint, zenbenchmark: NeonBenchmarker, ): """ @@ -229,114 +182,70 @@ def test_publisher_restart( sync_interval_min = 5 pgbench_duration = f"-T{test_duration_min * 60 * 2}" - pub_project = neon_api.create_project(pg_version) - pub_project_id = pub_project["project"]["id"] - pub_endpoint_id = pub_project["endpoints"][0]["id"] - neon_api.wait_for_operation_to_finish(pub_project_id) - error_occurred = False - try: - sub_project = neon_api.create_project(pg_version) - sub_project_id = sub_project["project"]["id"] - neon_api.wait_for_operation_to_finish(sub_project_id) - try: - pub_env = connection_parameters_to_env( - pub_project["connection_uris"][0]["connection_parameters"] - ) - sub_env = connection_parameters_to_env( - sub_project["connection_uris"][0]["connection_parameters"] - ) - pub_connstr = pub_project["connection_uris"][0]["connection_uri"] - sub_connstr = sub_project["connection_uris"][0]["connection_uri"] - - pg_bin.run_capture(["pgbench", "-i", "-s100"], env=pub_env) - pg_bin.run_capture(["pgbench", "-i", "-s100"], env=sub_env) - - pub_conn = psycopg2.connect(pub_connstr) - sub_conn = psycopg2.connect(sub_connstr) - pub_conn.autocommit = True - sub_conn.autocommit = True - with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: - sub_cur.execute("truncate table pgbench_accounts") - sub_cur.execute("truncate table pgbench_history") - - pub_cur.execute( - "create publication pub1 for table pgbench_accounts, pgbench_history" - ) - sub_cur.execute( - f"create subscription sub1 connection '{pub_connstr}' publication pub1" - ) + pub_env = benchmark_project_pub.pgbench_env + sub_env = benchmark_project_sub.pgbench_env + pub_connstr = benchmark_project_pub.connstr + sub_connstr = benchmark_project_sub.connstr - initial_sync_lag = measure_logical_replication_lag(sub_cur, pub_cur) - pub_conn.close() - sub_conn.close() + pg_bin.run_capture(["pgbench", "-i", "-s100"], env=pub_env) + pg_bin.run_capture(["pgbench", "-i", "-s100"], env=sub_env) - zenbenchmark.record( - "initial_sync_lag", initial_sync_lag, "s", MetricReport.LOWER_IS_BETTER - ) + pub_conn = psycopg2.connect(pub_connstr) + sub_conn = psycopg2.connect(sub_connstr) + pub_conn.autocommit = True + sub_conn.autocommit = True + with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: + if benchmark_project_pub.is_new: + pub_cur.execute("create publication pub1 for table pgbench_accounts, pgbench_history") + + if benchmark_project_sub.is_new: + sub_cur.execute("truncate table pgbench_accounts") + sub_cur.execute("truncate table pgbench_history") + + sub_cur.execute(f"create subscription sub1 connection '{pub_connstr}' publication pub1") + + initial_sync_lag = measure_logical_replication_lag(sub_cur, pub_cur) + pub_conn.close() + sub_conn.close() + + zenbenchmark.record("initial_sync_lag", initial_sync_lag, "s", MetricReport.LOWER_IS_BETTER) + + pub_workload = pg_bin.run_nonblocking( + ["pgbench", "-c10", pgbench_duration, "-Mprepared"], env=pub_env + ) + try: + sub_workload = pg_bin.run_nonblocking( + ["pgbench", "-c10", pgbench_duration, "-S"], + env=sub_env, + ) + try: + start = time.time() + while time.time() - start < test_duration_min * 60: + time.sleep(sync_interval_min * 60) + check_pgbench_still_running(pub_workload, "pub") + check_pgbench_still_running(sub_workload, "sub") - pub_workload = pg_bin.run_nonblocking( - ["pgbench", "-c10", pgbench_duration, "-Mprepared"], env=pub_env - ) - try: - sub_workload = pg_bin.run_nonblocking( - ["pgbench", "-c10", pgbench_duration, "-S"], - env=sub_env, - ) - try: - start = time.time() - while time.time() - start < test_duration_min * 60: - time.sleep(sync_interval_min * 60) - check_pgbench_still_running(pub_workload, "pub") - check_pgbench_still_running(sub_workload, "sub") - - pub_workload.terminate() - neon_api.restart_endpoint( - pub_project_id, - pub_endpoint_id, - ) - neon_api.wait_for_operation_to_finish(pub_project_id) - pub_workload = pg_bin.run_nonblocking( - ["pgbench", "-c10", pgbench_duration, "-Mprepared"], - env=pub_env, - ) - with psycopg2.connect(pub_connstr) as pub_conn, psycopg2.connect( - sub_connstr - ) as sub_conn: - with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: - lag = measure_logical_replication_lag(sub_cur, pub_cur) - - log.info(f"Replica lagged behind master by {lag} seconds") - zenbenchmark.record("replica_lag", lag, "s", MetricReport.LOWER_IS_BETTER) - - # Measure storage to make sure replication information isn't bloating storage - sub_storage = neon_api.get_project_details(sub_project_id)["project"][ - "synthetic_storage_size" - ] - pub_storage = neon_api.get_project_details(pub_project_id)["project"][ - "synthetic_storage_size" - ] - zenbenchmark.record( - "sub_storage", sub_storage, "B", MetricReport.LOWER_IS_BETTER - ) - zenbenchmark.record( - "pub_storage", pub_storage, "B", MetricReport.LOWER_IS_BETTER - ) - - finally: - sub_workload.terminate() - finally: pub_workload.terminate() - except Exception as e: - error_occurred = True - log.error(f"Caught exception {e}") - log.error(traceback.format_exc()) + benchmark_project_pub.restart() + pub_workload = pg_bin.run_nonblocking( + ["pgbench", "-c10", pgbench_duration, "-Mprepared"], + env=pub_env, + ) + with psycopg2.connect(pub_connstr) as pub_conn, psycopg2.connect( + sub_connstr + ) as sub_conn: + with pub_conn.cursor() as pub_cur, sub_conn.cursor() as sub_cur: + lag = measure_logical_replication_lag(sub_cur, pub_cur) + + log.info(f"Replica lagged behind master by {lag} seconds") + zenbenchmark.record("replica_lag", lag, "s", MetricReport.LOWER_IS_BETTER) + + # Measure storage to make sure replication information isn't bloating storage + sub_storage = benchmark_project_sub.get_synthetic_storage_size() + pub_storage = benchmark_project_pub.get_synthetic_storage_size() + zenbenchmark.record("sub_storage", sub_storage, "B", MetricReport.LOWER_IS_BETTER) + zenbenchmark.record("pub_storage", pub_storage, "B", MetricReport.LOWER_IS_BETTER) finally: - if not error_occurred: - neon_api.delete_project(sub_project_id) - except Exception as e: - error_occurred = True - log.error(f"Caught exception {e}") - log.error(traceback.format_exc()) + sub_workload.terminate() finally: - assert not error_occurred - neon_api.delete_project(pub_project_id) + pub_workload.terminate() From 5b16624bcc2104135485bcd9c2b7a57f3544e6da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Tue, 16 Jul 2024 02:16:18 +0200 Subject: [PATCH 157/194] Allow the new clippy::doc_lazy_continuation lint (#8388) The `doc_lazy_continuation` lint of clippy is still unknown on latest rust stable. Fixes fall-out from #8151. --- pageserver/src/tenant/timeline.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 239dce878640..58c6257c658d 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -3409,6 +3409,7 @@ impl Timeline { } } + #[allow(unknown_lints)] // doc_lazy_continuation is still a new lint #[allow(clippy::doc_lazy_continuation)] /// Get the data needed to reconstruct all keys in the provided keyspace /// From ea5460843c889bb5c2ebb4585e92ad554fc6380e Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 16 Jul 2024 08:52:49 +0100 Subject: [PATCH 158/194] pageserver: un-Arc Timeline::layers (#8386) ## Problem This structure was in an Arc<> unnecessarily, making it harder to reason about its lifetime (i.e. it was superficially possible for LayerManager to outlive timeline, even though no code used it that way) ## Summary of changes - Remove the Arc<> --- pageserver/src/tenant/timeline.rs | 4 ++-- pageserver/src/tenant/timeline/compaction.rs | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 58c6257c658d..48a5b2d32bf7 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -198,7 +198,7 @@ impl PartialOrd for Hole { /// Temporary function for immutable storage state refactor, ensures we are dropping mutex guard instead of other things. /// Can be removed after all refactors are done. -fn drop_rlock(rlock: tokio::sync::OwnedRwLockReadGuard) { +fn drop_rlock(rlock: tokio::sync::RwLockReadGuard) { drop(rlock) } @@ -271,7 +271,7 @@ pub struct Timeline { /// /// In the future, we'll be able to split up the tuple of LayerMap and `LayerFileManager`, /// so that e.g. on-demand-download/eviction, and layer spreading, can operate just on `LayerFileManager`. - pub(crate) layers: Arc>, + pub(crate) layers: tokio::sync::RwLock, last_freeze_at: AtomicLsn, // Atomic would be more appropriate here. diff --git a/pageserver/src/tenant/timeline/compaction.rs b/pageserver/src/tenant/timeline/compaction.rs index efaa6144af95..eec5e5e53cf7 100644 --- a/pageserver/src/tenant/timeline/compaction.rs +++ b/pageserver/src/tenant/timeline/compaction.rs @@ -27,8 +27,8 @@ use utils::id::TimelineId; use crate::context::{AccessStatsBehavior, RequestContext, RequestContextBuilder}; use crate::page_cache; use crate::tenant::storage_layer::{AsLayerDesc, PersistentLayerDesc}; -use crate::tenant::timeline::{drop_rlock, Hole, ImageLayerCreationOutcome}; -use crate::tenant::timeline::{DeltaLayerWriter, ImageLayerWriter}; +use crate::tenant::timeline::{drop_rlock, DeltaLayerWriter, ImageLayerWriter}; +use crate::tenant::timeline::{Hole, ImageLayerCreationOutcome}; use crate::tenant::timeline::{Layer, ResidentLayer}; use crate::tenant::DeltaLayer; use crate::virtual_file::{MaybeFatalIo, VirtualFile}; @@ -379,7 +379,7 @@ impl Timeline { }; let begin = tokio::time::Instant::now(); - let phase1_layers_locked = Arc::clone(&self.layers).read_owned().await; + let phase1_layers_locked = self.layers.read().await; let now = tokio::time::Instant::now(); stats.read_lock_acquisition_micros = DurationRecorder::Recorded(RecordedDuration(now - begin), now); @@ -399,9 +399,9 @@ impl Timeline { } /// Level0 files first phase of compaction, explained in the [`Self::compact_legacy`] comment. - async fn compact_level0_phase1( - self: &Arc, - guard: tokio::sync::OwnedRwLockReadGuard, + async fn compact_level0_phase1<'a>( + self: &'a Arc, + guard: tokio::sync::RwLockReadGuard<'a, LayerManager>, mut stats: CompactLevel0Phase1StatsBuilder, target_file_size: u64, ctx: &RequestContext, From 2ede9d7a25259a8f226eb44c1f2439485b417e3b Mon Sep 17 00:00:00 2001 From: Anastasia Lubennikova Date: Mon, 15 Jul 2024 12:48:53 +0100 Subject: [PATCH 159/194] Compute: add compatibility patch for rum Fixes #8251 --- Dockerfile.compute-node | 3 +++ patches/rum.patch | 54 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 patches/rum.patch diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node index 7ab685625a8b..48a52bfc6d04 100644 --- a/Dockerfile.compute-node +++ b/Dockerfile.compute-node @@ -311,9 +311,12 @@ RUN wget https://github.com/iCyberon/pg_hashids/archive/refs/tags/v1.2.1.tar.gz FROM build-deps AS rum-pg-build COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ +COPY patches/rum.patch /rum.patch + RUN wget https://github.com/postgrespro/rum/archive/refs/tags/1.3.13.tar.gz -O rum.tar.gz && \ echo "6ab370532c965568df6210bd844ac6ba649f53055e48243525b0b7e5c4d69a7d rum.tar.gz" | sha256sum --check && \ mkdir rum-src && cd rum-src && tar xzf ../rum.tar.gz --strip-components=1 -C . && \ + patch -p1 < /rum.patch && \ make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config USE_PGXS=1 && \ make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config USE_PGXS=1 && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/rum.control diff --git a/patches/rum.patch b/patches/rum.patch new file mode 100644 index 000000000000..3041f8df81d6 --- /dev/null +++ b/patches/rum.patch @@ -0,0 +1,54 @@ +commit 68f3b3b0d594f08aacc4a082ee210749ed5677eb +Author: Anastasia Lubennikova +Date: Mon Jul 15 12:31:56 2024 +0100 + + Neon: fix unlogged index build patch + +diff --git a/src/ruminsert.c b/src/ruminsert.c +index e8b209d..e89bf2a 100644 +--- a/src/ruminsert.c ++++ b/src/ruminsert.c +@@ -628,6 +628,10 @@ rumbuild(Relation heap, Relation index, struct IndexInfo *indexInfo) + elog(ERROR, "index \"%s\" already contains data", + RelationGetRelationName(index)); + ++#ifdef NEON_SMGR ++ smgr_start_unlogged_build(index->rd_smgr); ++#endif ++ + initRumState(&buildstate.rumstate, index); + buildstate.rumstate.isBuild = true; + buildstate.indtuples = 0; +@@ -693,6 +697,10 @@ rumbuild(Relation heap, Relation index, struct IndexInfo *indexInfo) + buildstate.buildStats.nTotalPages = RelationGetNumberOfBlocks(index); + rumUpdateStats(index, &buildstate.buildStats, buildstate.rumstate.isBuild); + ++#ifdef NEON_SMGR ++ smgr_finish_unlogged_build_phase_1(index->rd_smgr); ++#endif ++ + /* + * Write index to xlog + */ +@@ -713,6 +721,21 @@ rumbuild(Relation heap, Relation index, struct IndexInfo *indexInfo) + UnlockReleaseBuffer(buffer); + } + ++#ifdef NEON_SMGR ++ { ++#if PG_VERSION_NUM >= 160000 ++ RelFileLocator rlocator = RelationGetSmgr(index)->smgr_rlocator.locator; ++#else ++ RelFileNode rlocator = RelationGetSmgr(index)->smgr_rnode.node; ++#endif ++ ++ SetLastWrittenLSNForBlockRange(XactLastRecEnd, rlocator, MAIN_FORKNUM, 0, RelationGetNumberOfBlocks(index)); ++ SetLastWrittenLSNForRelation(XactLastRecEnd, rlocator, MAIN_FORKNUM); ++ ++ smgr_end_unlogged_build(index->rd_smgr); ++ } ++#endif ++ + /* + * Return statistics + */ From 9dc71f5a884dd25be14757369072cfb37efd3a9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Tue, 16 Jul 2024 12:19:28 +0200 Subject: [PATCH 160/194] Avoid the storage controller in test_tenant_creation_fails (#8392) As described in #8385, the likely source for flakiness in test_tenant_creation_fails is the following sequence of events: 1. test instructs the storage controller to create the tenant 2. storage controller adds the tenant and persists it to the database. issues a creation request 3. the pageserver restarts with the failpoint disabled 4. storage controller's background reconciliation still wants to create the tenant 5. pageserver gets new request to create the tenant from background reconciliation This commit just avoids the storage controller entirely. It has its own set of issues, as the re-attach request will obviously not include the tenant, but it's still useful to test for non-existence of the tenant. The generation is also not optional any more during tenant attachment. If you omit it, the pageserver yields an error. We change the signature of `tenant_attach` to reflect that. Alternative to #8385 Fixes #8266 --- test_runner/fixtures/neon_fixtures.py | 2 +- test_runner/fixtures/pageserver/http.py | 2 +- test_runner/regress/test_tenants.py | 13 +++---------- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index fe4a33445834..625e9096f58f 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2786,8 +2786,8 @@ def tenant_attach( ) return client.tenant_attach( tenant_id, + generation, config, - generation=generation, ) def tenant_detach(self, tenant_id: TenantId): diff --git a/test_runner/fixtures/pageserver/http.py b/test_runner/fixtures/pageserver/http.py index d66b94948a8e..f1e3d1a30941 100644 --- a/test_runner/fixtures/pageserver/http.py +++ b/test_runner/fixtures/pageserver/http.py @@ -238,8 +238,8 @@ def tenant_list(self) -> List[Dict[Any, Any]]: def tenant_attach( self, tenant_id: Union[TenantId, TenantShardId], + generation: int, config: None | Dict[str, Any] = None, - generation: Optional[int] = None, ): config = config or {} diff --git a/test_runner/regress/test_tenants.py b/test_runner/regress/test_tenants.py index 04b3fdd80fa5..0ebf714de080 100644 --- a/test_runner/regress/test_tenants.py +++ b/test_runner/regress/test_tenants.py @@ -45,17 +45,10 @@ def test_tenant_creation_fails(neon_simple_env: NeonEnv): # Failure to write a config to local disk makes the pageserver assume that local disk is bad and abort the process pageserver_http.configure_failpoints(("tenant-config-before-write", "return")) - # Storage controller will see a torn TCP connection when the crash point is reached, and follow an unclean 500 error path - neon_simple_env.storage_controller.allowed_errors.extend( - [ - ".*Reconcile not done yet while creating tenant.*", - ".*Reconcile error: receive body: error sending request.*", - ".*Error processing HTTP request: InternalServerError.*", - ] - ) + tenant_id = TenantId.generate() - with pytest.raises(Exception, match="error sending request"): - _ = neon_simple_env.neon_cli.create_tenant() + with pytest.raises(requests.exceptions.ConnectionError, match="Connection aborted"): + neon_simple_env.pageserver.http_client().tenant_attach(tenant_id=tenant_id, generation=1) # Any files left behind on disk during failed creation do not prevent # a retry from succeeding. Restart pageserver with no failpoints. From bf7de92dc2ae15263c03d825ce1dfcd78b5825bb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jul 2024 12:20:23 +0200 Subject: [PATCH 161/194] build(deps): bump setuptools from 65.5.1 to 70.0.0 (#8387) Bumps [setuptools](https://github.com/pypa/setuptools) from 65.5.1 to 70.0.0. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: a-masterov <72613290+a-masterov@users.noreply.github.com> --- poetry.lock | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index 809114141188..5192a574ccbd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2641,19 +2641,18 @@ pbr = "*" [[package]] name = "setuptools" -version = "65.5.1" +version = "70.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "setuptools-65.5.1-py3-none-any.whl", hash = "sha256:d0b9a8433464d5800cbe05094acf5c6d52a91bfac9b52bcfc4d41382be5d5d31"}, - {file = "setuptools-65.5.1.tar.gz", hash = "sha256:e197a19aa8ec9722928f2206f8de752def0e4c9fc6953527360d1c36d94ddb2f"}, + {file = "setuptools-70.0.0-py3-none-any.whl", hash = "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4"}, + {file = "setuptools-70.0.0.tar.gz", hash = "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" From bff505426e711138eec3f851bce3293526657942 Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 16 Jul 2024 14:54:54 +0100 Subject: [PATCH 162/194] pageserver: clean up GcCutoffs names (#8379) - `horizon` is a confusing term, it's not at all obvious that this means space-based retention limit, rather than the total GC history limit. Rename to `GcCutoffs::space`. - `pitr` is less confusing, but still an unecessary level of indirection from what we really mean: a time-based condition. The fact that we use that that time-history for Point In Time Recovery doesn't mean we have to refer to time as "pitr" everywhere. Rename to `GcCutoffs::time`. --- pageserver/src/tenant.rs | 14 +-- pageserver/src/tenant/size.rs | 61 +++++-------- pageserver/src/tenant/timeline.rs | 94 +++++++++----------- pageserver/src/tenant/timeline/compaction.rs | 4 +- 4 files changed, 75 insertions(+), 98 deletions(-) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 6333fd3b6341..dc6f42eaebaf 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -2912,7 +2912,7 @@ impl Tenant { if let Some(ancestor_id) = timeline.get_ancestor_timeline_id() { if let Some(ancestor_gc_cutoffs) = gc_cutoffs.get(&ancestor_id) { target.within_ancestor_pitr = - timeline.get_ancestor_lsn() >= ancestor_gc_cutoffs.pitr; + timeline.get_ancestor_lsn() >= ancestor_gc_cutoffs.time; } } @@ -2928,7 +2928,7 @@ impl Tenant { timeline.metrics.pitr_history_size.set( timeline .get_last_record_lsn() - .checked_sub(target.cutoffs.pitr) + .checked_sub(target.cutoffs.time) .unwrap_or(Lsn(0)) .0, ); @@ -4262,7 +4262,7 @@ mod tests { .source() .unwrap() .to_string() - .contains("is earlier than latest GC horizon")); + .contains("is earlier than latest GC cutoff")); } } @@ -6718,8 +6718,8 @@ mod tests { { // Update GC info let mut guard = tline.gc_info.write().unwrap(); - guard.cutoffs.pitr = Lsn(0x30); - guard.cutoffs.horizon = Lsn(0x30); + guard.cutoffs.time = Lsn(0x30); + guard.cutoffs.space = Lsn(0x30); } let expected_result = [ @@ -7109,8 +7109,8 @@ mod tests { *guard = GcInfo { retain_lsns: vec![], cutoffs: GcCutoffs { - pitr: Lsn(0x30), - horizon: Lsn(0x30), + time: Lsn(0x30), + space: Lsn(0x30), }, leases: Default::default(), within_ancestor_pitr: false, diff --git a/pageserver/src/tenant/size.rs b/pageserver/src/tenant/size.rs index 23354417e788..e4728ca8a8cb 100644 --- a/pageserver/src/tenant/size.rs +++ b/pageserver/src/tenant/size.rs @@ -135,11 +135,9 @@ pub struct TimelineInputs { ancestor_lsn: Lsn, last_record: Lsn, latest_gc_cutoff: Lsn, - horizon_cutoff: Lsn, - pitr_cutoff: Lsn, /// Cutoff point based on GC settings - next_gc_cutoff: Lsn, + next_pitr_cutoff: Lsn, /// Cutoff point calculated from the user-supplied 'max_retention_period' retention_param_cutoff: Option, @@ -150,7 +148,7 @@ pub struct TimelineInputs { /// Gathers the inputs for the tenant sizing model. /// -/// Tenant size does not consider the latest state, but only the state until next_gc_cutoff, which +/// Tenant size does not consider the latest state, but only the state until next_pitr_cutoff, which /// is updated on-demand, during the start of this calculation and separate from the /// [`TimelineInputs::latest_gc_cutoff`]. /// @@ -158,11 +156,8 @@ pub struct TimelineInputs { /// /// ```text /// 0-----|---------|----|------------| · · · · · |·> lsn -/// initdb_lsn branchpoints* next_gc_cutoff latest +/// initdb_lsn branchpoints* next_pitr_cutoff latest /// ``` -/// -/// Until gc_horizon_cutoff > `Timeline::last_record_lsn` for any of the tenant's timelines, the -/// tenant size will be zero. pub(super) async fn gather_inputs( tenant: &Tenant, limit: &Arc, @@ -172,7 +167,7 @@ pub(super) async fn gather_inputs( cancel: &CancellationToken, ctx: &RequestContext, ) -> Result { - // refresh is needed to update gc related pitr_cutoff and horizon_cutoff + // refresh is needed to update [`timeline::GcCutoffs`] tenant.refresh_gc_info(cancel, ctx).await?; // Collect information about all the timelines @@ -236,20 +231,18 @@ pub(super) async fn gather_inputs( // we don't consider the `Timeline::disk_consistent_lsn` at all, because we are not // actually removing files. // - // We only consider [`GcInfo::pitr_cutoff`], and not [`GcInfo::horizon_cutoff`], because from + // We only consider [`timeline::GcCutoffs::time`], and not [`timeline::GcCutoffs::space`], because from // a user's perspective they have only requested retention up to the time bound (pitr_cutoff), rather - // than a space bound (horizon cutoff). This means that if someone drops a database and waits for their + // than our internal space cutoff. This means that if someone drops a database and waits for their // PITR interval, they will see synthetic size decrease, even if we are still storing data inside - // horizon_cutoff. - let pitr_cutoff = gc_info.cutoffs.pitr; - let horizon_cutoff = gc_info.cutoffs.horizon; - let mut next_gc_cutoff = pitr_cutoff; + // the space cutoff. + let mut next_pitr_cutoff = gc_info.cutoffs.time; // If the caller provided a shorter retention period, use that instead of the GC cutoff. let retention_param_cutoff = if let Some(max_retention_period) = max_retention_period { let param_cutoff = Lsn(last_record_lsn.0.saturating_sub(max_retention_period)); - if next_gc_cutoff < param_cutoff { - next_gc_cutoff = param_cutoff; + if next_pitr_cutoff < param_cutoff { + next_pitr_cutoff = param_cutoff; } Some(param_cutoff) } else { @@ -263,7 +256,7 @@ pub(super) async fn gather_inputs( .copied() .collect::>(); - // next_gc_cutoff in parent branch are not of interest (right now at least), nor do we + // next_pitr_cutoff in parent branch are not of interest (right now at least), nor do we // want to query any logical size before initdb_lsn. let branch_start_lsn = cmp::max(ancestor_lsn, timeline.initdb_lsn); @@ -291,10 +284,10 @@ pub(super) async fn gather_inputs( ) } - // Add a point for the GC cutoff - let branch_start_needed = next_gc_cutoff <= branch_start_lsn; + // Add a point for the PITR cutoff + let branch_start_needed = next_pitr_cutoff <= branch_start_lsn; if !branch_start_needed { - lsns.push((next_gc_cutoff, LsnKind::GcCutOff)); + lsns.push((next_pitr_cutoff, LsnKind::GcCutOff)); } lsns.sort_unstable(); @@ -333,7 +326,7 @@ pub(super) async fn gather_inputs( parent: Some(parent), lsn: lsn.0, size: None, - needed: lsn > next_gc_cutoff, + needed: lsn > next_pitr_cutoff, }, timeline_id: timeline.timeline_id, kind, @@ -357,8 +350,8 @@ pub(super) async fn gather_inputs( segment: Segment { parent: Some(lease_parent), lsn: lsn.0, - size: None, // Filled in later, if necessary - needed: lsn > next_gc_cutoff, // only needed if the point is within rentention. + size: None, // Filled in later, if necessary + needed: lsn > next_pitr_cutoff, // only needed if the point is within rentention. }, timeline_id: timeline.timeline_id, kind: LsnKind::LeaseStart, @@ -398,9 +391,7 @@ pub(super) async fn gather_inputs( last_record: last_record_lsn, // this is not used above, because it might not have updated recently enough latest_gc_cutoff: *timeline.get_latest_gc_cutoff_lsn(), - horizon_cutoff, - pitr_cutoff, - next_gc_cutoff, + next_pitr_cutoff, retention_param_cutoff, lease_points, }); @@ -742,9 +733,7 @@ fn verify_size_for_multiple_branches() { "ancestor_lsn": "0/18D3D98", "last_record": "0/2230CD0", "latest_gc_cutoff": "0/1698C48", - "horizon_cutoff": "0/2210CD0", - "pitr_cutoff": "0/2210CD0", - "next_gc_cutoff": "0/2210CD0", + "next_pitr_cutoff": "0/2210CD0", "retention_param_cutoff": null, "lease_points": [] }, @@ -753,9 +742,7 @@ fn verify_size_for_multiple_branches() { "ancestor_lsn": "0/176D998", "last_record": "0/1837770", "latest_gc_cutoff": "0/1698C48", - "horizon_cutoff": "0/1817770", - "pitr_cutoff": "0/1817770", - "next_gc_cutoff": "0/1817770", + "next_pitr_cutoff": "0/1817770", "retention_param_cutoff": null, "lease_points": [] }, @@ -764,9 +751,7 @@ fn verify_size_for_multiple_branches() { "ancestor_lsn": "0/0", "last_record": "0/18D3D98", "latest_gc_cutoff": "0/1698C48", - "horizon_cutoff": "0/18B3D98", - "pitr_cutoff": "0/18B3D98", - "next_gc_cutoff": "0/18B3D98", + "next_pitr_cutoff": "0/18B3D98", "retention_param_cutoff": null, "lease_points": [] } @@ -820,9 +805,7 @@ fn verify_size_for_one_branch() { "ancestor_lsn": "0/0", "last_record": "47/280A5860", "latest_gc_cutoff": "47/240A5860", - "horizon_cutoff": "47/240A5860", - "pitr_cutoff": "47/240A5860", - "next_gc_cutoff": "47/240A5860", + "next_pitr_cutoff": "47/240A5860", "retention_param_cutoff": "0/0", "lease_points": [] } diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 48a5b2d32bf7..3d3d3ac34de1 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -478,37 +478,32 @@ impl GcInfo { } } -/// The `GcInfo` component describing which Lsns need to be retained. +/// The `GcInfo` component describing which Lsns need to be retained. Functionally, this +/// is a single number (the oldest LSN which we must retain), but it internally distinguishes +/// between time-based and space-based retention for observability and consumption metrics purposes. #[derive(Debug)] pub(crate) struct GcCutoffs { - /// Keep everything newer than this point. - /// - /// This is calculated by subtracting 'gc_horizon' setting from - /// last-record LSN - /// - /// FIXME: is this inclusive or exclusive? - pub(crate) horizon: Lsn, + /// Calculated from the [`TenantConf::gc_horizon`], this LSN indicates how much + /// history we must keep to retain a specified number of bytes of WAL. + pub(crate) space: Lsn, - /// In addition to 'retain_lsns' and 'horizon_cutoff', keep everything newer than this - /// point. - /// - /// This is calculated by finding a number such that a record is needed for PITR - /// if only if its LSN is larger than 'pitr_cutoff'. - pub(crate) pitr: Lsn, + /// Calculated from [`TenantConf::pitr_interval`], this LSN indicates how much + /// history we must keep to enable reading back at least the PITR interval duration. + pub(crate) time: Lsn, } impl Default for GcCutoffs { fn default() -> Self { Self { - horizon: Lsn::INVALID, - pitr: Lsn::INVALID, + space: Lsn::INVALID, + time: Lsn::INVALID, } } } impl GcCutoffs { fn select_min(&self) -> Lsn { - std::cmp::min(self.horizon, self.pitr) + std::cmp::min(self.space, self.time) } } @@ -867,7 +862,7 @@ impl Timeline { let gc_info = self.gc_info.read().unwrap(); let history = self .get_last_record_lsn() - .checked_sub(gc_info.cutoffs.pitr) + .checked_sub(gc_info.cutoffs.time) .unwrap_or(Lsn(0)) .0; (history, gc_info.within_ancestor_pitr) @@ -1566,7 +1561,7 @@ impl Timeline { ) -> anyhow::Result<()> { ensure!( lsn >= **latest_gc_cutoff_lsn, - "LSN {} is earlier than latest GC horizon {} (we might've already garbage collected needed data)", + "LSN {} is earlier than latest GC cutoff {} (we might've already garbage collected needed data)", lsn, **latest_gc_cutoff_lsn, ); @@ -4944,18 +4939,18 @@ impl Timeline { /// garbage collection. /// /// We calculate two cutoffs, one based on time and one based on WAL size. `pitr` - /// controls the time cutoff (or ZERO to disable time-based retention), and `cutoff_horizon` controls + /// controls the time cutoff (or ZERO to disable time-based retention), and `space_cutoff` controls /// the space-based retention. /// /// This function doesn't simply to calculate time & space based retention: it treats time-based /// retention as authoritative if enabled, and falls back to space-based retention if calculating /// the LSN for a time point isn't possible. Therefore the GcCutoffs::horizon in the response might - /// be different to the `cutoff_horizon` input. Callers should treat the min() of the two cutoffs + /// be different to the `space_cutoff` input. Callers should treat the min() of the two cutoffs /// in the response as the GC cutoff point for the timeline. #[instrument(skip_all, fields(timeline_id=%self.timeline_id))] pub(super) async fn find_gc_cutoffs( &self, - cutoff_horizon: Lsn, + space_cutoff: Lsn, pitr: Duration, cancel: &CancellationToken, ctx: &RequestContext, @@ -4972,8 +4967,8 @@ impl Timeline { // Unit tests which specify zero PITR interval expect to avoid doing any I/O for timestamp lookup if pitr == Duration::ZERO { return Ok(GcCutoffs { - pitr: self.get_last_record_lsn(), - horizon: cutoff_horizon, + time: self.get_last_record_lsn(), + space: space_cutoff, }); } } @@ -4981,8 +4976,7 @@ impl Timeline { // Calculate a time-based limit on how much to retain: // - if PITR interval is set, then this is our cutoff. // - if PITR interval is not set, then we do a lookup - // based on DEFAULT_PITR_INTERVAL, so that size-based retention (horizon) - // does not result in keeping history around permanently on idle databases. + // based on DEFAULT_PITR_INTERVAL, so that size-based retention does not result in keeping history around permanently on idle databases. let time_cutoff = { let now = SystemTime::now(); let time_range = if pitr == Duration::ZERO { @@ -5023,31 +5017,31 @@ impl Timeline { // PITR is not set. Retain the size-based limit, or the default time retention, // whichever requires less data. GcCutoffs { - pitr: std::cmp::max(time_cutoff, cutoff_horizon), - horizon: std::cmp::max(time_cutoff, cutoff_horizon), + time: self.get_last_record_lsn(), + space: std::cmp::max(time_cutoff, space_cutoff), } } (Duration::ZERO, None) => { // PITR is not set, and time lookup failed GcCutoffs { - pitr: self.get_last_record_lsn(), - horizon: cutoff_horizon, + time: self.get_last_record_lsn(), + space: space_cutoff, } } (_, None) => { // PITR interval is set & we didn't look up a timestamp successfully. Conservatively assume PITR // cannot advance beyond what was already GC'd, and respect space-based retention GcCutoffs { - pitr: *self.get_latest_gc_cutoff_lsn(), - horizon: cutoff_horizon, + time: *self.get_latest_gc_cutoff_lsn(), + space: space_cutoff, } } (_, Some(time_cutoff)) => { // PITR interval is set and we looked up timestamp successfully. Ignore // size based retention and make time cutoff authoritative GcCutoffs { - pitr: time_cutoff, - horizon: time_cutoff, + time: time_cutoff, + space: time_cutoff, } } }) @@ -5074,11 +5068,11 @@ impl Timeline { return Err(GcError::TimelineCancelled); } - let (horizon_cutoff, pitr_cutoff, retain_lsns, max_lsn_with_valid_lease) = { + let (space_cutoff, time_cutoff, retain_lsns, max_lsn_with_valid_lease) = { let gc_info = self.gc_info.read().unwrap(); - let horizon_cutoff = min(gc_info.cutoffs.horizon, self.get_disk_consistent_lsn()); - let pitr_cutoff = gc_info.cutoffs.pitr; + let space_cutoff = min(gc_info.cutoffs.space, self.get_disk_consistent_lsn()); + let time_cutoff = gc_info.cutoffs.time; let retain_lsns = gc_info.retain_lsns.clone(); // Gets the maximum LSN that holds the valid lease. @@ -5088,14 +5082,14 @@ impl Timeline { let max_lsn_with_valid_lease = gc_info.leases.last_key_value().map(|(lsn, _)| *lsn); ( - horizon_cutoff, - pitr_cutoff, + space_cutoff, + time_cutoff, retain_lsns, max_lsn_with_valid_lease, ) }; - let mut new_gc_cutoff = Lsn::min(horizon_cutoff, pitr_cutoff); + let mut new_gc_cutoff = Lsn::min(space_cutoff, time_cutoff); let standby_horizon = self.standby_horizon.load(); // Hold GC for the standby, but as a safety guard do it only within some // reasonable lag. @@ -5124,8 +5118,8 @@ impl Timeline { let res = self .gc_timeline( - horizon_cutoff, - pitr_cutoff, + space_cutoff, + time_cutoff, retain_lsns, max_lsn_with_valid_lease, new_gc_cutoff, @@ -5143,8 +5137,8 @@ impl Timeline { async fn gc_timeline( &self, - horizon_cutoff: Lsn, - pitr_cutoff: Lsn, + space_cutoff: Lsn, + time_cutoff: Lsn, retain_lsns: Vec, max_lsn_with_valid_lease: Option, new_gc_cutoff: Lsn, @@ -5205,22 +5199,22 @@ impl Timeline { result.layers_total += 1; // 1. Is it newer than GC horizon cutoff point? - if l.get_lsn_range().end > horizon_cutoff { + if l.get_lsn_range().end > space_cutoff { debug!( - "keeping {} because it's newer than horizon_cutoff {}", + "keeping {} because it's newer than space_cutoff {}", l.layer_name(), - horizon_cutoff, + space_cutoff, ); result.layers_needed_by_cutoff += 1; continue 'outer; } // 2. It is newer than PiTR cutoff point? - if l.get_lsn_range().end > pitr_cutoff { + if l.get_lsn_range().end > time_cutoff { debug!( - "keeping {} because it's newer than pitr_cutoff {}", + "keeping {} because it's newer than time_cutoff {}", l.layer_name(), - pitr_cutoff, + time_cutoff, ); result.layers_needed_by_pitr += 1; continue 'outer; diff --git a/pageserver/src/tenant/timeline/compaction.rs b/pageserver/src/tenant/timeline/compaction.rs index eec5e5e53cf7..cbb330334104 100644 --- a/pageserver/src/tenant/timeline/compaction.rs +++ b/pageserver/src/tenant/timeline/compaction.rs @@ -195,7 +195,7 @@ impl Timeline { tracing::info!( "latest_gc_cutoff: {}, pitr cutoff {}", *latest_gc_cutoff, - self.gc_info.read().unwrap().cutoffs.pitr + self.gc_info.read().unwrap().cutoffs.time ); let layers = self.layers.read().await; @@ -990,7 +990,7 @@ impl Timeline { "enhanced legacy compaction currently does not support retain_lsns (branches)" ))); } - let gc_cutoff = Lsn::min(gc_info.cutoffs.horizon, gc_info.cutoffs.pitr); + let gc_cutoff = gc_info.cutoffs.select_min(); let mut selected_layers = Vec::new(); // TODO: consider retain_lsns drop(gc_info); From 7e818ee390b541115167e20059948d1ccaf041f9 Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Fri, 12 Jul 2024 13:46:14 -0500 Subject: [PATCH 163/194] Rename compute migrations to start at 1 This matches what we put into the neon_migration.migration_id table. --- compute_tools/src/migration.rs | 17 +++++++++++++--- ...sql => 0001-neon_superuser_bypass_rls.sql} | 0 ...1-alter_roles.sql => 0002-alter_roles.sql} | 0 ...create_subscription_to_neon_superuser.sql} | 0 ...04-grant_pg_monitor_to_neon_superuser.sql} | 0 ...grant_all_on_tables_to_neon_superuser.sql} | 0 ...nt_all_on_sequences_to_neon_superuser.sql} | 0 ...s_to_neon_superuser_with_grant_option.sql} | 0 ...s_to_neon_superuser_with_grant_option.sql} | 0 ...lication_for_previously_allowed_roles.sql} | 0 ...nchronization_funcs_to_neon_superuser.sql} | 0 compute_tools/src/spec.rs | 20 +++++++++---------- 12 files changed, 24 insertions(+), 13 deletions(-) rename compute_tools/src/migrations/{0000-neon_superuser_bypass_rls.sql => 0001-neon_superuser_bypass_rls.sql} (100%) rename compute_tools/src/migrations/{0001-alter_roles.sql => 0002-alter_roles.sql} (100%) rename compute_tools/src/migrations/{0002-grant_pg_create_subscription_to_neon_superuser.sql => 0003-grant_pg_create_subscription_to_neon_superuser.sql} (100%) rename compute_tools/src/migrations/{0003-grant_pg_monitor_to_neon_superuser.sql => 0004-grant_pg_monitor_to_neon_superuser.sql} (100%) rename compute_tools/src/migrations/{0004-grant_all_on_tables_to_neon_superuser.sql => 0005-grant_all_on_tables_to_neon_superuser.sql} (100%) rename compute_tools/src/migrations/{0005-grant_all_on_sequences_to_neon_superuser.sql => 0006-grant_all_on_sequences_to_neon_superuser.sql} (100%) rename compute_tools/src/migrations/{0006-grant_all_on_tables_to_neon_superuser_with_grant_option.sql => 0007-grant_all_on_tables_to_neon_superuser_with_grant_option.sql} (100%) rename compute_tools/src/migrations/{0007-grant_all_on_sequences_to_neon_superuser_with_grant_option.sql => 0008-grant_all_on_sequences_to_neon_superuser_with_grant_option.sql} (100%) rename compute_tools/src/migrations/{0008-revoke_replication_for_previously_allowed_roles.sql => 0009-revoke_replication_for_previously_allowed_roles.sql} (100%) rename compute_tools/src/migrations/{0009-grant_snapshot_synchronization_funcs_to_neon_superuser.sql => 0010-grant_snapshot_synchronization_funcs_to_neon_superuser.sql} (100%) diff --git a/compute_tools/src/migration.rs b/compute_tools/src/migration.rs index 61dcf01c8448..241ccd41001a 100644 --- a/compute_tools/src/migration.rs +++ b/compute_tools/src/migration.rs @@ -66,17 +66,28 @@ impl<'m> MigrationRunner<'m> { .context("run_migrations begin")?; while current_migration < self.migrations.len() { + macro_rules! migration_id { + ($cm:expr) => { + ($cm + 1) as i64 + }; + } + let migration = self.migrations[current_migration]; if migration.starts_with("-- SKIP") { - info!("Skipping migration id={}", current_migration); + info!("Skipping migration id={}", migration_id!(current_migration)); } else { info!( "Running migration id={}:\n{}\n", - current_migration, migration + migration_id!(current_migration), + migration ); + self.client.simple_query(migration).with_context(|| { - format!("run_migration current_migration={}", current_migration) + format!( + "run_migration migration id={}", + migration_id!(current_migration) + ) })?; } diff --git a/compute_tools/src/migrations/0000-neon_superuser_bypass_rls.sql b/compute_tools/src/migrations/0001-neon_superuser_bypass_rls.sql similarity index 100% rename from compute_tools/src/migrations/0000-neon_superuser_bypass_rls.sql rename to compute_tools/src/migrations/0001-neon_superuser_bypass_rls.sql diff --git a/compute_tools/src/migrations/0001-alter_roles.sql b/compute_tools/src/migrations/0002-alter_roles.sql similarity index 100% rename from compute_tools/src/migrations/0001-alter_roles.sql rename to compute_tools/src/migrations/0002-alter_roles.sql diff --git a/compute_tools/src/migrations/0002-grant_pg_create_subscription_to_neon_superuser.sql b/compute_tools/src/migrations/0003-grant_pg_create_subscription_to_neon_superuser.sql similarity index 100% rename from compute_tools/src/migrations/0002-grant_pg_create_subscription_to_neon_superuser.sql rename to compute_tools/src/migrations/0003-grant_pg_create_subscription_to_neon_superuser.sql diff --git a/compute_tools/src/migrations/0003-grant_pg_monitor_to_neon_superuser.sql b/compute_tools/src/migrations/0004-grant_pg_monitor_to_neon_superuser.sql similarity index 100% rename from compute_tools/src/migrations/0003-grant_pg_monitor_to_neon_superuser.sql rename to compute_tools/src/migrations/0004-grant_pg_monitor_to_neon_superuser.sql diff --git a/compute_tools/src/migrations/0004-grant_all_on_tables_to_neon_superuser.sql b/compute_tools/src/migrations/0005-grant_all_on_tables_to_neon_superuser.sql similarity index 100% rename from compute_tools/src/migrations/0004-grant_all_on_tables_to_neon_superuser.sql rename to compute_tools/src/migrations/0005-grant_all_on_tables_to_neon_superuser.sql diff --git a/compute_tools/src/migrations/0005-grant_all_on_sequences_to_neon_superuser.sql b/compute_tools/src/migrations/0006-grant_all_on_sequences_to_neon_superuser.sql similarity index 100% rename from compute_tools/src/migrations/0005-grant_all_on_sequences_to_neon_superuser.sql rename to compute_tools/src/migrations/0006-grant_all_on_sequences_to_neon_superuser.sql diff --git a/compute_tools/src/migrations/0006-grant_all_on_tables_to_neon_superuser_with_grant_option.sql b/compute_tools/src/migrations/0007-grant_all_on_tables_to_neon_superuser_with_grant_option.sql similarity index 100% rename from compute_tools/src/migrations/0006-grant_all_on_tables_to_neon_superuser_with_grant_option.sql rename to compute_tools/src/migrations/0007-grant_all_on_tables_to_neon_superuser_with_grant_option.sql diff --git a/compute_tools/src/migrations/0007-grant_all_on_sequences_to_neon_superuser_with_grant_option.sql b/compute_tools/src/migrations/0008-grant_all_on_sequences_to_neon_superuser_with_grant_option.sql similarity index 100% rename from compute_tools/src/migrations/0007-grant_all_on_sequences_to_neon_superuser_with_grant_option.sql rename to compute_tools/src/migrations/0008-grant_all_on_sequences_to_neon_superuser_with_grant_option.sql diff --git a/compute_tools/src/migrations/0008-revoke_replication_for_previously_allowed_roles.sql b/compute_tools/src/migrations/0009-revoke_replication_for_previously_allowed_roles.sql similarity index 100% rename from compute_tools/src/migrations/0008-revoke_replication_for_previously_allowed_roles.sql rename to compute_tools/src/migrations/0009-revoke_replication_for_previously_allowed_roles.sql diff --git a/compute_tools/src/migrations/0009-grant_snapshot_synchronization_funcs_to_neon_superuser.sql b/compute_tools/src/migrations/0010-grant_snapshot_synchronization_funcs_to_neon_superuser.sql similarity index 100% rename from compute_tools/src/migrations/0009-grant_snapshot_synchronization_funcs_to_neon_superuser.sql rename to compute_tools/src/migrations/0010-grant_snapshot_synchronization_funcs_to_neon_superuser.sql diff --git a/compute_tools/src/spec.rs b/compute_tools/src/spec.rs index 1d12b88c7ce3..6a872638219f 100644 --- a/compute_tools/src/spec.rs +++ b/compute_tools/src/spec.rs @@ -777,21 +777,21 @@ pub fn handle_migrations(client: &mut Client) -> Result<()> { // Add new migrations in numerical order. let migrations = [ - include_str!("./migrations/0000-neon_superuser_bypass_rls.sql"), - include_str!("./migrations/0001-alter_roles.sql"), - include_str!("./migrations/0002-grant_pg_create_subscription_to_neon_superuser.sql"), - include_str!("./migrations/0003-grant_pg_monitor_to_neon_superuser.sql"), - include_str!("./migrations/0004-grant_all_on_tables_to_neon_superuser.sql"), - include_str!("./migrations/0005-grant_all_on_sequences_to_neon_superuser.sql"), + include_str!("./migrations/0001-neon_superuser_bypass_rls.sql"), + include_str!("./migrations/0002-alter_roles.sql"), + include_str!("./migrations/0003-grant_pg_create_subscription_to_neon_superuser.sql"), + include_str!("./migrations/0004-grant_pg_monitor_to_neon_superuser.sql"), + include_str!("./migrations/0005-grant_all_on_tables_to_neon_superuser.sql"), + include_str!("./migrations/0006-grant_all_on_sequences_to_neon_superuser.sql"), include_str!( - "./migrations/0006-grant_all_on_tables_to_neon_superuser_with_grant_option.sql" + "./migrations/0007-grant_all_on_tables_to_neon_superuser_with_grant_option.sql" ), include_str!( - "./migrations/0007-grant_all_on_sequences_to_neon_superuser_with_grant_option.sql" + "./migrations/0008-grant_all_on_sequences_to_neon_superuser_with_grant_option.sql" ), - include_str!("./migrations/0008-revoke_replication_for_previously_allowed_roles.sql"), + include_str!("./migrations/0009-revoke_replication_for_previously_allowed_roles.sql"), include_str!( - "./migrations/0009-grant_snapshot_synchronization_funcs_to_neon_superuser.sql" + "./migrations/0010-grant_snapshot_synchronization_funcs_to_neon_superuser.sql" ), ]; From 85d47637ee2875b42d3068ff2cbb9567d1c49e56 Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Fri, 12 Jul 2024 13:38:51 -0500 Subject: [PATCH 164/194] Run each migration in its own transaction Previously, every migration was run in the same transaction. This is preparatory work for fixing CVE-2024-4317. --- compute_tools/src/migration.rs | 46 +++++++++++--------------- test_runner/fixtures/neon_fixtures.py | 6 ++-- test_runner/regress/test_migrations.py | 7 +--- 3 files changed, 24 insertions(+), 35 deletions(-) diff --git a/compute_tools/src/migration.rs b/compute_tools/src/migration.rs index 241ccd41001a..22ab145edaec 100644 --- a/compute_tools/src/migration.rs +++ b/compute_tools/src/migration.rs @@ -9,6 +9,9 @@ pub(crate) struct MigrationRunner<'m> { impl<'m> MigrationRunner<'m> { pub fn new(client: &'m mut Client, migrations: &'m [&'m str]) -> Self { + // The neon_migration.migration_id::id column is a bigint, which is equivalent to an i64 + assert!(migrations.len() + 1 < i64::MAX as usize); + Self { client, migrations } } @@ -22,11 +25,8 @@ impl<'m> MigrationRunner<'m> { Ok(row.get::<&str, i64>("id")) } - fn update_migration_id(&mut self) -> Result<()> { - let setval = format!( - "UPDATE neon_migration.migration_id SET id={}", - self.migrations.len() - ); + fn update_migration_id(&mut self, migration_id: i64) -> Result<()> { + let setval = format!("UPDATE neon_migration.migration_id SET id={}", migration_id); self.client .simple_query(&setval) @@ -57,14 +57,7 @@ impl<'m> MigrationRunner<'m> { pub fn run_migrations(mut self) -> Result<()> { self.prepare_migrations()?; - let mut current_migration: usize = self.get_migration_id()? as usize; - let starting_migration_id = current_migration; - - let query = "BEGIN"; - self.client - .simple_query(query) - .context("run_migrations begin")?; - + let mut current_migration = self.get_migration_id()? as usize; while current_migration < self.migrations.len() { macro_rules! migration_id { ($cm:expr) => { @@ -83,28 +76,29 @@ impl<'m> MigrationRunner<'m> { migration ); + self.client + .simple_query("BEGIN") + .context("begin migration")?; + self.client.simple_query(migration).with_context(|| { format!( - "run_migration migration id={}", + "run_migrations migration id={}", migration_id!(current_migration) ) })?; - } - current_migration += 1; - } + // Migration IDs start at 1 + self.update_migration_id(migration_id!(current_migration))?; - self.update_migration_id()?; + self.client + .simple_query("COMMIT") + .context("commit migration")?; - let query = "COMMIT"; - self.client - .simple_query(query) - .context("run_migrations commit")?; + info!("Finished migration id={}", migration_id!(current_migration)); + } - info!( - "Ran {} migrations", - (self.migrations.len() - starting_migration_id) - ); + current_migration += 1; + } Ok(()) } diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 625e9096f58f..4766b7251624 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -3798,13 +3798,13 @@ def respec(self, **kwargs): json.dump(dict(data_dict, **kwargs), file, indent=4) # Please note: Migrations only run if pg_skip_catalog_updates is false - def wait_for_migrations(self): + def wait_for_migrations(self, num_migrations: int = 10): with self.cursor() as cur: def check_migrations_done(): cur.execute("SELECT id FROM neon_migration.migration_id") - migration_id = cur.fetchall()[0][0] - assert migration_id != 0 + migration_id: int = cur.fetchall()[0][0] + assert migration_id >= num_migrations wait_until(20, 0.5, check_migrations_done) diff --git a/test_runner/regress/test_migrations.py b/test_runner/regress/test_migrations.py index 91bd3ea50caf..880dead4e8d9 100644 --- a/test_runner/regress/test_migrations.py +++ b/test_runner/regress/test_migrations.py @@ -11,17 +11,14 @@ def test_migrations(neon_simple_env: NeonEnv): endpoint.respec(skip_pg_catalog_updates=False) endpoint.start() - endpoint.wait_for_migrations() - num_migrations = 10 + endpoint.wait_for_migrations(num_migrations=num_migrations) with endpoint.cursor() as cur: cur.execute("SELECT id FROM neon_migration.migration_id") migration_id = cur.fetchall() assert migration_id[0][0] == num_migrations - endpoint.assert_log_contains(f"INFO handle_migrations: Ran {num_migrations} migrations") - endpoint.stop() endpoint.start() # We don't have a good way of knowing that the migrations code path finished executing @@ -31,5 +28,3 @@ def test_migrations(neon_simple_env: NeonEnv): cur.execute("SELECT id FROM neon_migration.migration_id") migration_id = cur.fetchall() assert migration_id[0][0] == num_migrations - - endpoint.assert_log_contains("INFO handle_migrations: Ran 0 migrations") From ad5d784fb776de2ba6c2d4adf88e9ababe554cd7 Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Mon, 15 Jul 2024 10:30:04 -0500 Subject: [PATCH 165/194] Hide import behind TYPE_CHECKING --- test_runner/regress/test_migrations.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test_runner/regress/test_migrations.py b/test_runner/regress/test_migrations.py index 880dead4e8d9..bdc5ca907ec1 100644 --- a/test_runner/regress/test_migrations.py +++ b/test_runner/regress/test_migrations.py @@ -1,6 +1,10 @@ +from __future__ import annotations + import time +from typing import TYPE_CHECKING -from fixtures.neon_fixtures import NeonEnv +if TYPE_CHECKING: + from fixtures.neon_fixtures import NeonEnv def test_migrations(neon_simple_env: NeonEnv): From 18e7c2b7a10f779a0253b713c0ca3ffd7a1b6c1a Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Mon, 15 Jul 2024 10:35:49 -0500 Subject: [PATCH 166/194] Add some typing to Endpoint.respec() --- test_runner/fixtures/neon_fixtures.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 4766b7251624..2765ff916e63 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -3785,12 +3785,12 @@ def reconfigure( self.endpoint_id, self.tenant_id, pageserver_id, self.active_safekeepers ) - def respec(self, **kwargs): + def respec(self, **kwargs: Any) -> None: """Update the endpoint.json file used by control_plane.""" # Read config config_path = os.path.join(self.endpoint_path(), "endpoint.json") with open(config_path, "r") as f: - data_dict = json.load(f) + data_dict: dict[str, Any] = json.load(f) # Write it back updated with open(config_path, "w") as file: From abe3b4e005a918f5d5ed242095d3cf9a9b4b57a5 Mon Sep 17 00:00:00 2001 From: "Alex Chi Z." Date: Tue, 16 Jul 2024 15:43:24 -0400 Subject: [PATCH 167/194] fix(pageserver): limit num of delta layers for l0 compaction (#8391) ## Problem close https://github.com/neondatabase/neon/issues/8389 ## Summary of changes A quick mitigation for tenants with fast writes. We compact at most 60 delta layers at a time, expecting a memory footprint of 15GB. We will pick the oldest 60 L0 layers. This should be a relatively safe change so no test is added. Question is whether to make this parameter configurable via tenant config. --------- Signed-off-by: Alex Chi Z Co-authored-by: John Spray --- pageserver/src/tenant/timeline/compaction.rs | 31 ++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/pageserver/src/tenant/timeline/compaction.rs b/pageserver/src/tenant/timeline/compaction.rs index cbb330334104..f251b667c2fb 100644 --- a/pageserver/src/tenant/timeline/compaction.rs +++ b/pageserver/src/tenant/timeline/compaction.rs @@ -26,6 +26,7 @@ use utils::id::TimelineId; use crate::context::{AccessStatsBehavior, RequestContext, RequestContextBuilder}; use crate::page_cache; +use crate::tenant::config::defaults::{DEFAULT_CHECKPOINT_DISTANCE, DEFAULT_COMPACTION_THRESHOLD}; use crate::tenant::storage_layer::{AsLayerDesc, PersistentLayerDesc}; use crate::tenant::timeline::{drop_rlock, DeltaLayerWriter, ImageLayerWriter}; use crate::tenant::timeline::{Hole, ImageLayerCreationOutcome}; @@ -415,6 +416,7 @@ impl Timeline { .map(|x| guard.get_from_desc(&x)) .collect_vec(); stats.level0_deltas_count = Some(level0_deltas.len()); + // Only compact if enough layers have accumulated. let threshold = self.get_compaction_threshold(); if level0_deltas.is_empty() || level0_deltas.len() < threshold { @@ -445,6 +447,22 @@ impl Timeline { let mut prev_lsn_end = first_level0_delta.layer_desc().lsn_range.end; let mut deltas_to_compact = Vec::with_capacity(level0_deltas.len()); + // Accumulate the size of layers in `deltas_to_compact` + let mut deltas_to_compact_bytes = 0; + + // Under normal circumstances, we will accumulate up to compaction_interval L0s of size + // checkpoint_distance each. To avoid edge cases using extra system resources, bound our + // work in this function to only operate on this much delta data at once. + // + // Take the max of the configured value & the default, so that tests that configure tiny values + // can still use a sensible amount of memory, but if a deployed system configures bigger values we + // still let them compact a full stack of L0s in one go. + let delta_size_limit = std::cmp::max( + self.get_compaction_threshold(), + DEFAULT_COMPACTION_THRESHOLD, + ) as u64 + * std::cmp::max(self.get_checkpoint_distance(), DEFAULT_CHECKPOINT_DISTANCE); + deltas_to_compact.push(first_level0_delta.download_and_keep_resident().await?); for l in level0_deltas_iter { let lsn_range = &l.layer_desc().lsn_range; @@ -453,7 +471,20 @@ impl Timeline { break; } deltas_to_compact.push(l.download_and_keep_resident().await?); + deltas_to_compact_bytes += l.metadata().file_size; prev_lsn_end = lsn_range.end; + + if deltas_to_compact_bytes >= delta_size_limit { + info!( + l0_deltas_selected = deltas_to_compact.len(), + l0_deltas_total = level0_deltas.len(), + "L0 compaction picker hit max delta layer size limit: {}", + delta_size_limit + ); + + // Proceed with compaction, but only a subset of L0s + break; + } } let lsn_range = Range { start: deltas_to_compact From b21e131d115a90128d3a3c1b02ecc23fff7dda6c Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 16 Jul 2024 20:55:29 +0100 Subject: [PATCH 168/194] pageserver: exclude un-read layers from short residence statistic (#8396) ## Problem The `evictions_with_low_residence_duration` is used as an indicator of cache thrashing. However, there are situations where it is quite legitimate to only have a short residence during compaction, where a delta is downloaded, used to generate an image layer, and then discarded. This can lead to false positive alerts. ## Summary of changes - Only track low residence duration for layers that have been accessed at least once (compaction doesn't count as an access). This will give us a metric that indicates thrashing on layers that the _user_ is using, rather than those we're downloading for housekeeping purposes. Once we add "layer visibility" as an explicit property of layers, this can also be used as a cleaner condition (residence of non-visible layers should never be alertable) --- pageserver/src/tenant/storage_layer.rs | 20 ++++++++++++++++++++ pageserver/src/tenant/storage_layer/layer.rs | 20 ++++++++++++++------ test_runner/regress/test_tenant_conf.py | 11 +++++++++++ 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index 62730f88b260..2f0c45317d9a 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -676,6 +676,26 @@ impl LayerAccessStats { }, } } + + /// Whether this layer has been accessed (excluding in [`AccessStatsBehavior::Skip`]). + /// + /// This indicates whether the layer has been used for some purpose that would motivate + /// us to keep it on disk, such as for serving a getpage request. + fn accessed(&self) -> bool { + let locked = self.0.lock().unwrap(); + let inner = &locked.for_eviction_policy; + + // Consider it accessed if the most recent access is more recent than + // the most recent change in residence status. + match ( + inner.last_accesses.recent(), + inner.last_residence_changes.recent(), + ) { + (None, _) => false, + (Some(_), None) => true, + (Some(a), Some(r)) => a.when >= r.timestamp, + } + } } /// Get a layer descriptor from a layer. diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index 02069c29d264..4500bc94dd66 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -1469,14 +1469,22 @@ impl LayerInner { let duration = SystemTime::now().duration_since(local_layer_mtime); match duration { Ok(elapsed) => { - timeline - .metrics - .evictions_with_low_residence_duration - .read() - .unwrap() - .observe(elapsed); + let accessed = self.access_stats.accessed(); + if accessed { + // Only layers used for reads contribute to our "low residence" metric that is used + // to detect thrashing. Layers promoted for other reasons (e.g. compaction) are allowed + // to be rapidly evicted without contributing to this metric. + timeline + .metrics + .evictions_with_low_residence_duration + .read() + .unwrap() + .observe(elapsed); + } + tracing::info!( residence_millis = elapsed.as_millis(), + accessed, "evicted layer after known residence period" ); } diff --git a/test_runner/regress/test_tenant_conf.py b/test_runner/regress/test_tenant_conf.py index 1a8bc3b98363..9fb7324fa15c 100644 --- a/test_runner/regress/test_tenant_conf.py +++ b/test_runner/regress/test_tenant_conf.py @@ -8,6 +8,7 @@ from fixtures.pageserver.utils import assert_tenant_state, wait_for_upload from fixtures.remote_storage import LocalFsStorage, RemoteStorageKind from fixtures.utils import wait_until +from fixtures.workload import Workload def test_tenant_config(neon_env_builder: NeonEnvBuilder): @@ -265,6 +266,13 @@ def test_live_reconfig_get_evictions_low_residence_duration_metric_threshold( (tenant_id, timeline_id) = env.initial_tenant, env.initial_timeline ps_http = env.pageserver.http_client() + # When we evict/download layers, we will use this Workload to generate getpage requests + # that touch some layers, as otherwise the pageserver doesn't report totally unused layers + # as problems when they have short residence duration. + workload = Workload(env, tenant_id, timeline_id) + workload.init() + workload.write_rows(100) + def get_metric(): metrics = ps_http.get_metrics() metric = metrics.query_one( @@ -285,6 +293,7 @@ def get_metric(): assert default_value == "1day" ps_http.download_all_layers(tenant_id, timeline_id) + workload.validate() ps_http.evict_all_layers(tenant_id, timeline_id) metric = get_metric() assert int(metric.value) > 0, "metric is updated" @@ -305,6 +314,7 @@ def get_metric(): assert int(metric.value) == 0 ps_http.download_all_layers(tenant_id, timeline_id) + workload.validate() ps_http.evict_all_layers(tenant_id, timeline_id) metric = get_metric() assert int(metric.labels["low_threshold_secs"]) == 2 * 24 * 60 * 60 @@ -318,6 +328,7 @@ def get_metric(): assert int(metric.value) == 0, "value resets if label changes" ps_http.download_all_layers(tenant_id, timeline_id) + workload.validate() ps_http.evict_all_layers(tenant_id, timeline_id) metric = get_metric() assert int(metric.labels["low_threshold_secs"]) == 2 * 60 * 60 From 07e78102bf613c723e1c98e752456770524f5250 Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 16 Jul 2024 21:36:17 +0100 Subject: [PATCH 169/194] pageserver: reduce size of delta layer ValueRef (#8401) ## Problem ValueRef is an unnecessarily large structure, because it carries a cursor. L0 compaction currently instantiates gigabytes of these under some circumstances. ## Summary of changes - Carry a ref to the parent layer instead of a cursor, and construct a cursor on demand. This reduces RSS high watermark during L0 compaction by about 20%. --- .../src/tenant/storage_layer/delta_layer.rs | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 2d36ac744277..64412fe4afd1 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -1180,9 +1180,7 @@ impl DeltaLayerInner { let delta_key = DeltaKey::from_slice(key); let val_ref = ValueRef { blob_ref: BlobRef(value), - reader: BlockCursor::new(crate::tenant::block_io::BlockReaderRef::Adapter( - Adapter(self), - )), + layer: self, }; let pos = BlobRef(value).pos(); if let Some(last) = all_keys.last_mut() { @@ -1426,7 +1424,7 @@ impl DeltaLayerInner { let keys = self.load_keys(ctx).await?; async fn dump_blob(val: &ValueRef<'_>, ctx: &RequestContext) -> anyhow::Result { - let buf = val.reader.read_blob(val.blob_ref.pos(), ctx).await?; + let buf = val.load_raw(ctx).await?; let val = Value::des(&buf)?; let desc = match val { Value::Image(img) => { @@ -1461,8 +1459,7 @@ impl DeltaLayerInner { use pageserver_api::key::CHECKPOINT_KEY; use postgres_ffi::CheckPoint; if key == CHECKPOINT_KEY { - let buf = val.reader.read_blob(val.blob_ref.pos(), ctx).await?; - let val = Value::des(&buf)?; + let val = val.load(ctx).await?; match val { Value::Image(img) => { let checkpoint = CheckPoint::decode(&img)?; @@ -1547,17 +1544,24 @@ pub struct DeltaEntry<'a> { /// Reference to an on-disk value pub struct ValueRef<'a> { blob_ref: BlobRef, - reader: BlockCursor<'a>, + layer: &'a DeltaLayerInner, } impl<'a> ValueRef<'a> { /// Loads the value from disk pub async fn load(&self, ctx: &RequestContext) -> Result { - // theoretically we *could* record an access time for each, but it does not really matter - let buf = self.reader.read_blob(self.blob_ref.pos(), ctx).await?; + let buf = self.load_raw(ctx).await?; let val = Value::des(&buf)?; Ok(val) } + + async fn load_raw(&self, ctx: &RequestContext) -> Result> { + let reader = BlockCursor::new(crate::tenant::block_io::BlockReaderRef::Adapter(Adapter( + self.layer, + ))); + let buf = reader.read_blob(self.blob_ref.pos(), ctx).await?; + Ok(buf) + } } pub(crate) struct Adapter(T); From d51ca338c453e0243428d6156cbee4fcbc8bdd05 Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 17 Jul 2024 15:25:35 +0100 Subject: [PATCH 170/194] docs/rfcs: timeline ancestor detach API (#6888) ## Problem When a tenant creates a new timeline that they will treat as their 'main' history, it is awkward to permanently retain an 'old main' timeline as its ancestor. Currently this is necessary because it is forbidden to delete a timeline which has descendents. ## Summary of changes A new pageserver API is proposed to 'adopt' data from a parent timeline into one of its children, such that the link between ancestor and child can be severed, leaving the parent in a state where it may then be deleted. --------- Co-authored-by: Joonas Koivunen --- docs/rfcs/034-ancestor-deletion.md | 252 +++++++++++++++++++++++++++++ 1 file changed, 252 insertions(+) create mode 100644 docs/rfcs/034-ancestor-deletion.md diff --git a/docs/rfcs/034-ancestor-deletion.md b/docs/rfcs/034-ancestor-deletion.md new file mode 100644 index 000000000000..7341d930e26d --- /dev/null +++ b/docs/rfcs/034-ancestor-deletion.md @@ -0,0 +1,252 @@ +# Ancestor Timeline Deletion + +Created on: 2024-02-23 + +Author: John Spray + +# Summary + +When a tenant creates a new timeline that they will treat as their 'main' history, +it is awkward to permanently retain an 'old main' timeline as its ancestor. Currently +this is necessary because it is forbidden to delete a timeline which has descendents. + +A new pageserver API is proposed to 'adopt' data from a parent timeline into +one of its children, such that the link between ancestor and child can be severed, +leaving the parent in a state where it may then be deleted. + +# Motivation + +Retaining parent timelines currently has two costs: + +- Cognitive load on users, who have to remember which is the "real" main timeline. +- Storage capacity cost, as the parent timeline will retain layers up to the + child's timeline point, even if the child fully covers its keyspace with image + layers and will never actually read from the parent. + +# Solution + +A new pageserver API `PUT /v1/tenant/:tenant_id/timeline/:timeline_id/detach_ancestor` +will be added. The `timeline_id` in this URL is that of the _child_ timeline that we +wish to detach from its parent. + +On success, this API will leave the following state: + +- The detached child timeline will no longer have an ancestor, and will contain all + the data needed to service reads without recursing into an ancestor. +- Any other children of the parent whose timeline points were at a lower LSN than + the detached child timeline will be modified to have the child timeline as their + new parent. +- The parent timeline will still exist, but the child will no longer have it as an + ancestor. If this was the last timeline that depended on the parent, then the + parent will become deletable. + +This API's implementation will consist of a series of retryable steps, such that +on failures/timeout it can safely be called again to reach the target state. + +## Example + +### Before + +The user has "rolled back" their project to LSN X, resulting in a "new main" +timeline. The parent "old main" timeline still exists, and they would like +to clean it up. + +They have two other timelines A and B. A is from before the rollback point, +and B is from after the rollback point. + +``` +----"old main" timeline-------X--------------------------------------------> + | | | + |-> child A | | + |-> "new main" timeline | + -> child B + +``` + +### After calling detach ancestor API + +The "new main" timeline is no longer dependent on old main, and neither +is child A, because it had a branch point before X. + +The user may now choose to delete child B and "old main" to get to +a pristine state. Child B is likely to be unwanted since the user +chose to roll back to X, and it branches from after X. However, we +don't assume this in the API; it is up to the user to delete it. + +``` +|----"old main" timeline----------------------------------------------------> + | + | + | + -> child B + +|----"new main" timeline---------> + | + |-> child A + + +``` + +### After removing timelines + +We end up with a totally clean state that leaves no trace that a rollback +ever happened: there is only one root timeline. + +``` +| ----"new main" timeline-----------> + | + |-> child A + + +``` + +## Caveats + +Important things for API users to bear in mind: + +- this API does not delete the parent timeline: you must still do that explicitly. +- if there are other child timelines ahead of the branch point of the detached + child, the parent won't be deletable: you must either delete or detach those + children. +- do _not_ simply loop over all children and detach them all: this can have an + extremely high storage cost. The detach ancestor API is intended for use on a single + timeline to make it the new "main". +- The detach ancestor API should also not be + exposed directly to the user as button/API, because they might decide + to click it for all the children and thereby generate many copies of the + parent's data -- the detach ancestor API should be used as part + of a high level "clean up after rollback" feature. + +## `detach_ancestor` API implementation + +Terms used in the following sections: + +- "the child": the timeline whose ID is specified in the detach ancestor API URL, also + called "new main" in the example. +- "the parent": the parent of "the child". Also called "old main" in the example. +- "the branch point" the ancestor_lsn of "the child" + +### Phase 1: write out adopted layers to S3 + +The child will "adopt" layers from the parent, such that its end state contains +all the parent's history as well as its own. + +For all layers in the parent's layer map whose high LSN is below the branch +point, issue S3 CopyObject requests to duplicate them into the child timeline's +prefix. Do not add them to the child's layer map yet. + +For delta layers in the parent's layer map which straddle the branch point, read them +and write out only content up to the branch point into new layer objects. + +This is a long running operation if the parent has many layers: it should be +implemented in a way that resumes rather than restarting from scratch, if the API +times out and is called again. + +As an optimization, if there are no other timelines that will be adopted into +the child, _and_ the child's image layers already full cover the branch LSN, +then we may skip adopting layers. + +### Phase 2: update the child's index + +Having written out all needed layers in phase 1, atomically link them all +into the child's IndexPart and upload to S3. This may be done while the +child Timeline is still running. + +### Phase 3: modify timelines ancestry + +Modify the child's ancestor to None, and upload its IndexPart to persist the change. + +For all timelines which have the same parent as the child, and have a branch +point lower than our branch point, switch their ancestor_timeline to the child, +and upload their IndexPart to persist the change. + +## Alternatives considered + +### Generate full image layer on child, rather than adopting parent deltas + +This would work for the case of a single child, but would prevent re-targeting +other timelines that depended on the parent. If we detached many children this +way, the storage cost would become prohibitive (consider a 1TB database with +100 child timelines: it would cost 100TiB if they all generated their own image layers). + +### Don't rewrite anything: just fake it in the API + +We could add a layer of indirection that let a child "pretend" that it had no +ancestor, when in reality it still had the parent. The pageserver API could +accept deletion of ancestor timelines, and just update child metadata to make +them look like they have no ancestor. + +This would not achieve the desired reduction in storage cost, and may well be more +complex to maintain than simply implementing the API described in this RFC. + +### Avoid copying objects: enable child index to use parent layers directly + +We could teach IndexPart to store a TimelineId for each layer, such that a child +timeline could reference a parent's layers directly, rather than copying them +into the child's prefix. + +This would impose a cost for the normal case of indices that only target the +timeline's own layers, add complexity, and break the useful simplifying +invariant that timelines "own" their own path. If child timelines were +referencing layers from the parent, we would have to ensure that the parent +never runs GC/compaction again, which would make the API less flexible (the +proposal in this RFC enables deletion of the parent but doesn't require it.) + +## Performance + +### Adopting layers + +- CopyObject is a relatively cheap operation, but we may need to issue tens of thousands + of such requests: this can take up to tens of seconds and will compete for RemoteStorage + semaphore units with other activity on the pageserver. +- If we are running on storage backend that doesn't implement CopyObject, then + this part will be much more expensive as we would stream all layer content + through the pageserver. This is no different to issuing a lot + of reads to a timeline that does not have a warm local cache: it will move + a lot of gigabytes, but that shouldn't break anything. +- Generating truncated layers for delta that straddle the branch point will + require streaming read/write of all the layers in question. + +### Updating timeline ancestry + +The simplest way to update timeline ancestry will probably be to stop and start +all the Timeline objects: this is preferable to the complexity of making their +ancestry mutable at runtime. + +There will be a corresponding "stutter" in the availability of the timelines, +of the order 10-100ms, which is the time taken to upload their IndexPart, and +restart the Timeline. + +# Interaction with other features + +## Concurrent timeline creation + +If new historic timelines are created using the parent as an ancestor while the +detach ancestor API is running, they will not be re-parented to the child. This +doesn't break anything, but it leaves the parent in a state where it might not +be possible to delete it. + +Since timeline creations are an explicit user action, this is not something we need to +worry about as the storage layer: a user who wants to delete their parent timeline will not create +new children, and if they do, they can choose to delete those children to +enable deleting the parent. + +For the least surprise to the user, before starting the detach ancestor branch +operation, the control plane should wait until all branches are created and not +allow any branches to be created before the branch point on the ancestor branch +while the operation is ongoing. + +## WAL based disaster recovery + +WAL based disaster recovery currently supports only restoring of the main +branch. Enabling WAL based disaster recovery in the future requires that we +keep a record which timeline generated the WAL and at which LSN was a parent +detached. Keep a list of timeline ids and the LSN in which they were detached in +the `index_part.json`. Limit the size of the list to 100 first entries, after +which the WAL disaster recovery will not be possible. + +## Sharded tenants + +For sharded tenants, calls to the detach ancestor API will pass through the storage +controller, which will handle them the same as timeline creations: invoke first +on shard zero, and then on all the other shards. From 9f796ebba9c089881fa66f402c08cb10df370b44 Mon Sep 17 00:00:00 2001 From: Peter Bendel Date: Wed, 17 Jul 2024 16:56:32 +0200 Subject: [PATCH 171/194] Bodobolero/pgbench compare azure (#8409) ## Problem We want to run performance tests on all supported cloud providers. We want to run most tests on the postgres version which is default for new projects in production, currently (July 24) this is postgres version 16 ## Summary of changes - change default postgres version for some (performance) tests to 16 (which is our default for new projects in prod anyhow) - add azure region to pgbench_compare jobs - add azure region to pgvector benchmarking jobs - re-used project `weathered-snowflake-88107345` was prepared with 1 million embeddings running on 7 minCU 7 maxCU in azure region to compare with AWS region (pgvector indexing and hnsw queries) - see job pgbench-pgvector - Note we now have a 11 environments combinations where we run pgbench-compare and 5 are for k8s-pod (deprecated) which we can remove in the future once auto-scaling team approves. ## Logs A current run with the changes from this pull request is running here https://github.com/neondatabase/neon/actions/runs/9972096222 Note that we currently expect some failures due to - https://github.com/neondatabase/neon/issues/8275 - instability of projects on azure region --- .../actions/neon-project-create/action.yml | 4 +- .github/workflows/benchmarking.yml | 70 ++++++++++++++----- 2 files changed, 56 insertions(+), 18 deletions(-) diff --git a/.github/actions/neon-project-create/action.yml b/.github/actions/neon-project-create/action.yml index 16759ad03820..d4029bd37c1b 100644 --- a/.github/actions/neon-project-create/action.yml +++ b/.github/actions/neon-project-create/action.yml @@ -9,8 +9,8 @@ inputs: description: 'Region ID, if not set the project will be created in the default region' default: aws-us-east-2 postgres_version: - description: 'Postgres version; default is 15' - default: '15' + description: 'Postgres version; default is 16' + default: '16' api_host: description: 'Neon API host' default: console-stage.neon.build diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index d038f64f15b0..d785156a29b1 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -59,7 +59,7 @@ jobs: strategy: matrix: include: - - DEFAULT_PG_VERSION: 14 + - DEFAULT_PG_VERSION: 16 PLATFORM: "neon-staging" region_id: ${{ github.event.inputs.region_id || 'aws-us-east-2' }} provisioner: 'k8s-pod' @@ -146,6 +146,7 @@ jobs: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} replication-tests: + if: ${{ github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null }} env: POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install DEFAULT_PG_VERSION: 14 @@ -190,6 +191,7 @@ jobs: run_in_parallel: false save_perf_report: ${{ env.SAVE_PERF_REPORT }} extra_params: -m remote_cluster --timeout 5400 + pg_version: ${{ env.DEFAULT_PG_VERSION }} env: VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" @@ -215,11 +217,14 @@ jobs: # Available platforms: # - neon-captest-new: Freshly created project (1 CU) # - neon-captest-freetier: Use freetier-sized compute (0.25 CU) + # - neonvm-captest-azure-new: Freshly created project (1 CU) in azure region + # - neonvm-captest-azure-freetier: Use freetier-sized compute (0.25 CU) in azure region # - neon-captest-reuse: Reusing existing project # - rds-aurora: Aurora Postgres Serverless v2 with autoscaling from 0.5 to 2 ACUs # - rds-postgres: RDS Postgres db.m5.large instance (2 vCPU, 8 GiB) with gp3 EBS storage env: RUN_AWS_RDS_AND_AURORA: ${{ github.event.inputs.run_AWS_RDS_AND_AURORA || 'false' }} + DEFAULT_REGION_ID: ${{ github.event.inputs.region_id || 'aws-us-east-2' }} runs-on: ubuntu-22.04 outputs: pgbench-compare-matrix: ${{ steps.pgbench-compare-matrix.outputs.matrix }} @@ -230,23 +235,33 @@ jobs: - name: Generate matrix for pgbench benchmark id: pgbench-compare-matrix run: | + region_id_default=${{ env.DEFAULT_REGION_ID }} matrix='{ + "pg_version" : [ + 16 + ], + "region_id" : [ + "'"$region_id_default"'" + ], "platform": [ "neon-captest-new", "neon-captest-reuse", "neonvm-captest-new" ], "db_size": [ "10gb" ], - "include": [{ "platform": "neon-captest-freetier", "db_size": "3gb" }, - { "platform": "neon-captest-new", "db_size": "50gb" }, - { "platform": "neonvm-captest-freetier", "db_size": "3gb" }, - { "platform": "neonvm-captest-new", "db_size": "50gb" }, - { "platform": "neonvm-captest-sharding-reuse", "db_size": "50gb" }] + "include": [{ "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neon-captest-freetier", "db_size": "3gb" }, + { "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neon-captest-new", "db_size": "50gb" }, + { "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-freetier", "db_size": "3gb" }, + { "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new", "db_size": "50gb" }, + { "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-freetier", "db_size": "3gb" }, + { "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-new", "db_size": "10gb" }, + { "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-new", "db_size": "50gb" }, + { "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-sharding-reuse", "db_size": "50gb" }] }' if [ "$(date +%A)" = "Saturday" ]; then - matrix=$(echo "$matrix" | jq '.include += [{ "platform": "rds-postgres", "db_size": "10gb"}, - { "platform": "rds-aurora", "db_size": "50gb"}]') + matrix=$(echo "$matrix" | jq '.include += [{ "pg_version": 14, "region_id": "'"$region_id_default"'", "platform": "rds-postgres", "db_size": "10gb"}, + { "pg_version": 14, "region_id": "'"$region_id_default"'", "platform": "rds-aurora", "db_size": "50gb"}]') fi echo "matrix=$(echo "$matrix" | jq --compact-output '.')" >> $GITHUB_OUTPUT @@ -298,7 +313,7 @@ jobs: TEST_PG_BENCH_DURATIONS_MATRIX: "60m" TEST_PG_BENCH_SCALES_MATRIX: ${{ matrix.db_size }} POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install - DEFAULT_PG_VERSION: 14 + DEFAULT_PG_VERSION: ${{ matrix.pg_version }} TEST_OUTPUT: /tmp/test_output BUILD_TYPE: remote SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }} @@ -323,14 +338,14 @@ jobs: prefix: latest - name: Create Neon Project - if: contains(fromJson('["neon-captest-new", "neon-captest-freetier", "neonvm-captest-new", "neonvm-captest-freetier"]'), matrix.platform) + if: contains(fromJson('["neon-captest-new", "neon-captest-freetier", "neonvm-captest-new", "neonvm-captest-freetier", "neonvm-azure-captest-freetier", "neonvm-azure-captest-new"]'), matrix.platform) id: create-neon-project uses: ./.github/actions/neon-project-create with: - region_id: ${{ github.event.inputs.region_id || 'aws-us-east-2' }} + region_id: ${{ matrix.region_id }} postgres_version: ${{ env.DEFAULT_PG_VERSION }} api_key: ${{ secrets.NEON_STAGING_API_KEY }} - compute_units: ${{ (matrix.platform == 'neon-captest-freetier' && '[0.25, 0.25]') || '[1, 1]' }} + compute_units: ${{ (contains(matrix.platform, 'captest-freetier') && '[0.25, 0.25]') || '[1, 1]' }} provisioner: ${{ (contains(matrix.platform, 'neonvm-') && 'k8s-neonvm') || 'k8s-pod' }} - name: Set up Connection String @@ -343,7 +358,7 @@ jobs: neonvm-captest-sharding-reuse) CONNSTR=${{ secrets.BENCHMARK_CAPTEST_SHARDING_CONNSTR }} ;; - neon-captest-new | neon-captest-freetier | neonvm-captest-new | neonvm-captest-freetier) + neon-captest-new | neon-captest-freetier | neonvm-captest-new | neonvm-captest-freetier | neonvm-azure-captest-new | neonvm-azure-captest-freetier) CONNSTR=${{ steps.create-neon-project.outputs.dsn }} ;; rds-aurora) @@ -368,6 +383,7 @@ jobs: run_in_parallel: false save_perf_report: ${{ env.SAVE_PERF_REPORT }} extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_init + pg_version: ${{ env.DEFAULT_PG_VERSION }} env: BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }} VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" @@ -381,6 +397,7 @@ jobs: run_in_parallel: false save_perf_report: ${{ env.SAVE_PERF_REPORT }} extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_simple_update + pg_version: ${{ env.DEFAULT_PG_VERSION }} env: BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }} VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" @@ -394,6 +411,7 @@ jobs: run_in_parallel: false save_perf_report: ${{ env.SAVE_PERF_REPORT }} extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_select_only + pg_version: ${{ env.DEFAULT_PG_VERSION }} env: BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }} VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" @@ -420,6 +438,12 @@ jobs: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} pgbench-pgvector: + strategy: + matrix: + include: + - PLATFORM: "neon-captest-pgvector" + - PLATFORM: "azure-captest-pgvector" + env: TEST_PG_BENCH_DURATIONS_MATRIX: "15m" TEST_PG_BENCH_SCALES_MATRIX: "1" @@ -428,7 +452,7 @@ jobs: TEST_OUTPUT: /tmp/test_output BUILD_TYPE: remote SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }} - PLATFORM: "neon-captest-pgvector" + PLATFORM: ${{ matrix.PLATFORM }} runs-on: [ self-hosted, us-east-2, x64 ] container: @@ -448,7 +472,18 @@ jobs: - name: Set up Connection String id: set-up-connstr run: | - CONNSTR=${{ secrets.BENCHMARK_PGVECTOR_CONNSTR }} + case "${PLATFORM}" in + neon-captest-pgvector) + CONNSTR=${{ secrets.BENCHMARK_PGVECTOR_CONNSTR }} + ;; + azure-captest-pgvector) + CONNSTR=${{ secrets.BENCHMARK_PGVECTOR_CONNSTR_AZURE }} + ;; + *) + echo >&2 "Unknown PLATFORM=${PLATFORM}" + exit 1 + ;; + esac echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT @@ -460,6 +495,7 @@ jobs: run_in_parallel: false save_perf_report: ${{ env.SAVE_PERF_REPORT }} extra_params: -m remote_cluster --timeout 21600 -k test_pgvector_indexing + pg_version: ${{ env.DEFAULT_PG_VERSION }} env: VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" @@ -473,6 +509,7 @@ jobs: run_in_parallel: false save_perf_report: ${{ env.SAVE_PERF_REPORT }} extra_params: -m remote_cluster --timeout 21600 + pg_version: ${{ env.DEFAULT_PG_VERSION }} env: BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }} VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" @@ -487,7 +524,7 @@ jobs: uses: slackapi/slack-github-action@v1 with: channel-id: "C033QLM5P7D" # dev-staging-stream - slack-message: "Periodic perf testing neon-captest-pgvector: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + slack-message: "Periodic perf testing ${PLATFORM}: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} @@ -735,6 +772,7 @@ jobs: run_in_parallel: false save_perf_report: ${{ env.SAVE_PERF_REPORT }} extra_params: -m remote_cluster --timeout 21600 -k test_user_examples + pg_version: ${{ env.DEFAULT_PG_VERSION }} env: VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" From a98ccd185b109cbfefe64a5d56d6ec30684a8d59 Mon Sep 17 00:00:00 2001 From: "Alex Chi Z." Date: Wed, 17 Jul 2024 11:22:38 -0400 Subject: [PATCH 172/194] test(pageserver): more k-merge tests on duplicated keys (#8404) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Existing tenants and some selection of layers might produce duplicated keys. Add tests to ensure the k-merge iterator handles it correctly. We also enforced ordering of the k-merge iterator to put images before deltas. part of https://github.com/neondatabase/neon/issues/8002 --------- Signed-off-by: Alex Chi Z Co-authored-by: Arpad Müller --- .../src/tenant/storage_layer/delta_layer.rs | 16 +- .../tenant/storage_layer/merge_iterator.rs | 163 ++++++++++++++++-- 2 files changed, 163 insertions(+), 16 deletions(-) diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 64412fe4afd1..43941b6e1739 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -1672,6 +1672,7 @@ pub(crate) mod test { use rand::RngCore; use super::*; + use crate::repository::Value; use crate::tenant::harness::TIMELINE_ID; use crate::tenant::vectored_blob_io::StreamingVectoredReadPlanner; use crate::tenant::Tenant; @@ -1681,6 +1682,7 @@ pub(crate) mod test { tenant::{disk_btree::tests::TestDisk, harness::TenantHarness}, DEFAULT_PG_VERSION, }; + use bytes::Bytes; /// Construct an index for a fictional delta layer and and then /// traverse in order to plan vectored reads for a query. Finally, @@ -2249,6 +2251,15 @@ pub(crate) mod test { (k1, l1).cmp(&(k2, l2)) } + pub(crate) fn sort_delta_value( + (k1, l1, v1): &(Key, Lsn, Value), + (k2, l2, v2): &(Key, Lsn, Value), + ) -> std::cmp::Ordering { + let order_1 = if v1.is_image() { 0 } else { 1 }; + let order_2 = if v2.is_image() { 0 } else { 1 }; + (k1, l1, order_1).cmp(&(k2, l2, order_2)) + } + pub(crate) async fn produce_delta_layer( tenant: &Tenant, tline: &Arc, @@ -2257,7 +2268,7 @@ pub(crate) mod test { ) -> anyhow::Result { deltas.sort_by(sort_delta); let (key_start, _, _) = deltas.first().unwrap(); - let (key_max, _, _) = deltas.first().unwrap(); + let (key_max, _, _) = deltas.last().unwrap(); let lsn_min = deltas.iter().map(|(_, lsn, _)| lsn).min().unwrap(); let lsn_max = deltas.iter().map(|(_, lsn, _)| lsn).max().unwrap(); let lsn_end = Lsn(lsn_max.0 + 1); @@ -2302,9 +2313,6 @@ pub(crate) mod test { #[tokio::test] async fn delta_layer_iterator() { - use crate::repository::Value; - use bytes::Bytes; - let harness = TenantHarness::create("delta_layer_iterator").unwrap(); let (tenant, ctx) = harness.load().await; diff --git a/pageserver/src/tenant/storage_layer/merge_iterator.rs b/pageserver/src/tenant/storage_layer/merge_iterator.rs index 68759f758576..0edfd4bd4075 100644 --- a/pageserver/src/tenant/storage_layer/merge_iterator.rs +++ b/pageserver/src/tenant/storage_layer/merge_iterator.rs @@ -96,15 +96,22 @@ impl<'a> std::cmp::PartialOrd for IteratorWrapper<'a> { impl<'a> std::cmp::Ord for IteratorWrapper<'a> { fn cmp(&self, other: &Self) -> std::cmp::Ordering { use std::cmp::Ordering; - let a = self.peek_next_key_lsn(); - let b = other.peek_next_key_lsn(); + let a = self.peek_next_key_lsn_value(); + let b = other.peek_next_key_lsn_value(); match (a, b) { - (Some((k1, l1)), Some((k2, l2))) => { - let loaded_1 = if self.is_loaded() { 1 } else { 0 }; - let loaded_2 = if other.is_loaded() { 1 } else { 0 }; + (Some((k1, l1, v1)), Some((k2, l2, v2))) => { + fn map_value_to_num(val: &Option<&Value>) -> usize { + match val { + None => 0, + Some(Value::Image(_)) => 1, + Some(Value::WalRecord(_)) => 2, + } + } + let order_1 = map_value_to_num(&v1); + let order_2 = map_value_to_num(&v2); // When key_lsn are the same, the unloaded iter will always appear before the loaded one. // And note that we do a reverse at the end of the comparison, so it works with the max heap. - (k1, l1, loaded_1).cmp(&(k2, l2, loaded_2)) + (k1, l1, order_1).cmp(&(k2, l2, order_2)) } (Some(_), None) => Ordering::Less, (None, Some(_)) => Ordering::Greater, @@ -137,13 +144,16 @@ impl<'a> IteratorWrapper<'a> { } } - fn peek_next_key_lsn(&self) -> Option<(&Key, Lsn)> { + fn peek_next_key_lsn_value(&self) -> Option<(&Key, Lsn, Option<&Value>)> { match self { - Self::Loaded { iter } => iter.peek().as_ref().map(|(key, lsn, _)| (key, *lsn)), + Self::Loaded { iter } => iter + .peek() + .as_ref() + .map(|(key, lsn, val)| (key, *lsn, Some(val))), Self::NotLoaded { first_key_lower_bound: (key, lsn), .. - } => Some((key, *lsn)), + } => Some((key, *lsn, None)), } } @@ -191,6 +201,13 @@ impl<'a> IteratorWrapper<'a> { } } +/// A merge iterator over delta/image layer iterators. When duplicated records are +/// found, the iterator will not perform any deduplication, and the caller should handle +/// these situation. By saying duplicated records, there are many possibilities: +/// * Two same delta at the same LSN. +/// * Two same image at the same LSN. +/// * Delta/image at the same LSN where the image has already applied the delta. +/// The iterator will always put the image before the delta. pub struct MergeIterator<'a> { heap: BinaryHeap>, } @@ -245,8 +262,9 @@ mod tests { use crate::{ tenant::{ harness::{TenantHarness, TIMELINE_ID}, - storage_layer::delta_layer::test::{produce_delta_layer, sort_delta}, + storage_layer::delta_layer::test::{produce_delta_layer, sort_delta, sort_delta_value}, }, + walrecord::NeonWalRecord, DEFAULT_PG_VERSION, }; @@ -407,6 +425,127 @@ mod tests { // TODO: test layers are loaded only when needed, reducing num of active iterators in k-merge } - // TODO: image layer merge, delta+image mixed merge - // TODO: is it possible to have duplicated delta at same LSN now? we might need to test that + #[tokio::test] + async fn delta_image_mixed_merge() { + use crate::repository::Value; + use bytes::Bytes; + + let harness = TenantHarness::create("merge_iterator_delta_image_mixed_merge").unwrap(); + let (tenant, ctx) = harness.load().await; + + let tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) + .await + .unwrap(); + + fn get_key(id: u32) -> Key { + let mut key = Key::from_hex("000000000033333333444444445500000000").unwrap(); + key.field6 = id; + key + } + // In this test case, we want to test if the iterator still works correctly with multiple copies + // of a delta+image at the same LSN, for example, the following sequence a@10=+a, a@10=+a, a@10=ab, a@10=ab. + // Duplicated deltas/images are possible for old tenants before the full L0 compaction file name fix. + // An incomplete compaction could produce multiple exactly-the-same delta layers. Force image generation + // could produce overlapping images. Apart from duplicated deltas/images, in the current storage implementation + // one key-lsn could have a delta in the delta layer and one image in the image layer. The iterator should + // correctly process these situations and return everything as-is, and the upper layer of the system + // will handle duplicated LSNs. + let test_deltas1 = vec![ + ( + get_key(0), + Lsn(0x10), + Value::WalRecord(NeonWalRecord::wal_init()), + ), + ( + get_key(0), + Lsn(0x18), + Value::WalRecord(NeonWalRecord::wal_append("a")), + ), + ( + get_key(5), + Lsn(0x10), + Value::WalRecord(NeonWalRecord::wal_init()), + ), + ( + get_key(5), + Lsn(0x18), + Value::WalRecord(NeonWalRecord::wal_append("b")), + ), + ]; + let resident_layer_1 = produce_delta_layer(&tenant, &tline, test_deltas1.clone(), &ctx) + .await + .unwrap(); + let mut test_deltas2 = test_deltas1.clone(); + test_deltas2.push(( + get_key(10), + Lsn(0x20), + Value::Image(Bytes::copy_from_slice(b"test")), + )); + let resident_layer_2 = produce_delta_layer(&tenant, &tline, test_deltas2.clone(), &ctx) + .await + .unwrap(); + let test_deltas3 = vec![ + ( + get_key(0), + Lsn(0x10), + Value::Image(Bytes::copy_from_slice(b"")), + ), + ( + get_key(5), + Lsn(0x18), + Value::Image(Bytes::copy_from_slice(b"b")), + ), + ( + get_key(15), + Lsn(0x20), + Value::Image(Bytes::copy_from_slice(b"test")), + ), + ]; + let resident_layer_3 = produce_delta_layer(&tenant, &tline, test_deltas3.clone(), &ctx) + .await + .unwrap(); + let mut test_deltas4 = test_deltas3.clone(); + test_deltas4.push(( + get_key(20), + Lsn(0x20), + Value::Image(Bytes::copy_from_slice(b"test")), + )); + let resident_layer_4 = produce_delta_layer(&tenant, &tline, test_deltas4.clone(), &ctx) + .await + .unwrap(); + let mut expect = Vec::new(); + expect.extend(test_deltas1); + expect.extend(test_deltas2); + expect.extend(test_deltas3); + expect.extend(test_deltas4); + expect.sort_by(sort_delta_value); + + // Test with different layer order for MergeIterator::create to ensure the order + // is stable. + + let mut merge_iter = MergeIterator::create( + &[ + resident_layer_4.get_as_delta(&ctx).await.unwrap(), + resident_layer_1.get_as_delta(&ctx).await.unwrap(), + resident_layer_3.get_as_delta(&ctx).await.unwrap(), + resident_layer_2.get_as_delta(&ctx).await.unwrap(), + ], + &[], + &ctx, + ); + assert_merge_iter_equal(&mut merge_iter, &expect).await; + + let mut merge_iter = MergeIterator::create( + &[ + resident_layer_1.get_as_delta(&ctx).await.unwrap(), + resident_layer_4.get_as_delta(&ctx).await.unwrap(), + resident_layer_3.get_as_delta(&ctx).await.unwrap(), + resident_layer_2.get_as_delta(&ctx).await.unwrap(), + ], + &[], + &ctx, + ); + assert_merge_iter_equal(&mut merge_iter, &expect).await; + } } From c150ad4ee25cf47b1d1f1d1837b39c9c72e678aa Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 17 Jul 2024 18:35:27 +0100 Subject: [PATCH 173/194] tests: add test_compaction_l0_memory (#8403) This test reproduces the case of a writer creating a deep stack of L0 layers. It uses realistic layer sizes and writes several gigabytes of data, therefore runs as a performance test although it is validating memory footprint rather than performance per se. It acts a regression test for two recent fixes: - https://github.com/neondatabase/neon/pull/8401 - https://github.com/neondatabase/neon/pull/8391 In future it will demonstrate the larger improvement of using a k-merge iterator for L0 compaction (#8184) This test can be extended to enforce limits on the memory consumption of other housekeeping steps, by restarting the pageserver and then running other things to do the same "how much did RSS increase" measurement. --- test_runner/fixtures/pageserver/http.py | 3 + test_runner/performance/test_compaction.py | 96 ++++++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/test_runner/fixtures/pageserver/http.py b/test_runner/fixtures/pageserver/http.py index f1e3d1a30941..c7cea4ec0476 100644 --- a/test_runner/fixtures/pageserver/http.py +++ b/test_runner/fixtures/pageserver/http.py @@ -117,6 +117,9 @@ def delta_layers(self) -> List[HistoricLayerInfo]: def image_layers(self) -> List[HistoricLayerInfo]: return [x for x in self.historic_layers if x.kind == "Image"] + def delta_l0_layers(self) -> List[HistoricLayerInfo]: + return [x for x in self.historic_layers if x.kind == "Delta" and x.l0] + def historic_by_name(self) -> Set[str]: return set(x.layer_file_name for x in self.historic_layers) diff --git a/test_runner/performance/test_compaction.py b/test_runner/performance/test_compaction.py index 326c4f5c6fad..077b76104ce1 100644 --- a/test_runner/performance/test_compaction.py +++ b/test_runner/performance/test_compaction.py @@ -2,6 +2,7 @@ import pytest from fixtures.compare_fixtures import NeonCompare +from fixtures.log_helper import log from fixtures.neon_fixtures import wait_for_last_flush_lsn @@ -56,3 +57,98 @@ def test_compaction(neon_compare: NeonCompare): pageserver_http.timeline_compact(tenant_id, timeline_id) neon_compare.report_size() + + +def test_compaction_l0_memory(neon_compare: NeonCompare): + """ + Generate a large stack of L0s pending compaction into L1s, and + measure the pageserver's peak RSS while doing so + """ + + env = neon_compare.env + pageserver_http = env.pageserver.http_client() + + tenant_id, timeline_id = env.neon_cli.create_tenant( + conf={ + # Initially disable compaction so that we will build up a stack of L0s + "compaction_period": "0s", + "gc_period": "0s", + } + ) + neon_compare.tenant = tenant_id + neon_compare.timeline = timeline_id + + endpoint = env.endpoints.create_start( + "main", tenant_id=tenant_id, config_lines=["shared_buffers=512MB"] + ) + + # Read tenant effective config and assert on checkpoint_distance and compaction_threshold, + # as we do want to test with defaults (to be same as the field), but this test's workload size makes assumptions about them. + # + # If these assertions fail, it probably means we changed the default. + tenant_conf = pageserver_http.tenant_config(tenant_id) + assert tenant_conf.effective_config["checkpoint_distance"] == 256 * 1024 * 1024 + assert tenant_conf.effective_config["compaction_threshold"] == 10 + + # Aim to write about 20 L0s, so that we will hit the limit on how many + # to compact at once + with closing(endpoint.connect()) as conn: + with conn.cursor() as cur: + for i in range(200): + cur.execute(f"create table tbl{i} (i int, j int);") + cur.execute(f"insert into tbl{i} values (generate_series(1, 1000), 0);") + for j in range(100): + cur.execute(f"update tbl{i} set j = {j};") + + wait_for_last_flush_lsn(env, endpoint, tenant_id, timeline_id) + endpoint.stop() + + # Check we have generated the L0 stack we expected + layers = pageserver_http.layer_map_info(tenant_id, timeline_id) + initial_l0s = len(layers.delta_l0_layers()) + initial_l0s_size = sum(x.layer_file_size for x in layers.delta_l0_layers()) + log.info(f"l0s before compaction {initial_l0s} ({initial_l0s_size})") + + def rss_hwm(): + v = pageserver_http.get_metric_value("libmetrics_maxrss_kb") + assert v is not None + assert v > 0 + return v * 1024 + + before = rss_hwm() + pageserver_http.timeline_compact(tenant_id, timeline_id) + after = rss_hwm() + + log.info(f"RSS across compaction: {before} -> {after} (grew {after - before})") + + layers = pageserver_http.layer_map_info(tenant_id, timeline_id) + final_l0s_size = sum(x.layer_file_size for x in layers.delta_l0_layers()) + log.info(f"l0s after compaction {len(layers.delta_l0_layers())} ({final_l0s_size})") + + assert after > before # If we didn't use some memory the test is probably buggy + compaction_mapped_rss = after - before + + # During L0 compaction, we require as much memory as the physical size of what we compacted, and then some, + # because the key->value mapping in L0s compaction is exhaustive, non-streaming, and does not de-duplicate + # repeated references to the same key. + # + # To be fixed in https://github.com/neondatabase/neon/issues/8184, after which + # this memory estimate can be revised far downwards to something that doesn't scale + # linearly with the layer sizes. + MEMORY_ESTIMATE = (initial_l0s_size - final_l0s_size) * 1.25 + + # If we find that compaction is using more memory, this may indicate a regression + assert compaction_mapped_rss < MEMORY_ESTIMATE + + # If we find that compaction is using <0.5 the expected memory then: + # - maybe we made a big efficiency improvement, in which case update the test + # - maybe something is functionally wrong with the test and it's not driving the system as expected + assert compaction_mapped_rss > MEMORY_ESTIMATE / 2 + + # We should have compacted some but not all of the l0s, based on the limit on how much + # l0 to compact in one go + assert len(layers.delta_l0_layers()) > 0 + assert len(layers.delta_l0_layers()) < initial_l0s + + # The pageserver should have logged when it hit the compaction size limit + env.pageserver.assert_log_contains(".*hit max delta layer size limit.*") From ae1af558b4ba48c1b48415967e00839584688c50 Mon Sep 17 00:00:00 2001 From: Yuchen Liang <70461588+yliang412@users.noreply.github.com> Date: Wed, 17 Jul 2024 15:19:40 -0400 Subject: [PATCH 174/194] docs: update storage controller db name in doc (#8411) The db name was renamed to storage_controller from attachment_service. Doc was stale. --- docs/storage_controller.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/storage_controller.md b/docs/storage_controller.md index daf4d0c8b74c..6d2ef929a43d 100644 --- a/docs/storage_controller.md +++ b/docs/storage_controller.md @@ -44,7 +44,7 @@ If you need to modify the database schema, here’s how to create a migration: - Use `diesel migration generate ` to create a new migration - Populate the SQL files in the `migrations/` subdirectory - Use `DATABASE_URL=... diesel migration run` to apply the migration you just wrote: this will update the `[schema.rs](http://schema.rs)` file automatically. - - This requires a running database: the easiest way to do that is to just run `cargo neon init ; cargo neon start`, which will leave a database available at `postgresql://localhost:1235/attachment_service` + - This requires a running database: the easiest way to do that is to just run `cargo neon init ; cargo neon start`, which will leave a database available at `postgresql://localhost:1235/storage_controller` - Commit the migration files and the changes to schema.rs - If you need to iterate, you can rewind migrations with `diesel migration revert -a` and then `diesel migration run` again. - The migrations are build into the storage controller binary, and automatically run at startup after it is deployed, so once you’ve committed a migration no further steps are needed. From ef3ebfaf67592d30462f585b6828ca86175be381 Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 17 Jul 2024 21:55:20 +0100 Subject: [PATCH 175/194] pageserver: layer count & size metrics (#8410) ## Problem We lack insight into: - How much of a tenant's physical size is image vs. delta layers - Average sizes of image vs. delta layers - Total layer counts per timeline, indicating size of index_part object As well as general observability love, this is motivated by https://github.com/neondatabase/neon/issues/6738, where we need to define some sensible thresholds for storage amplification, and using total physical size may not work well (if someone does a lot of DROPs then it's legitimate for the physical-synthetic ratio to be huge), but the ratio between image layer size and delta layer size may be a better indicator of whether we're generating unreasonable quantities of image layers. ## Summary of changes - Add pageserver_layer_bytes and pageserver_layer_count metrics, labelled by timeline and `kind` (delta or image) - Add & subtract these with LayerInner's lifetime. I'm intentionally avoiding using a generic metric RAII guard object, to avoid bloating LayerInner: it already has all the information it needs to update metric on new+drop. --- pageserver/src/metrics.rs | 94 ++++++++++++++++++++ pageserver/src/tenant/storage_layer/layer.rs | 21 +++++ test_runner/fixtures/metrics.py | 2 + 3 files changed, 117 insertions(+) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index abad4b44b802..753f5524c55d 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -473,6 +473,31 @@ static PITR_HISTORY_SIZE: Lazy = Lazy::new(|| { .expect("failed to define a metric") }); +#[derive(strum_macros::EnumString, strum_macros::Display, strum_macros::IntoStaticStr)] +#[strum(serialize_all = "kebab_case")] +pub(crate) enum MetricLayerKind { + Delta, + Image, +} + +static TIMELINE_LAYER_SIZE: Lazy = Lazy::new(|| { + register_uint_gauge_vec!( + "pageserver_layer_bytes", + "Sum of layer physical sizes in bytes", + &["tenant_id", "shard_id", "timeline_id", "kind"] + ) + .expect("failed to define a metric") +}); + +static TIMELINE_LAYER_COUNT: Lazy = Lazy::new(|| { + register_uint_gauge_vec!( + "pageserver_layer_count", + "Number of layers that exist", + &["tenant_id", "shard_id", "timeline_id", "kind"] + ) + .expect("failed to define a metric") +}); + static TIMELINE_ARCHIVE_SIZE: Lazy = Lazy::new(|| { register_uint_gauge_vec!( "pageserver_archive_size", @@ -2141,6 +2166,10 @@ pub(crate) struct TimelineMetrics { pub last_record_gauge: IntGauge, pub pitr_history_size: UIntGauge, pub archival_size: UIntGauge, + pub(crate) layer_size_image: UIntGauge, + pub(crate) layer_count_image: UIntGauge, + pub(crate) layer_size_delta: UIntGauge, + pub(crate) layer_count_delta: UIntGauge, pub standby_horizon_gauge: IntGauge, pub resident_physical_size_gauge: UIntGauge, /// copy of LayeredTimeline.current_logical_size @@ -2223,6 +2252,42 @@ impl TimelineMetrics { .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id]) .unwrap(); + let layer_size_image = TIMELINE_LAYER_SIZE + .get_metric_with_label_values(&[ + &tenant_id, + &shard_id, + &timeline_id, + MetricLayerKind::Image.into(), + ]) + .unwrap(); + + let layer_count_image = TIMELINE_LAYER_COUNT + .get_metric_with_label_values(&[ + &tenant_id, + &shard_id, + &timeline_id, + MetricLayerKind::Image.into(), + ]) + .unwrap(); + + let layer_size_delta = TIMELINE_LAYER_SIZE + .get_metric_with_label_values(&[ + &tenant_id, + &shard_id, + &timeline_id, + MetricLayerKind::Delta.into(), + ]) + .unwrap(); + + let layer_count_delta = TIMELINE_LAYER_COUNT + .get_metric_with_label_values(&[ + &tenant_id, + &shard_id, + &timeline_id, + MetricLayerKind::Delta.into(), + ]) + .unwrap(); + let standby_horizon_gauge = STANDBY_HORIZON .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id]) .unwrap(); @@ -2277,6 +2342,10 @@ impl TimelineMetrics { last_record_gauge, pitr_history_size, archival_size, + layer_size_image, + layer_count_image, + layer_size_delta, + layer_count_delta, standby_horizon_gauge, resident_physical_size_gauge, current_logical_size_gauge, @@ -2338,6 +2407,31 @@ impl TimelineMetrics { let _ = TIMELINE_ARCHIVE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]); let _ = PITR_HISTORY_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]); + let _ = TIMELINE_LAYER_SIZE.remove_label_values(&[ + tenant_id, + shard_id, + timeline_id, + MetricLayerKind::Image.into(), + ]); + let _ = TIMELINE_LAYER_COUNT.remove_label_values(&[ + tenant_id, + shard_id, + timeline_id, + MetricLayerKind::Image.into(), + ]); + let _ = TIMELINE_LAYER_SIZE.remove_label_values(&[ + tenant_id, + shard_id, + timeline_id, + MetricLayerKind::Delta.into(), + ]); + let _ = TIMELINE_LAYER_COUNT.remove_label_values(&[ + tenant_id, + shard_id, + timeline_id, + MetricLayerKind::Delta.into(), + ]); + let _ = EVICTIONS.remove_label_values(&[tenant_id, shard_id, timeline_id]); let _ = AUX_FILE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]); let _ = VALID_LSN_LEASE_COUNT.remove_label_values(&[tenant_id, shard_id, timeline_id]); diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index 4500bc94dd66..dbf6c60aaee0 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -693,6 +693,18 @@ impl Drop for LayerInner { // and we could be delaying shutdown for nothing. } + if let Some(timeline) = self.timeline.upgrade() { + // Only need to decrement metrics if the timeline still exists: otherwise + // it will have already de-registered these metrics via TimelineMetrics::shutdown + if self.desc.is_delta() { + timeline.metrics.layer_count_delta.dec(); + timeline.metrics.layer_size_delta.sub(self.desc.file_size); + } else { + timeline.metrics.layer_count_image.dec(); + timeline.metrics.layer_size_image.sub(self.desc.file_size); + } + } + if !*self.wanted_deleted.get_mut() { return; } @@ -791,6 +803,15 @@ impl LayerInner { (heavier_once_cell::OnceCell::default(), 0, Status::Evicted) }; + // This object acts as a RAII guard on these metrics: increment on construction + if desc.is_delta() { + timeline.metrics.layer_count_delta.inc(); + timeline.metrics.layer_size_delta.add(desc.file_size); + } else { + timeline.metrics.layer_count_image.inc(); + timeline.metrics.layer_size_image.add(desc.file_size); + } + LayerInner { conf, debug_str: { diff --git a/test_runner/fixtures/metrics.py b/test_runner/fixtures/metrics.py index c019cbbc7790..4836d42db5be 100644 --- a/test_runner/fixtures/metrics.py +++ b/test_runner/fixtures/metrics.py @@ -146,6 +146,8 @@ def histogram(prefix_without_trailing_underscore: str) -> List[str]: "pageserver_smgr_query_seconds_sum", "pageserver_archive_size", "pageserver_pitr_history_size", + "pageserver_layer_bytes", + "pageserver_layer_count", "pageserver_storage_operations_seconds_count_total", "pageserver_storage_operations_seconds_sum_total", "pageserver_evictions_total", From ff174a88c0544d1270a6f993810d67a6eb4cc0b2 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 18 Jul 2024 00:03:02 +0300 Subject: [PATCH 176/194] test: allow requests to any pageserver get cancelled (#8413) Fix flakyness on `test_sharded_timeline_detach_ancestor` which does not reproduce on a fast enough runner by allowing cancelled request before completing on all pageservers. It was only allowed on half of the pageservers. Failure evidence: https://neon-github-public-dev.s3.amazonaws.com/reports/pr-8352/9972357040/index.html#suites/a1c2be32556270764423c495fad75d47/7cca3e3d94fe12f2 --- .../regress/test_timeline_detach_ancestor.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/test_runner/regress/test_timeline_detach_ancestor.py b/test_runner/regress/test_timeline_detach_ancestor.py index d75ab4c0604f..38f8dfa88553 100644 --- a/test_runner/regress/test_timeline_detach_ancestor.py +++ b/test_runner/regress/test_timeline_detach_ancestor.py @@ -702,20 +702,16 @@ def test_sharded_timeline_detach_ancestor(neon_env_builder: NeonEnvBuilder): # make another of the nodes get stuck, then restart stuck = pageservers[int(shards[0]["node_id"])] - stuck.allowed_errors.append(".*: request was dropped before completing") - env.storage_controller.allowed_errors.append(".*: request was dropped before completing") + log.info(f"stuck pageserver is id={stuck.id}") stuck_http = stuck.http_client() stuck_http.configure_failpoints( ("timeline-detach-ancestor::before_starting_after_locking_pausable", "pause") ) restarted = pageservers[int(shards[1]["node_id"])] - restarted.allowed_errors.extend( - [ - ".*: request was dropped before completing", - ".*: Cancelled request finished with an error: ShuttingDown", - ] - ) + log.info(f"restarted pageserver is id={restarted.id}") + # this might be hit; see `restart_restarted` + restarted.allowed_errors.append(".*: Cancelled request finished with an error: ShuttingDown") assert restarted.id != stuck.id restarted_http = restarted.http_client() restarted_http.configure_failpoints( @@ -724,6 +720,14 @@ def test_sharded_timeline_detach_ancestor(neon_env_builder: NeonEnvBuilder): ] ) + for info in shards: + pageserver = pageservers[int(info["node_id"])] + # the first request can cause these, but does not repeatedly + pageserver.allowed_errors.append(".*: request was dropped before completing") + + # first request again + env.storage_controller.allowed_errors.append(".*: request was dropped before completing") + target = env.storage_controller.pageserver_api() with pytest.raises(ReadTimeout): From 82a2081d61c423d19b79fb8c08ca3945c3feadcd Mon Sep 17 00:00:00 2001 From: dotdister Date: Thu, 18 Jul 2024 17:33:46 +0900 Subject: [PATCH 177/194] Fix comment in Control Plane (#8406) ## Problem There are something wrong in the comment of `control_plane/src/broker.rs` and `control_plane/src/pageserver.rs` ## Summary of changes Fixed the comment about component name and their data path in `control_plane/src/broker.rs` and `control_plane/src/pageserver.rs`. --- control_plane/src/broker.rs | 4 ++-- control_plane/src/pageserver.rs | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/control_plane/src/broker.rs b/control_plane/src/broker.rs index c3cfc140da2f..c8ac5d8981a5 100644 --- a/control_plane/src/broker.rs +++ b/control_plane/src/broker.rs @@ -1,9 +1,9 @@ //! Code to manage the storage broker //! -//! In the local test environment, the data for each safekeeper is stored in +//! In the local test environment, the storage broker stores its data directly in //! //! ```text -//! .neon/safekeepers/ +//! .neon //! ``` use std::time::Duration; diff --git a/control_plane/src/pageserver.rs b/control_plane/src/pageserver.rs index 5f2373e95a68..e3d1d0e11004 100644 --- a/control_plane/src/pageserver.rs +++ b/control_plane/src/pageserver.rs @@ -1,8 +1,10 @@ //! Code to manage pageservers //! -//! In the local test environment, the pageserver stores its data directly in +//! In the local test environment, the data for each pageserver is stored in //! -//! .neon/ +//! ```text +//! .neon/pageserver_ +//! ``` //! use std::collections::HashMap; From 3d2c2ce139de5221ce4ac0c046f0c850bd174652 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Thu, 18 Jul 2024 10:56:07 +0200 Subject: [PATCH 178/194] NeonEnv.from_repo_dir: use storage_controller_db instead of `attachments.json` (#8382) When `NeonEnv.from_repo_dir` was introduced, storage controller stored its state exclusively `attachments.json`. Since then, it has moved to using Postgres, which stores its state in `storage_controller_db`. But `NeonEnv.from_repo_dir` wasn't adjusted to do this. This PR rectifies the situation. Context for this is failures in `test_pageserver_characterize_throughput_with_n_tenants` CF: https://neondb.slack.com/archives/C033RQ5SPDH/p1721035799502239?thread_ts=1720901332.293769&cid=C033RQ5SPDH Notably, `from_repo_dir` is also used by the backwards- and forwards-compatibility. Thus, the changes in this PR affect those tests as well. However, it turns out that the compatibility snapshot already contains the `storage_controller_db`. Thus, it should just work and in fact we can remove hacks like `fixup_storage_controller`. Follow-ups created as part of this work: * https://github.com/neondatabase/neon/issues/8399 * https://github.com/neondatabase/neon/issues/8400 --- Cargo.lock | 27 +++++ Cargo.toml | 1 + control_plane/Cargo.toml | 1 + control_plane/src/storage_controller.rs | 87 +++++++++++---- storage_controller/src/main.rs | 19 +--- storage_controller/src/persistence.rs | 100 ++---------------- test_runner/fixtures/neon_fixtures.py | 27 ++++- ...er_max_throughput_getpage_at_latest_lsn.py | 8 -- test_runner/regress/test_compatibility.py | 25 ----- 9 files changed, 133 insertions(+), 162 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88973647017a..d08da0babd36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1368,6 +1368,7 @@ dependencies = [ "tracing", "url", "utils", + "whoami", "workspace_hack", ] @@ -4603,6 +4604,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "regex" version = "1.10.2" @@ -6972,6 +6982,12 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.92" @@ -7124,6 +7140,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "whoami" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" +dependencies = [ + "redox_syscall 0.4.1", + "wasite", + "web-sys", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 4f42203683d1..b9b4bafb4f69 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -191,6 +191,7 @@ uuid = { version = "1.6.1", features = ["v4", "v7", "serde"] } walkdir = "2.3.2" rustls-native-certs = "0.7" x509-parser = "0.15" +whoami = "1.5.1" ## TODO replace this with tracing env_logger = "0.10" diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index e62f3b8a4780..487ac8f047ed 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -40,6 +40,7 @@ safekeeper_api.workspace = true postgres_connection.workspace = true storage_broker.workspace = true utils.workspace = true +whoami.workspace = true compute_api.workspace = true workspace_hack.workspace = true diff --git a/control_plane/src/storage_controller.rs b/control_plane/src/storage_controller.rs index 47103a2e0ac5..d7aedd711ae0 100644 --- a/control_plane/src/storage_controller.rs +++ b/control_plane/src/storage_controller.rs @@ -29,7 +29,6 @@ use utils::{ pub struct StorageController { env: LocalEnv, listen: String, - path: Utf8PathBuf, private_key: Option>, public_key: Option, postgres_port: u16, @@ -41,6 +40,8 @@ const COMMAND: &str = "storage_controller"; const STORAGE_CONTROLLER_POSTGRES_VERSION: u32 = 16; +const DB_NAME: &str = "storage_controller"; + #[derive(Serialize, Deserialize)] pub struct AttachHookRequest { pub tenant_shard_id: TenantShardId, @@ -65,10 +66,6 @@ pub struct InspectResponse { impl StorageController { pub fn from_env(env: &LocalEnv) -> Self { - let path = Utf8PathBuf::from_path_buf(env.base_data_dir.clone()) - .unwrap() - .join("attachments.json"); - // Makes no sense to construct this if pageservers aren't going to use it: assume // pageservers have control plane API set let listen_url = env.control_plane_api.clone().unwrap(); @@ -128,7 +125,6 @@ impl StorageController { Self { env: env.clone(), - path, listen, private_key, public_key, @@ -203,7 +199,6 @@ impl StorageController { /// /// Returns the database url pub async fn setup_database(&self) -> anyhow::Result { - const DB_NAME: &str = "storage_controller"; let database_url = format!("postgresql://localhost:{}/{DB_NAME}", self.postgres_port); let pg_bin_dir = self.get_pg_bin_dir().await?; @@ -232,6 +227,30 @@ impl StorageController { Ok(database_url) } + pub async fn connect_to_database( + &self, + ) -> anyhow::Result<( + tokio_postgres::Client, + tokio_postgres::Connection, + )> { + tokio_postgres::Config::new() + .host("localhost") + .port(self.postgres_port) + // The user is the ambient operating system user name. + // That is an impurity which we want to fix in => TODO https://github.com/neondatabase/neon/issues/8400 + // + // Until we get there, use the ambient operating system user name. + // Recent tokio-postgres versions default to this if the user isn't specified. + // But tokio-postgres fork doesn't have this upstream commit: + // https://github.com/sfackler/rust-postgres/commit/cb609be758f3fb5af537f04b584a2ee0cebd5e79 + // => we should rebase our fork => TODO https://github.com/neondatabase/neon/issues/8399 + .user(&whoami::username()) + .dbname(DB_NAME) + .connect(tokio_postgres::NoTls) + .await + .map_err(anyhow::Error::new) + } + pub async fn start(&self, retry_timeout: &Duration) -> anyhow::Result<()> { // Start a vanilla Postgres process used by the storage controller for persistence. let pg_data_path = Utf8PathBuf::from_path_buf(self.env.base_data_dir.clone()) @@ -256,18 +275,21 @@ impl StorageController { if !status.success() { anyhow::bail!("initdb failed with status {status}"); } - - // Write a minimal config file: - // - Specify the port, since this is chosen dynamically - // - Switch off fsync, since we're running on lightweight test environments and when e.g. scale testing - // the storage controller we don't want a slow local disk to interfere with that. - tokio::fs::write( - &pg_data_path.join("postgresql.conf"), - format!("port = {}\nfsync=off\n", self.postgres_port), - ) - .await?; }; + // Write a minimal config file: + // - Specify the port, since this is chosen dynamically + // - Switch off fsync, since we're running on lightweight test environments and when e.g. scale testing + // the storage controller we don't want a slow local disk to interfere with that. + // + // NB: it's important that we rewrite this file on each start command so we propagate changes + // from `LocalEnv`'s config file (`.neon/config`). + tokio::fs::write( + &pg_data_path.join("postgresql.conf"), + format!("port = {}\nfsync=off\n", self.postgres_port), + ) + .await?; + println!("Starting storage controller database..."); let db_start_args = [ "-w", @@ -296,11 +318,38 @@ impl StorageController { // Run migrations on every startup, in case something changed. let database_url = self.setup_database().await?; + // We support running a startup SQL script to fiddle with the database before we launch storcon. + // This is used by the test suite. + let startup_script_path = self + .env + .base_data_dir + .join("storage_controller_db.startup.sql"); + let startup_script = match tokio::fs::read_to_string(&startup_script_path).await { + Ok(script) => { + tokio::fs::remove_file(startup_script_path).await?; + script + } + Err(e) => { + if e.kind() == std::io::ErrorKind::NotFound { + // always run some startup script so that this code path doesn't bit rot + "BEGIN; COMMIT;".to_string() + } else { + anyhow::bail!("Failed to read startup script: {e}") + } + } + }; + let (mut client, conn) = self.connect_to_database().await?; + let conn = tokio::spawn(conn); + let tx = client.build_transaction(); + let tx = tx.start().await?; + tx.batch_execute(&startup_script).await?; + tx.commit().await?; + drop(client); + conn.await??; + let mut args = vec![ "-l", &self.listen, - "-p", - self.path.as_ref(), "--dev", "--database-url", &database_url, diff --git a/storage_controller/src/main.rs b/storage_controller/src/main.rs index f1eb0b30fc38..4bf6b528f49e 100644 --- a/storage_controller/src/main.rs +++ b/storage_controller/src/main.rs @@ -1,5 +1,4 @@ use anyhow::{anyhow, Context}; -use camino::Utf8PathBuf; use clap::Parser; use diesel::Connection; use metrics::launch_timestamp::LaunchTimestamp; @@ -51,10 +50,6 @@ struct Cli { #[arg(long)] compute_hook_url: Option, - /// Path to the .json file to store state (will be created if it doesn't exist) - #[arg(short, long)] - path: Option, - /// URL to connect to postgres, like postgresql://localhost:1234/storage_controller #[arg(long)] database_url: Option, @@ -206,11 +201,10 @@ async fn async_main() -> anyhow::Result<()> { let args = Cli::parse(); tracing::info!( - "version: {}, launch_timestamp: {}, build_tag {}, state at {}, listening on {}", + "version: {}, launch_timestamp: {}, build_tag {}, listening on {}", GIT_VERSION, launch_ts.to_string(), BUILD_TAG, - args.path.as_ref().unwrap_or(&Utf8PathBuf::from("")), args.listen ); @@ -277,8 +271,7 @@ async fn async_main() -> anyhow::Result<()> { .await .context("Running database migrations")?; - let json_path = args.path; - let persistence = Arc::new(Persistence::new(secrets.database_url, json_path.clone())); + let persistence = Arc::new(Persistence::new(secrets.database_url)); let service = Service::spawn(config, persistence.clone()).await?; @@ -316,14 +309,6 @@ async fn async_main() -> anyhow::Result<()> { } tracing::info!("Terminating on signal"); - if json_path.is_some() { - // Write out a JSON dump on shutdown: this is used in compat tests to avoid passing - // full postgres dumps around. - if let Err(e) = persistence.write_tenants_json().await { - tracing::error!("Failed to write JSON on shutdown: {e}") - } - } - // Stop HTTP server first, so that we don't have to service requests // while shutting down Service server_shutdown.cancel(); diff --git a/storage_controller/src/persistence.rs b/storage_controller/src/persistence.rs index 9f7b2f775e97..d8f31e86e589 100644 --- a/storage_controller/src/persistence.rs +++ b/storage_controller/src/persistence.rs @@ -5,8 +5,6 @@ use std::time::Duration; use std::time::Instant; use self::split_state::SplitState; -use camino::Utf8Path; -use camino::Utf8PathBuf; use diesel::pg::PgConnection; use diesel::prelude::*; use diesel::Connection; @@ -55,11 +53,6 @@ use crate::node::Node; /// we can UPDATE a node's scheduling mode reasonably quickly to mark a bad node offline. pub struct Persistence { connection_pool: diesel::r2d2::Pool>, - - // In test environments, we support loading+saving a JSON file. This is temporary, for the benefit of - // test_compatibility.py, so that we don't have to commit to making the database contents fully backward/forward - // compatible just yet. - json_path: Option, } /// Legacy format, for use in JSON compat objects in test environment @@ -124,7 +117,7 @@ impl Persistence { const IDLE_CONNECTION_TIMEOUT: Duration = Duration::from_secs(10); const MAX_CONNECTION_LIFETIME: Duration = Duration::from_secs(60); - pub fn new(database_url: String, json_path: Option) -> Self { + pub fn new(database_url: String) -> Self { let manager = diesel::r2d2::ConnectionManager::::new(database_url); // We will use a connection pool: this is primarily to _limit_ our connection count, rather than to optimize time @@ -139,10 +132,7 @@ impl Persistence { .build(manager) .expect("Could not build connection pool"); - Self { - connection_pool, - json_path, - } + Self { connection_pool } } /// A helper for use during startup, where we would like to tolerate concurrent restarts of the @@ -302,85 +292,13 @@ impl Persistence { /// At startup, load the high level state for shards, such as their config + policy. This will /// be enriched at runtime with state discovered on pageservers. pub(crate) async fn list_tenant_shards(&self) -> DatabaseResult> { - let loaded = self - .with_measured_conn( - DatabaseOperation::ListTenantShards, - move |conn| -> DatabaseResult<_> { - Ok(crate::schema::tenant_shards::table.load::(conn)?) - }, - ) - .await?; - - if loaded.is_empty() { - if let Some(path) = &self.json_path { - if tokio::fs::try_exists(path) - .await - .map_err(|e| DatabaseError::Logical(format!("Error stat'ing JSON file: {e}")))? - { - tracing::info!("Importing from legacy JSON format at {path}"); - return self.list_tenant_shards_json(path).await; - } - } - } - Ok(loaded) - } - - /// Shim for automated compatibility tests: load tenants from a JSON file instead of database - pub(crate) async fn list_tenant_shards_json( - &self, - path: &Utf8Path, - ) -> DatabaseResult> { - let bytes = tokio::fs::read(path) - .await - .map_err(|e| DatabaseError::Logical(format!("Failed to load JSON: {e}")))?; - - let mut decoded = serde_json::from_slice::(&bytes) - .map_err(|e| DatabaseError::Logical(format!("Deserialization error: {e}")))?; - for shard in decoded.tenants.values_mut() { - if shard.placement_policy == "\"Single\"" { - // Backward compat for test data after PR https://github.com/neondatabase/neon/pull/7165 - shard.placement_policy = "{\"Attached\":0}".to_string(); - } - - if shard.scheduling_policy.is_empty() { - shard.scheduling_policy = - serde_json::to_string(&ShardSchedulingPolicy::default()).unwrap(); - } - } - - let tenants: Vec = decoded.tenants.into_values().collect(); - - // Synchronize database with what is in the JSON file - self.insert_tenant_shards(tenants.clone()).await?; - - Ok(tenants) - } - - /// For use in testing environments, where we dump out JSON on shutdown. - pub async fn write_tenants_json(&self) -> anyhow::Result<()> { - let Some(path) = &self.json_path else { - anyhow::bail!("Cannot write JSON if path isn't set (test environment bug)"); - }; - tracing::info!("Writing state to {path}..."); - let tenants = self.list_tenant_shards().await?; - let mut tenants_map = HashMap::new(); - for tsp in tenants { - let tenant_shard_id = TenantShardId { - tenant_id: TenantId::from_str(tsp.tenant_id.as_str())?, - shard_number: ShardNumber(tsp.shard_number as u8), - shard_count: ShardCount::new(tsp.shard_count as u8), - }; - - tenants_map.insert(tenant_shard_id, tsp); - } - let json = serde_json::to_string(&JsonPersistence { - tenants: tenants_map, - })?; - - tokio::fs::write(path, &json).await?; - tracing::info!("Wrote {} bytes to {path}...", json.len()); - - Ok(()) + self.with_measured_conn( + DatabaseOperation::ListTenantShards, + move |conn| -> DatabaseResult<_> { + Ok(crate::schema::tenant_shards::table.load::(conn)?) + }, + ) + .await } /// Tenants must be persisted before we schedule them for the first time. This enables us diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 2765ff916e63..fcfd4ea676b9 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -31,6 +31,7 @@ import httpx import jwt import psycopg2 +import psycopg2.sql import pytest import requests import toml @@ -727,8 +728,30 @@ def from_repo_dir( self.repo_dir / "local_fs_remote_storage", ) - if (attachments_json := Path(repo_dir / "attachments.json")).exists(): - shutil.copyfile(attachments_json, self.repo_dir / attachments_json.name) + # restore storage controller (the db is small, don't bother with overlayfs) + storcon_db_from_dir = repo_dir / "storage_controller_db" + storcon_db_to_dir = self.repo_dir / "storage_controller_db" + log.info(f"Copying storage_controller_db from {storcon_db_from_dir} to {storcon_db_to_dir}") + assert storcon_db_from_dir.is_dir() + assert not storcon_db_to_dir.exists() + + def ignore_postgres_log(path: str, _names): + if Path(path) == storcon_db_from_dir: + return {"postgres.log"} + return set() + + shutil.copytree(storcon_db_from_dir, storcon_db_to_dir, ignore=ignore_postgres_log) + assert not (storcon_db_to_dir / "postgres.log").exists() + # NB: neon_local rewrites postgresql.conf on each start based on neon_local config. No need to patch it. + # However, in this new NeonEnv, the pageservers listen on different ports, and the storage controller + # will currently reject re-attach requests from them because the NodeMetadata isn't identical. + # So, from_repo_dir patches up the the storcon database. + patch_script_path = self.repo_dir / "storage_controller_db.startup.sql" + assert not patch_script_path.exists() + patch_script = "" + for ps in self.env.pageservers: + patch_script += f"UPDATE nodes SET listen_http_port={ps.service_port.http}, listen_pg_port={ps.service_port.pg} WHERE node_id = '{ps.id}';" + patch_script_path.write_text(patch_script) # Update the config with info about tenants and timelines with (self.repo_dir / "config").open("r") as f: diff --git a/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py b/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py index 60861cf939b8..949813c984f9 100644 --- a/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py +++ b/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py @@ -255,11 +255,3 @@ def run_pagebench_benchmark( unit="ms", report=MetricReport.LOWER_IS_BETTER, ) - - env.storage_controller.allowed_errors.append( - # The test setup swaps NeonEnv instances, hence different - # pg instances are used for the storage controller db. This means - # the storage controller doesn't know about the nodes mentioned - # in attachments.json at start-up. - ".* Scheduler missing node 1", - ) diff --git a/test_runner/regress/test_compatibility.py b/test_runner/regress/test_compatibility.py index 1e5e320e0eff..65649e0c0a84 100644 --- a/test_runner/regress/test_compatibility.py +++ b/test_runner/regress/test_compatibility.py @@ -93,29 +93,6 @@ ) -def fixup_storage_controller(env: NeonEnv): - """ - After importing a repo_dir, we need to massage the storage controller's state a bit: it will have - initially started up with no nodes, but some tenants, and thereby those tenants won't be scheduled - anywhere. - - After NeonEnv.start() is done (i.e. nodes are started + registered), call this function to get - the storage controller into a good state. - - This function should go away once compat tests carry the controller database in their snapshots, so - that the controller properly remembers nodes between creating + restoring the snapshot. - """ - env.storage_controller.allowed_errors.extend( - [ - ".*Tenant shard .+ references non-existent node.*", - ".*Failed to schedule tenant .+ at startup.*", - ] - ) - env.storage_controller.stop() - env.storage_controller.start() - env.storage_controller.reconcile_until_idle() - - @pytest.mark.xdist_group("compatibility") @pytest.mark.order(before="test_forward_compatibility") def test_create_snapshot( @@ -198,7 +175,6 @@ def test_backward_compatibility( neon_env_builder.num_safekeepers = 3 env = neon_env_builder.from_repo_dir(compatibility_snapshot_dir / "repo") neon_env_builder.start() - fixup_storage_controller(env) check_neon_works( env, @@ -287,7 +263,6 @@ def test_forward_compatibility( assert not env.pageserver.log_contains("git-env:" + prev_pageserver_version) neon_env_builder.start() - fixup_storage_controller(env) # ensure the specified pageserver is running assert env.pageserver.log_contains("git-env:" + prev_pageserver_version) From de9bf2af6c37f2a48b7897ed5dd2b2c9d15aa419 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 18 Jul 2024 10:14:56 +0100 Subject: [PATCH 179/194] tests: fix metrics check in test_s3_eviction (#8419) ## Problem This test would occasionally fail its metric check. This could happen in the rare case that the nodes had all been restarted before their most recent eviction. The metric check was added in https://github.com/neondatabase/neon/pull/8348 ## Summary of changes - Check metrics before each restart, accumulate into a bool that we assert on at the end of the test --- test_runner/regress/test_wal_acceptor.py | 43 +++++++++++++----------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/test_runner/regress/test_wal_acceptor.py b/test_runner/regress/test_wal_acceptor.py index 2e906e616051..f02f19c588dd 100644 --- a/test_runner/regress/test_wal_acceptor.py +++ b/test_runner/regress/test_wal_acceptor.py @@ -2242,6 +2242,8 @@ def test_s3_eviction( check_values = [0] * n_timelines + event_metrics_seen = False + n_iters = 20 for _ in range(n_iters): if log.isEnabledFor(logging.DEBUG): @@ -2266,6 +2268,27 @@ def test_s3_eviction( # update remote_consistent_lsn on pageserver ps_client.timeline_checkpoint(env.initial_tenant, timelines[i], wait_until_uploaded=True) + # Do metrics check before restarts, since these will reset to zero across a restart + event_metrics_seen |= any( + sk.http_client().get_metric_value( + "safekeeper_eviction_events_started_total", {"kind": "evict"} + ) + or 0 > 0 + and sk.http_client().get_metric_value( + "safekeeper_eviction_events_completed_total", {"kind": "evict"} + ) + or 0 > 0 + and sk.http_client().get_metric_value( + "safekeeper_eviction_events_started_total", {"kind": "restore"} + ) + or 0 > 0 + and sk.http_client().get_metric_value( + "safekeeper_eviction_events_completed_total", {"kind": "restore"} + ) + or 0 > 0 + for sk in env.safekeepers + ) + # restarting random safekeepers for sk in env.safekeepers: if random.random() < restart_chance: @@ -2280,22 +2303,4 @@ def test_s3_eviction( for sk in env.safekeepers ) - assert any( - sk.http_client().get_metric_value( - "safekeeper_eviction_events_started_total", {"kind": "evict"} - ) - or 0 > 0 - and sk.http_client().get_metric_value( - "safekeeper_eviction_events_completed_total", {"kind": "evict"} - ) - or 0 > 0 - and sk.http_client().get_metric_value( - "safekeeper_eviction_events_started_total", {"kind": "restore"} - ) - or 0 > 0 - and sk.http_client().get_metric_value( - "safekeeper_eviction_events_completed_total", {"kind": "restore"} - ) - or 0 > 0 - for sk in env.safekeepers - ) + assert event_metrics_seen From 27da0e9cf50247138c69d1b29515e66a7622b456 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 18 Jul 2024 10:23:17 +0100 Subject: [PATCH 180/194] tests: increase test_pg_regress and test_isolation timeouts (#8418) ## Problem These tests time out ~1 in 50 runs when in debug mode. There is no indication of a real issue: they're just wrappers that have large numbers of individual tests contained within on pytest case. ## Summary of changes - Bump pg_regress timeout from 600 to 900s - Bump test_isolation timeout from 300s (default) to 600s In future it would be nice to break out these tests to run individual cases (or batches thereof) as separate tests, rather than this monolith. --- test_runner/regress/test_pg_regress.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_runner/regress/test_pg_regress.py b/test_runner/regress/test_pg_regress.py index 54b493ec705d..d5b5ac3f7570 100644 --- a/test_runner/regress/test_pg_regress.py +++ b/test_runner/regress/test_pg_regress.py @@ -117,7 +117,7 @@ def post_checks(env: NeonEnv, test_output_dir: Path, db_name: str, endpoint: End # Run the main PostgreSQL regression tests, in src/test/regress. # -@pytest.mark.timeout(600) +@pytest.mark.timeout(900) # Contains many sub-tests, is slow in debug builds @pytest.mark.parametrize("shard_count", [None, 4]) def test_pg_regress( neon_env_builder: NeonEnvBuilder, @@ -186,6 +186,7 @@ def test_pg_regress( # Run the PostgreSQL "isolation" tests, in src/test/isolation. # +@pytest.mark.timeout(600) # Contains many sub-tests, is slow in debug builds @pytest.mark.parametrize("shard_count", [None, 4]) def test_isolation( neon_env_builder: NeonEnvBuilder, From 9868bb3346d27841e3cee67165745992270f65c9 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 18 Jul 2024 12:59:14 +0100 Subject: [PATCH 181/194] tests: turn on safekeeper eviction by default (#8352) ## Problem Ahead of enabling eviction in the field, where it will become the normal/default mode, let's enable it by default throughout our tests in case any issues become visible there. ## Summary of changes - Make default `extra_opts` for safekeepers enable offload & deletion - Set low timeouts in `extra_opts` so that tests running for tens of seconds have a chance to hit some of these background operations. --- test_runner/fixtures/neon_fixtures.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index fcfd4ea676b9..567ca532f97a 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -4077,6 +4077,22 @@ def __init__( self.id = id self.running = running self.logfile = Path(self.data_dir) / f"safekeeper-{id}.log" + + if extra_opts is None: + # Testing defaults: enable everything, and set short timeouts so that background + # work will happen during short tests. + # **Note**: Any test that explicitly sets extra_opts will not get these defaults. + extra_opts = [ + "--enable-offload", + "--delete-offloaded-wal", + "--partial-backup-timeout", + "10s", + "--control-file-save-interval", + "1s", + "--eviction-min-resident", + "10s", + ] + self.extra_opts = extra_opts def start( From 9f1ba2c4bfdb366c2710e0fc9629932ecde99d51 Mon Sep 17 00:00:00 2001 From: Arthur Petukhovsky Date: Thu, 18 Jul 2024 13:46:00 +0100 Subject: [PATCH 182/194] Fix partial upload bug with invalid remote state (#8383) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We have an issue that some partial uploaded segments can be actually missing in remote storage. I found this issue when was looking at the logs in staging, and it can be triggered by failed uploads: 1. Code tries to upload `SEG_TERM_LSN_LSN_sk5.partial`, but receives error from S3 2. The failed attempt is saved to `segments` vec 3. After some time, the code tries to upload `SEG_TERM_LSN_LSN_sk5.partial` again 4. This time the upload is successful and code calls `gc()` to delete previous uploads 5. Since new object and old object share the same name, uploaded data gets deleted from remote storage This commit fixes the issue by patching `gc()` not to delete objects with the same name as currently uploaded. --------- Co-authored-by: Arpad Müller --- safekeeper/src/timeline_eviction.rs | 5 +---- safekeeper/src/wal_backup_partial.rs | 12 ++++++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/safekeeper/src/timeline_eviction.rs b/safekeeper/src/timeline_eviction.rs index 0b8d58ee8a52..7947d83eb4bf 100644 --- a/safekeeper/src/timeline_eviction.rs +++ b/safekeeper/src/timeline_eviction.rs @@ -199,10 +199,7 @@ async fn redownload_partial_segment( file.flush().await?; let final_path = local_segment_path(mgr, partial); - info!( - "downloaded {} bytes, renaming to {}", - final_path, final_path, - ); + info!("downloaded {actual_len} bytes, renaming to {final_path}"); if let Err(e) = durable_rename(&tmp_file, &final_path, !mgr.conf.no_sync).await { // Probably rename succeeded, but fsync of it failed. Remove // the file then to avoid using it. diff --git a/safekeeper/src/wal_backup_partial.rs b/safekeeper/src/wal_backup_partial.rs index 825851c97c9a..b1efa9749f19 100644 --- a/safekeeper/src/wal_backup_partial.rs +++ b/safekeeper/src/wal_backup_partial.rs @@ -289,6 +289,18 @@ impl PartialBackup { }) .collect(); + if new_segments.len() == 1 { + // we have an uploaded segment, it must not be deleted from remote storage + segments_to_delete.retain(|name| name != &new_segments[0].name); + } else { + // there should always be zero or one uploaded segment + assert!( + new_segments.is_empty(), + "too many uploaded segments: {:?}", + new_segments + ); + } + info!("deleting objects: {:?}", segments_to_delete); let mut objects_to_delete = vec![]; for seg in segments_to_delete.iter() { From f87b031876e042d46b3b3228e34ab1aab3159e28 Mon Sep 17 00:00:00 2001 From: "Alex Chi Z." Date: Thu, 18 Jul 2024 12:16:44 -0400 Subject: [PATCH 183/194] pageserver: integrate k-merge with bottom-most compaction (#8415) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the k-merge iterator in the compaction process to reduce memory footprint. part of https://github.com/neondatabase/neon/issues/8002 ## Summary of changes * refactor the bottom-most compaction code to use k-merge iterator * add Send bound on some structs as it is used across the await points --------- Signed-off-by: Alex Chi Z Co-authored-by: Arpad Müller --- pageserver/src/tenant.rs | 2 +- pageserver/src/tenant/disk_btree.rs | 4 +- pageserver/src/tenant/storage_layer.rs | 2 - .../src/tenant/storage_layer/delta_layer.rs | 21 +++--- .../src/tenant/storage_layer/image_layer.rs | 23 +++--- pageserver/src/tenant/storage_layer/layer.rs | 5 +- .../tenant/storage_layer/merge_iterator.rs | 4 ++ pageserver/src/tenant/timeline/compaction.rs | 70 ++++++++----------- pageserver/src/tenant/vectored_blob_io.rs | 2 - 9 files changed, 62 insertions(+), 71 deletions(-) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index dc6f42eaebaf..637051413f16 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -6810,7 +6810,7 @@ mod tests { vec![ // Image layer at GC horizon PersistentLayerKey { - key_range: Key::MIN..get_key(10), + key_range: Key::MIN..Key::MAX, lsn_range: Lsn(0x30)..Lsn(0x31), is_delta: false }, diff --git a/pageserver/src/tenant/disk_btree.rs b/pageserver/src/tenant/disk_btree.rs index 251d2ab4aded..1583a3826af5 100644 --- a/pageserver/src/tenant/disk_btree.rs +++ b/pageserver/src/tenant/disk_btree.rs @@ -262,7 +262,7 @@ where pub fn iter<'a>(self, start_key: &'a [u8; L], ctx: &'a RequestContext) -> DiskBtreeIterator<'a> where - R: 'a, + R: 'a + Send, { DiskBtreeIterator { stream: Box::pin(self.into_stream(start_key, ctx)), @@ -521,7 +521,7 @@ where pub struct DiskBtreeIterator<'a> { #[allow(clippy::type_complexity)] stream: std::pin::Pin< - Box, u64), DiskBtreeError>> + 'a>, + Box, u64), DiskBtreeError>> + 'a + Send>, >, } diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index 2f0c45317d9a..a389358f0d27 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -6,8 +6,6 @@ pub(crate) mod inmemory_layer; pub(crate) mod layer; mod layer_desc; mod layer_name; - -#[cfg(test)] pub mod merge_iterator; use crate::context::{AccessStatsBehavior, RequestContext}; diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 43941b6e1739..c34923320aee 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -33,11 +33,14 @@ use crate::page_cache::{self, FileId, PAGE_SZ}; use crate::repository::{Key, Value, KEY_SIZE}; use crate::tenant::blob_io::BlobWriter; use crate::tenant::block_io::{BlockBuf, BlockCursor, BlockLease, BlockReader, FileBlockReader}; -use crate::tenant::disk_btree::{DiskBtreeBuilder, DiskBtreeReader, VisitDirection}; +use crate::tenant::disk_btree::{ + DiskBtreeBuilder, DiskBtreeIterator, DiskBtreeReader, VisitDirection, +}; use crate::tenant::storage_layer::{Layer, ValueReconstructResult, ValueReconstructState}; use crate::tenant::timeline::GetVectoredError; use crate::tenant::vectored_blob_io::{ - BlobFlag, MaxVectoredReadBytes, VectoredBlobReader, VectoredRead, VectoredReadPlanner, + BlobFlag, MaxVectoredReadBytes, StreamingVectoredReadPlanner, VectoredBlobReader, VectoredRead, + VectoredReadPlanner, }; use crate::tenant::{PageReconstructError, Timeline}; use crate::virtual_file::{self, VirtualFile}; @@ -53,6 +56,7 @@ use pageserver_api::models::{ImageCompressionAlgorithm, LayerAccessKind}; use pageserver_api::shard::TenantShardId; use rand::{distributions::Alphanumeric, Rng}; use serde::{Deserialize, Serialize}; +use std::collections::VecDeque; use std::fs::File; use std::io::SeekFrom; use std::ops::Range; @@ -747,12 +751,10 @@ impl DeltaLayer { } impl DeltaLayerInner { - #[cfg(test)] pub(crate) fn key_range(&self) -> &Range { &self.layer_key_range } - #[cfg(test)] pub(crate) fn lsn_range(&self) -> &Range { &self.layer_lsn_range } @@ -1512,7 +1514,6 @@ impl DeltaLayerInner { offset } - #[cfg(test)] pub(crate) fn iter<'a>(&'a self, ctx: &'a RequestContext) -> DeltaLayerIterator<'a> { let block_reader = FileBlockReader::new(&self.file, self.file_id); let tree_reader = @@ -1523,7 +1524,7 @@ impl DeltaLayerInner { index_iter: tree_reader.iter(&[0; DELTA_KEY_SIZE], ctx), key_values_batch: std::collections::VecDeque::new(), is_end: false, - planner: crate::tenant::vectored_blob_io::StreamingVectoredReadPlanner::new( + planner: StreamingVectoredReadPlanner::new( 1024 * 8192, // The default value. Unit tests might use a different value. 1024 * 8K = 8MB buffer. 1024, // The default value. Unit tests might use a different value ), @@ -1595,17 +1596,15 @@ impl<'a> pageserver_compaction::interface::CompactionDeltaEntry<'a, Key> for Del } } -#[cfg(test)] pub struct DeltaLayerIterator<'a> { delta_layer: &'a DeltaLayerInner, ctx: &'a RequestContext, - planner: crate::tenant::vectored_blob_io::StreamingVectoredReadPlanner, - index_iter: crate::tenant::disk_btree::DiskBtreeIterator<'a>, - key_values_batch: std::collections::VecDeque<(Key, Lsn, Value)>, + planner: StreamingVectoredReadPlanner, + index_iter: DiskBtreeIterator<'a>, + key_values_batch: VecDeque<(Key, Lsn, Value)>, is_end: bool, } -#[cfg(test)] impl<'a> DeltaLayerIterator<'a> { /// Retrieve a batch of key-value pairs into the iterator buffer. async fn next_batch(&mut self) -> anyhow::Result<()> { diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index a88a1e642958..c7f41b66befc 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -29,13 +29,16 @@ use crate::page_cache::{self, FileId, PAGE_SZ}; use crate::repository::{Key, Value, KEY_SIZE}; use crate::tenant::blob_io::BlobWriter; use crate::tenant::block_io::{BlockBuf, BlockReader, FileBlockReader}; -use crate::tenant::disk_btree::{DiskBtreeBuilder, DiskBtreeReader, VisitDirection}; +use crate::tenant::disk_btree::{ + DiskBtreeBuilder, DiskBtreeIterator, DiskBtreeReader, VisitDirection, +}; use crate::tenant::storage_layer::{ LayerAccessStats, ValueReconstructResult, ValueReconstructState, }; use crate::tenant::timeline::GetVectoredError; use crate::tenant::vectored_blob_io::{ - BlobFlag, MaxVectoredReadBytes, VectoredBlobReader, VectoredRead, VectoredReadPlanner, + BlobFlag, MaxVectoredReadBytes, StreamingVectoredReadPlanner, VectoredBlobReader, VectoredRead, + VectoredReadPlanner, }; use crate::tenant::{PageReconstructError, Timeline}; use crate::virtual_file::{self, VirtualFile}; @@ -50,6 +53,7 @@ use pageserver_api::models::LayerAccessKind; use pageserver_api::shard::{ShardIdentity, TenantShardId}; use rand::{distributions::Alphanumeric, Rng}; use serde::{Deserialize, Serialize}; +use std::collections::VecDeque; use std::fs::File; use std::io::SeekFrom; use std::ops::Range; @@ -369,12 +373,10 @@ impl ImageLayer { } impl ImageLayerInner { - #[cfg(test)] pub(crate) fn key_range(&self) -> &Range { &self.key_range } - #[cfg(test)] pub(crate) fn lsn(&self) -> Lsn { self.lsn } @@ -699,7 +701,6 @@ impl ImageLayerInner { } } - #[cfg(test)] pub(crate) fn iter<'a>(&'a self, ctx: &'a RequestContext) -> ImageLayerIterator<'a> { let block_reader = FileBlockReader::new(&self.file, self.file_id); let tree_reader = @@ -708,9 +709,9 @@ impl ImageLayerInner { image_layer: self, ctx, index_iter: tree_reader.iter(&[0; KEY_SIZE], ctx), - key_values_batch: std::collections::VecDeque::new(), + key_values_batch: VecDeque::new(), is_end: false, - planner: crate::tenant::vectored_blob_io::StreamingVectoredReadPlanner::new( + planner: StreamingVectoredReadPlanner::new( 1024 * 8192, // The default value. Unit tests might use a different value. 1024 * 8K = 8MB buffer. 1024, // The default value. Unit tests might use a different value ), @@ -974,17 +975,15 @@ impl Drop for ImageLayerWriter { } } -#[cfg(test)] pub struct ImageLayerIterator<'a> { image_layer: &'a ImageLayerInner, ctx: &'a RequestContext, - planner: crate::tenant::vectored_blob_io::StreamingVectoredReadPlanner, - index_iter: crate::tenant::disk_btree::DiskBtreeIterator<'a>, - key_values_batch: std::collections::VecDeque<(Key, Lsn, Value)>, + planner: StreamingVectoredReadPlanner, + index_iter: DiskBtreeIterator<'a>, + key_values_batch: VecDeque<(Key, Lsn, Value)>, is_end: bool, } -#[cfg(test)] impl<'a> ImageLayerIterator<'a> { /// Retrieve a batch of key-value pairs into the iterator buffer. async fn next_batch(&mut self) -> anyhow::Result<()> { diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index dbf6c60aaee0..d9cbaba529d5 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -385,6 +385,7 @@ impl Layer { } /// Get all key/values in the layer. Should be replaced with an iterator-based API in the future. + #[allow(dead_code)] pub(crate) async fn load_key_values( &self, ctx: &RequestContext, @@ -1918,7 +1919,7 @@ impl ResidentLayer { self.owner.metadata() } - #[cfg(test)] + /// Cast the layer to a delta, return an error if it is an image layer. pub(crate) async fn get_as_delta( &self, ctx: &RequestContext, @@ -1930,7 +1931,7 @@ impl ResidentLayer { } } - #[cfg(test)] + /// Cast the layer to an image, return an error if it is a delta layer. pub(crate) async fn get_as_image( &self, ctx: &RequestContext, diff --git a/pageserver/src/tenant/storage_layer/merge_iterator.rs b/pageserver/src/tenant/storage_layer/merge_iterator.rs index 0edfd4bd4075..6f59b2fd7765 100644 --- a/pageserver/src/tenant/storage_layer/merge_iterator.rs +++ b/pageserver/src/tenant/storage_layer/merge_iterator.rs @@ -547,5 +547,9 @@ mod tests { &ctx, ); assert_merge_iter_equal(&mut merge_iter, &expect).await; + + is_send(merge_iter); } + + fn is_send(_: impl Send) {} } diff --git a/pageserver/src/tenant/timeline/compaction.rs b/pageserver/src/tenant/timeline/compaction.rs index f251b667c2fb..a648432b4d08 100644 --- a/pageserver/src/tenant/timeline/compaction.rs +++ b/pageserver/src/tenant/timeline/compaction.rs @@ -27,6 +27,7 @@ use utils::id::TimelineId; use crate::context::{AccessStatsBehavior, RequestContext, RequestContextBuilder}; use crate::page_cache; use crate::tenant::config::defaults::{DEFAULT_CHECKPOINT_DISTANCE, DEFAULT_COMPACTION_THRESHOLD}; +use crate::tenant::storage_layer::merge_iterator::MergeIterator; use crate::tenant::storage_layer::{AsLayerDesc, PersistentLayerDesc}; use crate::tenant::timeline::{drop_rlock, DeltaLayerWriter, ImageLayerWriter}; use crate::tenant::timeline::{Hole, ImageLayerCreationOutcome}; @@ -1039,10 +1040,12 @@ impl Timeline { ); // Step 1: (In the future) construct a k-merge iterator over all layers. For now, simply collect all keys + LSNs. // Also, collect the layer information to decide when to split the new delta layers. - let mut all_key_values = Vec::new(); + let mut downloaded_layers = Vec::new(); let mut delta_split_points = BTreeSet::new(); for layer in &layer_selection { - all_key_values.extend(layer.load_key_values(ctx).await?); + let resident_layer = layer.download_and_keep_resident().await?; + downloaded_layers.push(resident_layer); + let desc = layer.layer_desc(); if desc.is_delta() { // TODO: is it correct to only record split points for deltas intersecting with the GC horizon? (exclude those below/above the horizon) @@ -1052,44 +1055,28 @@ impl Timeline { delta_split_points.insert(key_range.end); } } - // Key small to large, LSN low to high, if the same LSN has both image and delta due to the merge of delta layers and - // image layers, make image appear before than delta. - struct ValueWrapper<'a>(&'a crate::repository::Value); - impl Ord for ValueWrapper<'_> { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - use crate::repository::Value; - use std::cmp::Ordering; - match (self.0, other.0) { - (Value::Image(_), Value::WalRecord(_)) => Ordering::Less, - (Value::WalRecord(_), Value::Image(_)) => Ordering::Greater, - _ => Ordering::Equal, - } - } - } - impl PartialOrd for ValueWrapper<'_> { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } - } - impl PartialEq for ValueWrapper<'_> { - fn eq(&self, other: &Self) -> bool { - self.cmp(other) == std::cmp::Ordering::Equal + let mut delta_layers = Vec::new(); + let mut image_layers = Vec::new(); + for resident_layer in &downloaded_layers { + if resident_layer.layer_desc().is_delta() { + let layer = resident_layer.get_as_delta(ctx).await?; + delta_layers.push(layer); + } else { + let layer = resident_layer.get_as_image(ctx).await?; + image_layers.push(layer); } } - impl Eq for ValueWrapper<'_> {} - all_key_values.sort_by(|(k1, l1, v1), (k2, l2, v2)| { - (k1, l1, ValueWrapper(v1)).cmp(&(k2, l2, ValueWrapper(v2))) - }); + let mut merge_iter = MergeIterator::create(&delta_layers, &image_layers, ctx); // Step 2: Produce images+deltas. TODO: ensure newly-produced delta does not overlap with other deltas. // Data of the same key. let mut accumulated_values = Vec::new(); - let mut last_key = all_key_values.first().unwrap().0; // TODO: assert all_key_values not empty + let mut last_key: Option = None; /// Take a list of images and deltas, produce an image at the GC horizon, and a list of deltas above the GC horizon. async fn flush_accumulated_states( tline: &Arc, key: Key, - accumulated_values: &[&(Key, Lsn, crate::repository::Value)], + accumulated_values: &[(Key, Lsn, crate::repository::Value)], horizon: Lsn, ) -> anyhow::Result<(Vec<(Key, Lsn, crate::repository::Value)>, bytes::Bytes)> { let mut base_image = None; @@ -1190,7 +1177,7 @@ impl Timeline { self.conf, self.timeline_id, self.tenant_shard_id, - &(all_key_values.first().unwrap().0..all_key_values.last().unwrap().0.next()), + &(Key::MIN..Key::MAX), // covers the full key range gc_cutoff, ctx, ) @@ -1200,20 +1187,24 @@ impl Timeline { let delta_split_points = delta_split_points.into_iter().collect_vec(); let mut current_delta_split_point = 0; let mut delta_layers = Vec::new(); - for item @ (key, _, _) in &all_key_values { - if &last_key == key { - accumulated_values.push(item); + while let Some((key, lsn, val)) = merge_iter.next().await? { + if last_key.is_none() || last_key.as_ref() == Some(&key) { + if last_key.is_none() { + last_key = Some(key); + } + accumulated_values.push((key, lsn, val)); } else { + let last_key = last_key.as_mut().unwrap(); let (deltas, image) = - flush_accumulated_states(self, last_key, &accumulated_values, gc_cutoff) + flush_accumulated_states(self, *last_key, &accumulated_values, gc_cutoff) .await?; // Put the image into the image layer. Currently we have a single big layer for the compaction. - image_layer_writer.put_image(last_key, image, ctx).await?; + image_layer_writer.put_image(*last_key, image, ctx).await?; delta_values.extend(deltas); delta_layers.extend( flush_deltas( &mut delta_values, - last_key, + *last_key, &delta_split_points, &mut current_delta_split_point, self, @@ -1223,11 +1214,12 @@ impl Timeline { .await?, ); accumulated_values.clear(); - accumulated_values.push(item); - last_key = *key; + *last_key = key; + accumulated_values.push((key, lsn, val)); } } + let last_key = last_key.expect("no keys produced during compaction"); // TODO: move this part to the loop body let (deltas, image) = flush_accumulated_states(self, last_key, &accumulated_values, gc_cutoff).await?; diff --git a/pageserver/src/tenant/vectored_blob_io.rs b/pageserver/src/tenant/vectored_blob_io.rs index 5a0986ea12ec..54a3ad789b9f 100644 --- a/pageserver/src/tenant/vectored_blob_io.rs +++ b/pageserver/src/tenant/vectored_blob_io.rs @@ -396,7 +396,6 @@ impl<'a> VectoredBlobReader<'a> { /// Read planner used in [`crate::tenant::storage_layer::image_layer::ImageLayerIterator`]. It provides a streaming API for /// getting read blobs. It returns a batch when `handle` gets called and when the current key would just exceed the read_size and /// max_cnt constraints. -#[cfg(test)] pub struct StreamingVectoredReadPlanner { read_builder: Option, // Arguments for previous blob passed into [`StreamingVectoredReadPlanner::handle`] @@ -410,7 +409,6 @@ pub struct StreamingVectoredReadPlanner { cnt: usize, } -#[cfg(test)] impl StreamingVectoredReadPlanner { pub fn new(max_read_size: u64, max_cnt: usize) -> Self { assert!(max_cnt > 0); From 1b508a6082adee5506c44b3bbd05f2e7f67c400d Mon Sep 17 00:00:00 2001 From: Peter Bendel Date: Thu, 18 Jul 2024 18:18:18 +0200 Subject: [PATCH 184/194] Temporarily use vanilla pgbench and psql (client) for running pgvector benchmark (#8422) ## Problem https://github.com/neondatabase/neon/issues/8275 is not yet fixed Periodic benchmarking fails with SIGABRT in pgvector step, see https://github.com/neondatabase/neon/actions/runs/9967453263/job/27541159738#step:7:393 ## Summary of changes Instead of using pgbench and psql from Neon artifacts, download vanilla postgres binaries into the container and use those to run the client side of the test. --- .github/workflows/benchmarking.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index d785156a29b1..833a4ce33c7f 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -457,17 +457,21 @@ jobs: runs-on: [ self-hosted, us-east-2, x64 ] container: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:pinned - options: --init + options: --init --user root steps: - uses: actions/checkout@v4 - - name: Download Neon artifact - uses: ./.github/actions/download - with: - name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact - path: /tmp/neon/ - prefix: latest + # until https://github.com/neondatabase/neon/issues/8275 is fixed we temporarily install postgresql-16 + # instead of using Neon artifacts containing pgbench + - name: Install postgresql-16 where pytest expects it + run: | + apt-get update && apt-get install -y postgresql-common + /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y + apt-get -y install postgresql-16 + mkdir -p /tmp/neon/pg_install/v16/bin + ln -s /usr/bin/pgbench /tmp/neon/pg_install/v16/bin/pgbench + ln -s /usr/bin/psql /tmp/neon/pg_install/v16/bin/psql - name: Set up Connection String id: set-up-connstr From fceace835b318868c072af3c5cdaa6ddd152bb44 Mon Sep 17 00:00:00 2001 From: Arthur Petukhovsky Date: Thu, 18 Jul 2024 17:26:27 +0100 Subject: [PATCH 185/194] Change log level for GuardDrop error (#8305) The error means that manager exited earlier than `ResidenceGuard` and it's not unexpected with current deletion implementation. This commit changes log level to reduse noise. --- safekeeper/src/timeline_guard.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/safekeeper/src/timeline_guard.rs b/safekeeper/src/timeline_guard.rs index e249c859b4bb..dbdf46412ddb 100644 --- a/safekeeper/src/timeline_guard.rs +++ b/safekeeper/src/timeline_guard.rs @@ -4,7 +4,7 @@ use std::collections::HashSet; -use tracing::{debug, warn}; +use tracing::debug; use crate::timeline_manager::ManagerCtlMessage; @@ -23,7 +23,7 @@ impl Drop for ResidenceGuard { .manager_tx .send(ManagerCtlMessage::GuardDrop(self.guard_id)); if let Err(e) = res { - warn!("failed to send GuardDrop message: {:?}", e); + debug!("failed to send GuardDrop message: {:?}", e); } } } From ed7ee73cbae80f2c5e6cffb7dff278409c9a8bc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Thu, 18 Jul 2024 20:09:57 +0200 Subject: [PATCH 186/194] Enable zstd in tests (#8368) Successor of #8288 , just enable zstd in tests. Also adds a test that creates easily compressable data. Part of #5431 --------- Co-authored-by: John Spray Co-authored-by: Joonas Koivunen --- pageserver/src/metrics.rs | 16 ++++ .../src/tenant/storage_layer/image_layer.rs | 10 ++ test_runner/fixtures/neon_fixtures.py | 1 + test_runner/regress/test_compaction.py | 93 ++++++++++++++++++- .../regress/test_disk_usage_eviction.py | 3 + 5 files changed, 122 insertions(+), 1 deletion(-) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 753f5524c55d..c03567f6efb2 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -610,6 +610,22 @@ pub(crate) static CIRCUIT_BREAKERS_UNBROKEN: Lazy = Lazy::new(|| { .expect("failed to define a metric") }); +pub(crate) static COMPRESSION_IMAGE_INPUT_BYTES: Lazy = Lazy::new(|| { + register_int_counter!( + "pageserver_compression_image_in_bytes_total", + "Size of uncompressed data written into image layers" + ) + .expect("failed to define a metric") +}); + +pub(crate) static COMPRESSION_IMAGE_OUTPUT_BYTES: Lazy = Lazy::new(|| { + register_int_counter!( + "pageserver_compression_image_out_bytes_total", + "Size of compressed image layer written" + ) + .expect("failed to define a metric") +}); + pub(crate) mod initial_logical_size { use metrics::{register_int_counter, register_int_counter_vec, IntCounter, IntCounterVec}; use once_cell::sync::Lazy; diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index c7f41b66befc..45b47bb62b0c 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -738,6 +738,9 @@ struct ImageLayerWriterInner { key_range: Range, lsn: Lsn, + // Total uncompressed bytes passed into put_image + uncompressed_bytes: u64, + blob_writer: BlobWriter, tree: DiskBtreeBuilder, } @@ -793,6 +796,7 @@ impl ImageLayerWriterInner { lsn, tree: tree_builder, blob_writer, + uncompressed_bytes: 0, }; Ok(writer) @@ -811,6 +815,7 @@ impl ImageLayerWriterInner { ) -> anyhow::Result<()> { ensure!(self.key_range.contains(&key)); let compression = self.conf.image_compression; + self.uncompressed_bytes += img.len() as u64; let (_img, res) = self .blob_writer .write_blob_maybe_compressed(img, ctx, compression) @@ -836,6 +841,11 @@ impl ImageLayerWriterInner { let index_start_blk = ((self.blob_writer.size() + PAGE_SZ as u64 - 1) / PAGE_SZ as u64) as u32; + // Calculate compression ratio + let compressed_size = self.blob_writer.size() - PAGE_SZ as u64; // Subtract PAGE_SZ for header + crate::metrics::COMPRESSION_IMAGE_INPUT_BYTES.inc_by(self.uncompressed_bytes); + crate::metrics::COMPRESSION_IMAGE_OUTPUT_BYTES.inc_by(compressed_size); + let mut file = self.blob_writer.into_inner(); // Write out the index diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 567ca532f97a..db7269ad4148 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1158,6 +1158,7 @@ def __init__(self, config: NeonEnvBuilder): "listen_http_addr": f"localhost:{pageserver_port.http}", "pg_auth_type": pg_auth_type, "http_auth_type": http_auth_type, + "image_compression": "zstd", } if self.pageserver_virtual_file_io_engine is not None: ps_cfg["virtual_file_io_engine"] = self.pageserver_virtual_file_io_engine diff --git a/test_runner/regress/test_compaction.py b/test_runner/regress/test_compaction.py index f321c09b2729..be787e064262 100644 --- a/test_runner/regress/test_compaction.py +++ b/test_runner/regress/test_compaction.py @@ -6,7 +6,10 @@ import pytest from fixtures.log_helper import log -from fixtures.neon_fixtures import NeonEnvBuilder, generate_uploads_and_deletions +from fixtures.neon_fixtures import ( + NeonEnvBuilder, + generate_uploads_and_deletions, +) from fixtures.pageserver.http import PageserverApiException from fixtures.utils import wait_until from fixtures.workload import Workload @@ -142,6 +145,10 @@ def test_sharding_compaction( "image_layer_creation_check_threshold": 0, } + # Disable compression, as we can't estimate the size of layers with compression enabled + # TODO: implement eager layer cutting during compaction + neon_env_builder.pageserver_config_override = "image_compression='disabled'" + neon_env_builder.num_pageservers = 1 if shard_count is None else shard_count env = neon_env_builder.init_start( initial_tenant_conf=TENANT_CONF, @@ -320,3 +327,87 @@ def assert_broken(): or 0 ) == 0 assert not env.pageserver.log_contains(".*Circuit breaker failure ended.*") + + +@pytest.mark.parametrize("enabled", [True, False]) +def test_image_layer_compression(neon_env_builder: NeonEnvBuilder, enabled: bool): + tenant_conf = { + # small checkpointing and compaction targets to ensure we generate many upload operations + "checkpoint_distance": f"{128 * 1024}", + "compaction_threshold": "1", + "compaction_target_size": f"{128 * 1024}", + # no PITR horizon, we specify the horizon when we request on-demand GC + "pitr_interval": "0s", + # disable background compaction and GC. We invoke it manually when we want it to happen. + "gc_period": "0s", + "compaction_period": "0s", + # create image layers as eagerly as possible + "image_creation_threshold": "1", + "image_layer_creation_check_threshold": "0", + } + + # Explicitly enable/disable compression, rather than using default + if enabled: + neon_env_builder.pageserver_config_override = "image_compression='zstd'" + else: + neon_env_builder.pageserver_config_override = "image_compression='disabled'" + + env = neon_env_builder.init_start(initial_tenant_conf=tenant_conf) + + tenant_id = env.initial_tenant + timeline_id = env.initial_timeline + + pageserver = env.pageserver + ps_http = env.pageserver.http_client() + with env.endpoints.create_start( + "main", tenant_id=tenant_id, pageserver_id=pageserver.id + ) as endpoint: + endpoint.safe_psql("CREATE TABLE foo (id INTEGER PRIMARY KEY, val text)") + # Generate around 800k worth of easily compressible data to store + for v in range(100): + endpoint.safe_psql( + f"INSERT INTO foo (id, val) VALUES ({v}, repeat('abcde{v:0>3}', 500))" + ) + # run compaction to create image layers + ps_http.timeline_checkpoint(tenant_id, timeline_id, wait_until_uploaded=True) + + layer_map = ps_http.layer_map_info(tenant_id, timeline_id) + image_layer_count = 0 + delta_layer_count = 0 + for layer in layer_map.historic_layers: + if layer.kind == "Image": + image_layer_count += 1 + elif layer.kind == "Delta": + delta_layer_count += 1 + assert image_layer_count > 0 + assert delta_layer_count > 0 + + log.info(f"images: {image_layer_count}, deltas: {delta_layer_count}") + + bytes_in = pageserver.http_client().get_metric_value( + "pageserver_compression_image_in_bytes_total" + ) + bytes_out = pageserver.http_client().get_metric_value( + "pageserver_compression_image_out_bytes_total" + ) + assert bytes_in is not None + assert bytes_out is not None + log.info(f"Compression ratio: {bytes_out/bytes_in} ({bytes_out} in, {bytes_out} out)") + + if enabled: + # We are writing high compressible repetitive plain text, expect excellent compression + EXPECT_RATIO = 0.2 + assert bytes_out / bytes_in < EXPECT_RATIO + else: + # Nothing should be compressed if we disabled it. + assert bytes_out >= bytes_in + + # Destroy the endpoint and create a new one to resetthe caches + with env.endpoints.create_start( + "main", tenant_id=tenant_id, pageserver_id=pageserver.id + ) as endpoint: + for v in range(100): + res = endpoint.safe_psql( + f"SELECT count(*) FROM foo WHERE id={v} and val=repeat('abcde{v:0>3}', 500)" + ) + assert res[0][0] == 1 diff --git a/test_runner/regress/test_disk_usage_eviction.py b/test_runner/regress/test_disk_usage_eviction.py index fb8b7b22fa71..3c834f430b08 100644 --- a/test_runner/regress/test_disk_usage_eviction.py +++ b/test_runner/regress/test_disk_usage_eviction.py @@ -230,6 +230,9 @@ def _eviction_env( neon_env_builder.num_pageservers = num_pageservers neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) + # Disable compression support for EvictionEnv to get larger layer sizes + neon_env_builder.pageserver_config_override = "image_compression='disabled'" + # initial tenant will not be present on this pageserver env = neon_env_builder.init_configs() env.start() From b98b301d56d2abe67f5637fed3e62e368345f4d9 Mon Sep 17 00:00:00 2001 From: Peter Bendel Date: Fri, 19 Jul 2024 15:40:55 +0200 Subject: [PATCH 187/194] Bodobolero/fix root permissions (#8429) ## Problem My prior PR https://github.com/neondatabase/neon/pull/8422 caused leftovers in the GitHub action runner work directory with root permission. As an example see here https://github.com/neondatabase/neon/actions/runs/10001857641/job/27646237324#step:3:37 To work-around we install vanilla postgres as non-root using deb packages in /home/nonroot user directory ## Summary of changes - since we cannot use root we install the deb pkgs directly and create symbolic links for psql, pgbench and libs in expected places - continue jobs an aws even if azure jobs fail (because this region is currently unreliable) --- .github/workflows/benchmarking.yml | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index 833a4ce33c7f..c132b5b513ff 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -57,6 +57,7 @@ jobs: bench: if: ${{ github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null }} strategy: + fail-fast: false matrix: include: - DEFAULT_PG_VERSION: 16 @@ -439,6 +440,7 @@ jobs: pgbench-pgvector: strategy: + fail-fast: false matrix: include: - PLATFORM: "neon-captest-pgvector" @@ -451,13 +453,14 @@ jobs: DEFAULT_PG_VERSION: 16 TEST_OUTPUT: /tmp/test_output BUILD_TYPE: remote + LD_LIBRARY_PATH: /home/nonroot/pg/usr/lib/x86_64-linux-gnu SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }} PLATFORM: ${{ matrix.PLATFORM }} runs-on: [ self-hosted, us-east-2, x64 ] container: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:pinned - options: --init --user root + options: --init steps: - uses: actions/checkout@v4 @@ -466,12 +469,19 @@ jobs: # instead of using Neon artifacts containing pgbench - name: Install postgresql-16 where pytest expects it run: | - apt-get update && apt-get install -y postgresql-common - /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y - apt-get -y install postgresql-16 + cd /home/nonroot + wget -q https://apt.postgresql.org/pub/repos/apt/pool/main/p/postgresql-16/libpq5_16.3-1.pgdg110%2B1_amd64.deb + wget -q https://apt.postgresql.org/pub/repos/apt/pool/main/p/postgresql-16/postgresql-client-16_16.3-1.pgdg110%2B1_amd64.deb + wget -q https://apt.postgresql.org/pub/repos/apt/pool/main/p/postgresql-16/postgresql-16_16.3-1.pgdg110%2B1_amd64.deb + dpkg -x libpq5_16.3-1.pgdg110+1_amd64.deb pg + dpkg -x postgresql-client-16_16.3-1.pgdg110+1_amd64.deb pg + dpkg -x postgresql-16_16.3-1.pgdg110+1_amd64.deb pg mkdir -p /tmp/neon/pg_install/v16/bin - ln -s /usr/bin/pgbench /tmp/neon/pg_install/v16/bin/pgbench - ln -s /usr/bin/psql /tmp/neon/pg_install/v16/bin/psql + ln -s /home/nonroot/pg/usr/lib/postgresql/16/bin/pgbench /tmp/neon/pg_install/v16/bin/pgbench + ln -s /home/nonroot/pg/usr/lib/postgresql/16/bin/psql /tmp/neon/pg_install/v16/bin/psql + ln -s /home/nonroot/pg/usr/lib/x86_64-linux-gnu /tmp/neon/pg_install/v16/lib + /tmp/neon/pg_install/v16/bin/pgbench --version + /tmp/neon/pg_install/v16/bin/psql --version - name: Set up Connection String id: set-up-connstr @@ -532,7 +542,6 @@ jobs: env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} - clickbench-compare: # ClichBench DB for rds-aurora and rds-Postgres deployed to the same clusters # we use for performance testing in pgbench-compare. From 9b883e46516fab4191573a4494562fc764492cff Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Fri, 19 Jul 2024 18:01:02 +0200 Subject: [PATCH 188/194] pageserver: remove obsolete cached_metric_collection_interval (#8370) We're removing the usage of this long-meaningless config field in https://github.com/neondatabase/aws/pull/1599 Once that PR has been deployed to staging and prod, we can merge this PR. --- pageserver/src/bin/pageserver.rs | 1 - pageserver/src/config.rs | 24 ------------------- pageserver/src/consumption_metrics.rs | 7 ------ .../test_pageserver_metric_collection.py | 2 -- 4 files changed, 34 deletions(-) diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 9f705f0bc923..fceddfb7575c 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -622,7 +622,6 @@ fn start_pageserver( metric_collection_endpoint, &conf.metric_collection_bucket, conf.metric_collection_interval, - conf.cached_metric_collection_interval, conf.synthetic_size_calculation_interval, conf.id, local_disk_storage, diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index 5b103b551fb1..35b4e7936524 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -68,7 +68,6 @@ pub mod defaults { super::ConfigurableSemaphore::DEFAULT_INITIAL.get(); pub const DEFAULT_METRIC_COLLECTION_INTERVAL: &str = "10 min"; - pub const DEFAULT_CACHED_METRIC_COLLECTION_INTERVAL: &str = "0s"; pub const DEFAULT_METRIC_COLLECTION_ENDPOINT: Option = None; pub const DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL: &str = "10 min"; pub const DEFAULT_BACKGROUND_TASK_MAXIMUM_DELAY: &str = "10s"; @@ -123,7 +122,6 @@ pub mod defaults { #concurrent_tenant_warmup = '{DEFAULT_CONCURRENT_TENANT_WARMUP}' #metric_collection_interval = '{DEFAULT_METRIC_COLLECTION_INTERVAL}' -#cached_metric_collection_interval = '{DEFAULT_CACHED_METRIC_COLLECTION_INTERVAL}' #synthetic_size_calculation_interval = '{DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL}' #disk_usage_based_eviction = {{ max_usage_pct = .., min_avail_bytes = .., period = "10s"}} @@ -238,7 +236,6 @@ pub struct PageServerConf { // How often to collect metrics and send them to the metrics endpoint. pub metric_collection_interval: Duration, // How often to send unchanged cached metrics to the metrics endpoint. - pub cached_metric_collection_interval: Duration, pub metric_collection_endpoint: Option, pub metric_collection_bucket: Option, pub synthetic_size_calculation_interval: Duration, @@ -370,7 +367,6 @@ struct PageServerConfigBuilder { concurrent_tenant_size_logical_size_queries: BuilderValue, metric_collection_interval: BuilderValue, - cached_metric_collection_interval: BuilderValue, metric_collection_endpoint: BuilderValue>, synthetic_size_calculation_interval: BuilderValue, metric_collection_bucket: BuilderValue>, @@ -454,10 +450,6 @@ impl PageServerConfigBuilder { DEFAULT_METRIC_COLLECTION_INTERVAL, ) .expect("cannot parse default metric collection interval")), - cached_metric_collection_interval: Set(humantime::parse_duration( - DEFAULT_CACHED_METRIC_COLLECTION_INTERVAL, - ) - .expect("cannot parse default cached_metric_collection_interval")), synthetic_size_calculation_interval: Set(humantime::parse_duration( DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL, ) @@ -589,14 +581,6 @@ impl PageServerConfigBuilder { self.metric_collection_interval = BuilderValue::Set(metric_collection_interval) } - pub fn cached_metric_collection_interval( - &mut self, - cached_metric_collection_interval: Duration, - ) { - self.cached_metric_collection_interval = - BuilderValue::Set(cached_metric_collection_interval) - } - pub fn metric_collection_endpoint(&mut self, metric_collection_endpoint: Option) { self.metric_collection_endpoint = BuilderValue::Set(metric_collection_endpoint) } @@ -730,7 +714,6 @@ impl PageServerConfigBuilder { broker_keepalive_interval, log_format, metric_collection_interval, - cached_metric_collection_interval, metric_collection_endpoint, metric_collection_bucket, synthetic_size_calculation_interval, @@ -947,7 +930,6 @@ impl PageServerConf { NonZeroUsize::new(permits).context("initial semaphore permits out of range: 0, use other configuration to disable a feature")? }), "metric_collection_interval" => builder.metric_collection_interval(parse_toml_duration(key, item)?), - "cached_metric_collection_interval" => builder.cached_metric_collection_interval(parse_toml_duration(key, item)?), "metric_collection_endpoint" => { let endpoint = parse_toml_string(key, item)?.parse().context("failed to parse metric_collection_endpoint")?; builder.metric_collection_endpoint(Some(endpoint)); @@ -1080,7 +1062,6 @@ impl PageServerConf { eviction_task_immitated_concurrent_logical_size_queries: ConfigurableSemaphore::default( ), metric_collection_interval: Duration::from_secs(60), - cached_metric_collection_interval: Duration::from_secs(60 * 60), metric_collection_endpoint: defaults::DEFAULT_METRIC_COLLECTION_ENDPOINT, metric_collection_bucket: None, synthetic_size_calculation_interval: Duration::from_secs(60), @@ -1259,7 +1240,6 @@ initial_superuser_name = 'zzzz' id = 10 metric_collection_interval = '222 s' -cached_metric_collection_interval = '22200 s' metric_collection_endpoint = 'http://localhost:80/metrics' synthetic_size_calculation_interval = '333 s' @@ -1315,9 +1295,6 @@ background_task_maximum_delay = '334 s' metric_collection_interval: humantime::parse_duration( defaults::DEFAULT_METRIC_COLLECTION_INTERVAL )?, - cached_metric_collection_interval: humantime::parse_duration( - defaults::DEFAULT_CACHED_METRIC_COLLECTION_INTERVAL - )?, metric_collection_endpoint: defaults::DEFAULT_METRIC_COLLECTION_ENDPOINT, metric_collection_bucket: None, synthetic_size_calculation_interval: humantime::parse_duration( @@ -1396,7 +1373,6 @@ background_task_maximum_delay = '334 s' eviction_task_immitated_concurrent_logical_size_queries: ConfigurableSemaphore::default(), metric_collection_interval: Duration::from_secs(222), - cached_metric_collection_interval: Duration::from_secs(22200), metric_collection_endpoint: Some(Url::parse("http://localhost:80/metrics")?), metric_collection_bucket: None, synthetic_size_calculation_interval: Duration::from_secs(333), diff --git a/pageserver/src/consumption_metrics.rs b/pageserver/src/consumption_metrics.rs index 18c1a6cd9bc2..6861adad2c24 100644 --- a/pageserver/src/consumption_metrics.rs +++ b/pageserver/src/consumption_metrics.rs @@ -46,19 +46,12 @@ pub async fn collect_metrics( metric_collection_endpoint: &Url, metric_collection_bucket: &Option, metric_collection_interval: Duration, - _cached_metric_collection_interval: Duration, synthetic_size_calculation_interval: Duration, node_id: NodeId, local_disk_storage: Utf8PathBuf, cancel: CancellationToken, ctx: RequestContext, ) -> anyhow::Result<()> { - if _cached_metric_collection_interval != Duration::ZERO { - tracing::warn!( - "cached_metric_collection_interval is no longer used, please set it to zero." - ) - } - // spin up background worker that caclulates tenant sizes let worker_ctx = ctx.detached_child(TaskKind::CalculateSyntheticSize, DownloadBehavior::Download); diff --git a/test_runner/regress/test_pageserver_metric_collection.py b/test_runner/regress/test_pageserver_metric_collection.py index cea35a6acb73..24a37b04ec5f 100644 --- a/test_runner/regress/test_pageserver_metric_collection.py +++ b/test_runner/regress/test_pageserver_metric_collection.py @@ -58,7 +58,6 @@ def metrics_handler(request: Request) -> Response: metric_collection_interval="1s" metric_collection_endpoint="{metric_collection_endpoint}" metric_collection_bucket={remote_storage_to_toml_inline_table(neon_env_builder.pageserver_remote_storage)} - cached_metric_collection_interval="0s" synthetic_size_calculation_interval="3s" """ @@ -216,7 +215,6 @@ def metrics_handler(request: Request) -> Response: neon_env_builder.pageserver_config_override = f""" metric_collection_interval="1s" metric_collection_endpoint="{metric_collection_endpoint}" - cached_metric_collection_interval="0s" synthetic_size_calculation_interval="3s" """ From affe4084334a6700a858ba3e8c3a63e65ea85249 Mon Sep 17 00:00:00 2001 From: John Spray Date: Fri, 19 Jul 2024 17:07:59 +0100 Subject: [PATCH 189/194] storage scrubber: GC ancestor shard layers (#8196) ## Problem After a shard split, the pageserver leaves the ancestor shard's content in place. It may be referenced by child shards, but eventually child shards will de-reference most ancestor layers as they write their own data and do GC. We would like to eventually clean up those ancestor layers to reclaim space. ## Summary of changes - Extend the physical GC command with `--mode=full`, which includes cleaning up unreferenced ancestor shard layers - Add test `test_scrubber_physical_gc_ancestors` - Remove colored log output: in testing this is irritating ANSI code spam in logs, and in interactive use doesn't add much. - Refactor storage controller API client code out of storcon_client into a `storage_controller/client` crate - During physical GC of ancestors, call into the storage controller to check that the latest shards seen in S3 reflect the latest state of the tenant, and there is no shard split in progress. --- Cargo.lock | 41 +- Cargo.toml | 4 +- control_plane/storcon_cli/Cargo.toml | 1 + control_plane/storcon_cli/src/main.rs | 62 +-- libs/pageserver_api/src/controller_api.rs | 4 +- libs/utils/src/auth.rs | 4 + pageserver/src/auth.rs | 16 +- safekeeper/src/auth.rs | 16 +- storage_controller/client/Cargo.toml | 23 + storage_controller/client/src/control_api.rs | 62 +++ storage_controller/client/src/lib.rs | 1 + storage_controller/src/http.rs | 2 +- storage_controller/src/main.rs | 18 +- storage_controller/src/service.rs | 2 + storage_scrubber/Cargo.toml | 1 + storage_scrubber/src/lib.rs | 33 +- storage_scrubber/src/main.rs | 46 +- .../src/pageserver_physical_gc.rs | 481 +++++++++++++++--- test_runner/fixtures/neon_fixtures.py | 24 +- .../regress/test_pageserver_generations.py | 3 +- .../regress/test_pageserver_secondary.py | 6 +- test_runner/regress/test_sharding.py | 3 +- test_runner/regress/test_storage_scrubber.py | 237 ++++++++- test_runner/regress/test_tenant_delete.py | 6 +- 24 files changed, 905 insertions(+), 191 deletions(-) create mode 100644 storage_controller/client/Cargo.toml create mode 100644 storage_controller/client/src/control_api.rs create mode 100644 storage_controller/client/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index d08da0babd36..2505d4d3ed5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3234,16 +3234,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "nu-ansi-term" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" -dependencies = [ - "overload", - "winapi", -] - [[package]] name = "num" version = "0.4.1" @@ -3539,12 +3529,6 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - [[package]] name = "p256" version = "0.11.1" @@ -5822,6 +5806,28 @@ dependencies = [ "workspace_hack", ] +[[package]] +name = "storage_controller_client" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "futures", + "pageserver_api", + "pageserver_client", + "postgres", + "reqwest 0.12.4", + "serde", + "thiserror", + "tokio", + "tokio-postgres", + "tokio-stream", + "tokio-util", + "utils", + "workspace_hack", +] + [[package]] name = "storage_scrubber" version = "0.1.0" @@ -5856,6 +5862,7 @@ dependencies = [ "serde", "serde_json", "serde_with", + "storage_controller_client", "thiserror", "tokio", "tokio-postgres", @@ -5885,6 +5892,7 @@ dependencies = [ "reqwest 0.12.4", "serde", "serde_json", + "storage_controller_client", "thiserror", "tokio", "tracing", @@ -6611,7 +6619,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" dependencies = [ "matchers", - "nu-ansi-term", "once_cell", "regex", "serde", diff --git a/Cargo.toml b/Cargo.toml index b9b4bafb4f69..615f5472ec48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ members = [ "safekeeper", "storage_broker", "storage_controller", + "storage_controller/client", "storage_scrubber", "workspace_hack", "libs/compute_api", @@ -182,7 +183,7 @@ tower-service = "0.3.2" tracing = "0.1" tracing-error = "0.2.0" tracing-opentelemetry = "0.21.0" -tracing-subscriber = { version = "0.3", default-features = false, features = ["smallvec", "fmt", "tracing-log", "std", "env-filter", "json", "ansi"] } +tracing-subscriber = { version = "0.3", default-features = false, features = ["smallvec", "fmt", "tracing-log", "std", "env-filter", "json"] } twox-hash = { version = "1.6.3", default-features = false } typed-json = "0.1" url = "2.2" @@ -221,6 +222,7 @@ remote_storage = { version = "0.1", path = "./libs/remote_storage/" } safekeeper_api = { version = "0.1", path = "./libs/safekeeper_api" } desim = { version = "0.1", path = "./libs/desim" } storage_broker = { version = "0.1", path = "./storage_broker/" } # Note: main broker code is inside the binary crate, so linking with the library shouldn't be heavy. +storage_controller_client = { path = "./storage_controller/client" } tenant_size_model = { version = "0.1", path = "./libs/tenant_size_model/" } tracing-utils = { version = "0.1", path = "./libs/tracing-utils/" } utils = { version = "0.1", path = "./libs/utils/" } diff --git a/control_plane/storcon_cli/Cargo.toml b/control_plane/storcon_cli/Cargo.toml index f96f0084b2e3..be69208d0d12 100644 --- a/control_plane/storcon_cli/Cargo.toml +++ b/control_plane/storcon_cli/Cargo.toml @@ -17,6 +17,7 @@ pageserver_client.workspace = true reqwest.workspace = true serde.workspace = true serde_json = { workspace = true, features = ["raw_value"] } +storage_controller_client.workspace = true thiserror.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/control_plane/storcon_cli/src/main.rs b/control_plane/storcon_cli/src/main.rs index 777a717a7378..5c1add070aaf 100644 --- a/control_plane/storcon_cli/src/main.rs +++ b/control_plane/storcon_cli/src/main.rs @@ -14,15 +14,15 @@ use pageserver_api::{ }, shard::{ShardStripeSize, TenantShardId}, }; -use pageserver_client::mgmt_api::{self, ResponseErrorMessageExt}; +use pageserver_client::mgmt_api::{self}; use reqwest::{Method, StatusCode, Url}; -use serde::{de::DeserializeOwned, Serialize}; use utils::id::{NodeId, TenantId}; use pageserver_api::controller_api::{ NodeConfigureRequest, NodeRegisterRequest, NodeSchedulingPolicy, PlacementPolicy, TenantShardMigrateRequest, TenantShardMigrateResponse, }; +use storage_controller_client::control_api::Client; #[derive(Subcommand, Debug)] enum Command { @@ -249,64 +249,6 @@ impl FromStr for NodeAvailabilityArg { } } -struct Client { - base_url: Url, - jwt_token: Option, - client: reqwest::Client, -} - -impl Client { - fn new(base_url: Url, jwt_token: Option) -> Self { - Self { - base_url, - jwt_token, - client: reqwest::ClientBuilder::new() - .build() - .expect("Failed to construct http client"), - } - } - - /// Simple HTTP request wrapper for calling into storage controller - async fn dispatch( - &self, - method: Method, - path: String, - body: Option, - ) -> mgmt_api::Result - where - RQ: Serialize + Sized, - RS: DeserializeOwned + Sized, - { - // The configured URL has the /upcall path prefix for pageservers to use: we will strip that out - // for general purpose API access. - let url = Url::from_str(&format!( - "http://{}:{}/{path}", - self.base_url.host_str().unwrap(), - self.base_url.port().unwrap() - )) - .unwrap(); - - let mut builder = self.client.request(method, url); - if let Some(body) = body { - builder = builder.json(&body) - } - if let Some(jwt_token) = &self.jwt_token { - builder = builder.header( - reqwest::header::AUTHORIZATION, - format!("Bearer {jwt_token}"), - ); - } - - let response = builder.send().await.map_err(mgmt_api::Error::ReceiveBody)?; - let response = response.error_from_body().await?; - - response - .json() - .await - .map_err(pageserver_client::mgmt_api::Error::ReceiveBody) - } -} - #[tokio::main] async fn main() -> anyhow::Result<()> { let cli = Cli::parse(); diff --git a/libs/pageserver_api/src/controller_api.rs b/libs/pageserver_api/src/controller_api.rs index f05c1315eafa..d0e1eb6b2894 100644 --- a/libs/pageserver_api/src/controller_api.rs +++ b/libs/pageserver_api/src/controller_api.rs @@ -87,7 +87,7 @@ pub struct TenantLocateResponse { pub shard_params: ShardParameters, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug)] pub struct TenantDescribeResponse { pub tenant_id: TenantId, pub shards: Vec, @@ -110,7 +110,7 @@ pub struct NodeDescribeResponse { pub listen_pg_port: u16, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug)] pub struct TenantDescribeResponseShard { pub tenant_shard_id: TenantShardId, diff --git a/libs/utils/src/auth.rs b/libs/utils/src/auth.rs index 03e65f74fe12..a1170a460d99 100644 --- a/libs/utils/src/auth.rs +++ b/libs/utils/src/auth.rs @@ -33,6 +33,10 @@ pub enum Scope { GenerationsApi, // Allows access to control plane managment API and some storage controller endpoints. Admin, + + /// Allows access to storage controller APIs used by the scrubber, to interrogate the state + /// of a tenant & post scrub results. + Scrubber, } /// JWT payload. See docs/authentication.md for the format diff --git a/pageserver/src/auth.rs b/pageserver/src/auth.rs index 4785c8c4c5dc..9e3dedb75a11 100644 --- a/pageserver/src/auth.rs +++ b/pageserver/src/auth.rs @@ -14,12 +14,14 @@ pub fn check_permission(claims: &Claims, tenant_id: Option) -> Result< } (Scope::PageServerApi, None) => Ok(()), // access to management api for PageServerApi scope (Scope::PageServerApi, Some(_)) => Ok(()), // access to tenant api using PageServerApi scope - (Scope::Admin | Scope::SafekeeperData | Scope::GenerationsApi, _) => Err(AuthError( - format!( - "JWT scope '{:?}' is ineligible for Pageserver auth", - claims.scope - ) - .into(), - )), + (Scope::Admin | Scope::SafekeeperData | Scope::GenerationsApi | Scope::Scrubber, _) => { + Err(AuthError( + format!( + "JWT scope '{:?}' is ineligible for Pageserver auth", + claims.scope + ) + .into(), + )) + } } } diff --git a/safekeeper/src/auth.rs b/safekeeper/src/auth.rs index dd9058c4681a..b8bc3f3e0689 100644 --- a/safekeeper/src/auth.rs +++ b/safekeeper/src/auth.rs @@ -12,13 +12,15 @@ pub fn check_permission(claims: &Claims, tenant_id: Option) -> Result< } Ok(()) } - (Scope::Admin | Scope::PageServerApi | Scope::GenerationsApi, _) => Err(AuthError( - format!( - "JWT scope '{:?}' is ineligible for Safekeeper auth", - claims.scope - ) - .into(), - )), + (Scope::Admin | Scope::PageServerApi | Scope::GenerationsApi | Scope::Scrubber, _) => { + Err(AuthError( + format!( + "JWT scope '{:?}' is ineligible for Safekeeper auth", + claims.scope + ) + .into(), + )) + } (Scope::SafekeeperData, _) => Ok(()), } } diff --git a/storage_controller/client/Cargo.toml b/storage_controller/client/Cargo.toml new file mode 100644 index 000000000000..c3bfe2bfd2f9 --- /dev/null +++ b/storage_controller/client/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "storage_controller_client" +version = "0.1.0" +edition.workspace = true +license.workspace = true + +[dependencies] +pageserver_api.workspace = true +pageserver_client.workspace = true +thiserror.workspace = true +async-trait.workspace = true +reqwest.workspace = true +utils.workspace = true +serde.workspace = true +workspace_hack = { version = "0.1", path = "../../workspace_hack" } +tokio-postgres.workspace = true +tokio-stream.workspace = true +tokio.workspace = true +futures.workspace = true +tokio-util.workspace = true +anyhow.workspace = true +postgres.workspace = true +bytes.workspace = true diff --git a/storage_controller/client/src/control_api.rs b/storage_controller/client/src/control_api.rs new file mode 100644 index 000000000000..a981b5020e69 --- /dev/null +++ b/storage_controller/client/src/control_api.rs @@ -0,0 +1,62 @@ +use pageserver_client::mgmt_api::{self, ResponseErrorMessageExt}; +use reqwest::{Method, Url}; +use serde::{de::DeserializeOwned, Serialize}; +use std::str::FromStr; + +pub struct Client { + base_url: Url, + jwt_token: Option, + client: reqwest::Client, +} + +impl Client { + pub fn new(base_url: Url, jwt_token: Option) -> Self { + Self { + base_url, + jwt_token, + client: reqwest::ClientBuilder::new() + .build() + .expect("Failed to construct http client"), + } + } + + /// Simple HTTP request wrapper for calling into storage controller + pub async fn dispatch( + &self, + method: Method, + path: String, + body: Option, + ) -> mgmt_api::Result + where + RQ: Serialize + Sized, + RS: DeserializeOwned + Sized, + { + // The configured URL has the /upcall path prefix for pageservers to use: we will strip that out + // for general purpose API access. + let url = Url::from_str(&format!( + "http://{}:{}/{path}", + self.base_url.host_str().unwrap(), + self.base_url.port().unwrap() + )) + .unwrap(); + + let mut builder = self.client.request(method, url); + if let Some(body) = body { + builder = builder.json(&body) + } + if let Some(jwt_token) = &self.jwt_token { + builder = builder.header( + reqwest::header::AUTHORIZATION, + format!("Bearer {jwt_token}"), + ); + } + + let response = builder.send().await.map_err(mgmt_api::Error::ReceiveBody)?; + let response = response.error_from_body().await?; + + response + .json() + .await + .map_err(pageserver_client::mgmt_api::Error::ReceiveBody) + } +} diff --git a/storage_controller/client/src/lib.rs b/storage_controller/client/src/lib.rs new file mode 100644 index 000000000000..6d5e20294271 --- /dev/null +++ b/storage_controller/client/src/lib.rs @@ -0,0 +1 @@ +pub mod control_api; diff --git a/storage_controller/src/http.rs b/storage_controller/src/http.rs index 9ddf98eb3bb6..8fb4be93e001 100644 --- a/storage_controller/src/http.rs +++ b/storage_controller/src/http.rs @@ -430,7 +430,7 @@ async fn handle_tenant_describe( service: Arc, req: Request, ) -> Result, ApiError> { - check_permissions(&req, Scope::Admin)?; + check_permissions(&req, Scope::Scrubber)?; let tenant_id: TenantId = parse_request_param(&req, "tenant_id")?; json_response(StatusCode::OK, service.tenant_describe(tenant_id)?) diff --git a/storage_controller/src/main.rs b/storage_controller/src/main.rs index 4bf6b528f49e..789f96beb397 100644 --- a/storage_controller/src/main.rs +++ b/storage_controller/src/main.rs @@ -5,6 +5,7 @@ use metrics::launch_timestamp::LaunchTimestamp; use metrics::BuildInfo; use std::path::PathBuf; use std::sync::Arc; +use std::time::Duration; use storage_controller::http::make_router; use storage_controller::metrics::preinitialize_metrics; use storage_controller::persistence::Persistence; @@ -310,12 +311,21 @@ async fn async_main() -> anyhow::Result<()> { tracing::info!("Terminating on signal"); // Stop HTTP server first, so that we don't have to service requests - // while shutting down Service + // while shutting down Service. server_shutdown.cancel(); - if let Err(e) = server_task.await { - tracing::error!("Error joining HTTP server task: {e}") + match tokio::time::timeout(Duration::from_secs(5), server_task).await { + Ok(Ok(_)) => { + tracing::info!("Joined HTTP server task"); + } + Ok(Err(e)) => { + tracing::error!("Error joining HTTP server task: {e}") + } + Err(_) => { + tracing::warn!("Timed out joining HTTP server task"); + // We will fall through and shut down the service anyway, any request handlers + // in flight will experience cancellation & their clients will see a torn connection. + } } - tracing::info!("Joined HTTP server task"); service.shutdown().await; tracing::info!("Service shutdown complete"); diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 3c24433c422a..a163453dca40 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -3956,6 +3956,8 @@ impl Service { "failpoint".to_string() ))); + failpoint_support::sleep_millis_async!("shard-split-post-remote-sleep", &self.cancel); + tracing::info!( "Split {} into {}", parent_id, diff --git a/storage_scrubber/Cargo.toml b/storage_scrubber/Cargo.toml index 050be66483b9..5233afbebe47 100644 --- a/storage_scrubber/Cargo.toml +++ b/storage_scrubber/Cargo.toml @@ -34,6 +34,7 @@ camino.workspace = true rustls.workspace = true rustls-native-certs.workspace = true once_cell.workspace = true +storage_controller_client.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } chrono = { workspace = true, default-features = false, features = ["clock", "serde"] } diff --git a/storage_scrubber/src/lib.rs b/storage_scrubber/src/lib.rs index 9102ad9906f2..a0b6d7ea302d 100644 --- a/storage_scrubber/src/lib.rs +++ b/storage_scrubber/src/lib.rs @@ -24,6 +24,7 @@ use camino::{Utf8Path, Utf8PathBuf}; use clap::ValueEnum; use pageserver::tenant::TENANTS_SEGMENT_NAME; use pageserver_api::shard::TenantShardId; +use remote_storage::RemotePath; use reqwest::Url; use serde::{Deserialize, Serialize}; use tokio::io::AsyncReadExt; @@ -31,7 +32,7 @@ use tracing::error; use tracing_appender::non_blocking::WorkerGuard; use tracing_subscriber::{fmt, prelude::*, EnvFilter}; use utils::fs_ext; -use utils::id::{TenantId, TimelineId}; +use utils::id::{TenantId, TenantTimelineId, TimelineId}; const MAX_RETRIES: usize = 20; const CLOUD_ADMIN_API_TOKEN_ENV_VAR: &str = "CLOUD_ADMIN_API_TOKEN"; @@ -54,7 +55,7 @@ pub struct S3Target { /// in the pageserver, as all timeline objects existing in the scope of a particular /// tenant: the scrubber is different in that it handles collections of data referring to many /// TenantShardTimelineIds in on place. -#[derive(Serialize, Deserialize, Debug, Clone, Copy, Hash, PartialEq, Eq)] +#[derive(Serialize, Deserialize, Debug, Clone, Copy, Hash, PartialEq, Eq, PartialOrd, Ord)] pub struct TenantShardTimelineId { tenant_shard_id: TenantShardId, timeline_id: TimelineId, @@ -67,6 +68,10 @@ impl TenantShardTimelineId { timeline_id, } } + + fn as_tenant_timeline_id(&self) -> TenantTimelineId { + TenantTimelineId::new(self.tenant_shard_id.tenant_id, self.timeline_id) + } } impl Display for TenantShardTimelineId { @@ -179,6 +184,22 @@ impl RootTarget { .with_sub_segment(&id.timeline_id.to_string()) } + /// Given RemotePath "tenants/foo/timelines/bar/layerxyz", prefix it to a literal + /// key in the S3 bucket. + pub fn absolute_key(&self, key: &RemotePath) -> String { + let root = match self { + Self::Pageserver(root) => root, + Self::Safekeeper(root) => root, + }; + + let prefix = &root.prefix_in_bucket; + if prefix.ends_with('/') { + format!("{prefix}{key}") + } else { + format!("{prefix}/{key}") + } + } + pub fn bucket_name(&self) -> &str { match self { Self::Pageserver(root) => &root.bucket_name, @@ -216,6 +237,14 @@ impl BucketConfig { } } +pub struct ControllerClientConfig { + /// URL to storage controller. e.g. http://127.0.0.1:1234 when using `neon_local` + pub controller_api: Url, + + /// JWT token for authenticating with storage controller. Requires scope 'scrubber' or 'admin'. + pub controller_jwt: String, +} + pub struct ConsoleConfig { pub token: String, pub base_url: Url, diff --git a/storage_scrubber/src/main.rs b/storage_scrubber/src/main.rs index d81612119263..b3ed6f645177 100644 --- a/storage_scrubber/src/main.rs +++ b/storage_scrubber/src/main.rs @@ -1,11 +1,12 @@ -use anyhow::bail; +use anyhow::{anyhow, bail}; use camino::Utf8PathBuf; use pageserver_api::shard::TenantShardId; -use storage_scrubber::find_large_objects; +use reqwest::Url; use storage_scrubber::garbage::{find_garbage, purge_garbage, PurgeMode}; use storage_scrubber::pageserver_physical_gc::GcMode; use storage_scrubber::scan_pageserver_metadata::scan_metadata; use storage_scrubber::tenant_snapshot::SnapshotDownloader; +use storage_scrubber::{find_large_objects, ControllerClientConfig}; use storage_scrubber::{ init_logging, pageserver_physical_gc::pageserver_physical_gc, scan_safekeeper_metadata::scan_safekeeper_metadata, BucketConfig, ConsoleConfig, NodeKind, @@ -24,6 +25,14 @@ struct Cli { #[arg(short, long, default_value_t = false)] delete: bool, + + #[arg(long)] + /// URL to storage controller. e.g. http://127.0.0.1:1234 when using `neon_local` + controller_api: Option, + + #[arg(long)] + /// JWT token for authenticating with storage controller. Requires scope 'scrubber' or 'admin'. + controller_jwt: Option, } #[derive(Subcommand, Debug)] @@ -204,8 +213,37 @@ async fn main() -> anyhow::Result<()> { min_age, mode, } => { - let summary = - pageserver_physical_gc(bucket_config, tenant_ids, min_age.into(), mode).await?; + let controller_client_conf = cli.controller_api.map(|controller_api| { + ControllerClientConfig { + controller_api, + // Default to no key: this is a convenience when working in a development environment + controller_jwt: cli.controller_jwt.unwrap_or("".to_owned()), + } + }); + + match (&controller_client_conf, mode) { + (Some(_), _) => { + // Any mode may run when controller API is set + } + (None, GcMode::Full) => { + // The part of physical GC where we erase ancestor layers cannot be done safely without + // confirming the most recent complete shard split with the controller. Refuse to run, rather + // than doing it unsafely. + return Err(anyhow!("Full physical GC requires `--controller-api` and `--controller-jwt` to run")); + } + (None, GcMode::DryRun | GcMode::IndicesOnly) => { + // These GcModes do not require the controller to run. + } + } + + let summary = pageserver_physical_gc( + bucket_config, + controller_client_conf, + tenant_ids, + min_age.into(), + mode, + ) + .await?; println!("{}", serde_json::to_string(&summary).unwrap()); Ok(()) } diff --git a/storage_scrubber/src/pageserver_physical_gc.rs b/storage_scrubber/src/pageserver_physical_gc.rs index fb8fbc1635ae..e977fd49f779 100644 --- a/storage_scrubber/src/pageserver_physical_gc.rs +++ b/storage_scrubber/src/pageserver_physical_gc.rs @@ -1,22 +1,50 @@ -use std::time::{Duration, UNIX_EPOCH}; +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; +use std::time::{Duration, SystemTime}; use crate::checks::{list_timeline_blobs, BlobDataParseResult}; use crate::metadata_stream::{stream_tenant_timelines, stream_tenants}; -use crate::{init_remote, BucketConfig, NodeKind, RootTarget, TenantShardTimelineId}; +use crate::{ + init_remote, BucketConfig, ControllerClientConfig, NodeKind, RootTarget, TenantShardTimelineId, +}; use aws_sdk_s3::Client; use futures_util::{StreamExt, TryStreamExt}; -use pageserver::tenant::remote_timeline_client::parse_remote_index_path; +use pageserver::tenant::remote_timeline_client::index::LayerFileMetadata; +use pageserver::tenant::remote_timeline_client::{parse_remote_index_path, remote_layer_path}; +use pageserver::tenant::storage_layer::LayerName; use pageserver::tenant::IndexPart; -use pageserver_api::shard::TenantShardId; +use pageserver_api::controller_api::TenantDescribeResponse; +use pageserver_api::shard::{ShardIndex, TenantShardId}; use remote_storage::RemotePath; +use reqwest::Method; use serde::Serialize; +use storage_controller_client::control_api; use tracing::{info_span, Instrument}; use utils::generation::Generation; +use utils::id::{TenantId, TenantTimelineId}; #[derive(Serialize, Default)] pub struct GcSummary { indices_deleted: usize, remote_storage_errors: usize, + controller_api_errors: usize, + ancestor_layers_deleted: usize, +} + +impl GcSummary { + fn merge(&mut self, other: Self) { + let Self { + indices_deleted, + remote_storage_errors, + ancestor_layers_deleted, + controller_api_errors, + } = other; + + self.indices_deleted += indices_deleted; + self.remote_storage_errors += remote_storage_errors; + self.ancestor_layers_deleted += ancestor_layers_deleted; + self.controller_api_errors += controller_api_errors; + } } #[derive(clap::ValueEnum, Debug, Clone, Copy)] @@ -26,9 +54,9 @@ pub enum GcMode { // Enable only removing old-generation indices IndicesOnly, + // Enable all forms of GC - // TODO: this will be used when shard split ancestor layer deletion is added - // All, + Full, } impl std::fmt::Display for GcMode { @@ -36,8 +64,230 @@ impl std::fmt::Display for GcMode { match self { GcMode::DryRun => write!(f, "dry-run"), GcMode::IndicesOnly => write!(f, "indices-only"), + GcMode::Full => write!(f, "full"), + } + } +} + +mod refs { + use super::*; + // Map of cross-shard layer references, giving a refcount for each layer in each shard that is referenced by some other + // shard in the same tenant. This is sparse! The vast majority of timelines will have no cross-shard refs, and those that + // do have cross shard refs should eventually drop most of them via compaction. + // + // In our inner map type, the TTID in the key is shard-agnostic, and the ShardIndex in the value refers to the _ancestor + // which is is referenced_. + #[derive(Default)] + pub(super) struct AncestorRefs( + BTreeMap>, + ); + + impl AncestorRefs { + /// Insert references for layers discovered in a particular shard-timeline that refer to an ancestral shard-timeline. + pub(super) fn update( + &mut self, + ttid: TenantShardTimelineId, + layers: Vec<(LayerName, LayerFileMetadata)>, + ) { + let ttid_refs = self.0.entry(ttid.as_tenant_timeline_id()).or_default(); + for (layer_name, layer_metadata) in layers { + // Increment refcount of this layer in the ancestor shard + *(ttid_refs + .entry((layer_metadata.shard, layer_name)) + .or_default()) += 1; + } + } + + /// For a particular TTID, return the map of all ancestor layers referenced by a descendent to their refcount + /// + /// The `ShardIndex` in the result's key is the index of the _ancestor_, not the descendent. + pub(super) fn get_ttid_refcounts( + &self, + ttid: &TenantTimelineId, + ) -> Option<&HashMap<(ShardIndex, LayerName), usize>> { + self.0.get(ttid) + } + } +} + +use refs::AncestorRefs; + +// As we see shards for a tenant, acccumulate knowledge needed for cross-shard GC: +// - Are there any ancestor shards? +// - Are there any refs to ancestor shards' layers? +#[derive(Default)] +struct TenantRefAccumulator { + shards_seen: HashMap>, + + // For each shard that has refs to an ancestor's layers, the set of ancestor layers referred to + ancestor_ref_shards: AncestorRefs, +} + +impl TenantRefAccumulator { + fn update(&mut self, ttid: TenantShardTimelineId, index_part: &IndexPart) { + let this_shard_idx = ttid.tenant_shard_id.to_index(); + (*self + .shards_seen + .entry(ttid.tenant_shard_id.tenant_id) + .or_default()) + .push(this_shard_idx); + + let mut ancestor_refs = Vec::new(); + for (layer_name, layer_metadata) in &index_part.layer_metadata { + if layer_metadata.shard != this_shard_idx { + // This is a reference from this shard to a layer in an ancestor shard: we must track this + // as a marker to not GC this layer from the parent. + ancestor_refs.push((layer_name.clone(), layer_metadata.clone())); + } + } + + if !ancestor_refs.is_empty() { + tracing::info!(%ttid, "Found {} ancestor refs", ancestor_refs.len()); + self.ancestor_ref_shards.update(ttid, ancestor_refs); + } + } + + /// Consume Self and return a vector of ancestor tenant shards that should be GC'd, and map of referenced ancestor layers to preserve + async fn into_gc_ancestors( + self, + controller_client: &control_api::Client, + summary: &mut GcSummary, + ) -> (Vec, AncestorRefs) { + let mut ancestors_to_gc = Vec::new(); + for (tenant_id, mut shard_indices) in self.shards_seen { + // Find the highest shard count + let latest_count = shard_indices + .iter() + .map(|i| i.shard_count) + .max() + .expect("Always at least one shard"); + + let (mut latest_shards, ancestor_shards) = { + let at = + itertools::partition(&mut shard_indices, |i| i.shard_count == latest_count); + (shard_indices[0..at].to_owned(), &shard_indices[at..]) + }; + // Sort shards, as we will later compare them with a sorted list from the controller + latest_shards.sort(); + + // Check that we have a complete view of the latest shard count: this should always be the case unless we happened + // to scan the S3 bucket halfway through a shard split. + if latest_shards.len() != latest_count.count() as usize { + // This should be extremely rare, so we warn on it. + tracing::warn!(%tenant_id, "Missed some shards at count {:?}", latest_count); + continue; + } + + // Check if we have any non-latest-count shards + if ancestor_shards.is_empty() { + tracing::debug!(%tenant_id, "No ancestor shards to clean up"); + continue; + } + + // Based on S3 view, this tenant looks like it might have some ancestor shard work to do. We + // must only do this work if the tenant is not currently being split: otherwise, it is not safe + // to GC ancestors, because if the split fails then the controller will try to attach ancestor + // shards again. + match controller_client + .dispatch::<(), TenantDescribeResponse>( + Method::GET, + format!("control/v1/tenant/{tenant_id}"), + None, + ) + .await + { + Err(e) => { + // We were not able to learn the latest shard split state from the controller, so we will not + // do ancestor GC on this tenant. + tracing::warn!(%tenant_id, "Failed to query storage controller, will not do ancestor GC: {e}"); + summary.controller_api_errors += 1; + continue; + } + Ok(desc) => { + // We expect to see that the latest shard count matches the one we saw in S3, and that none + // of the shards indicate splitting in progress. + + let controller_indices: Vec = desc + .shards + .iter() + .map(|s| s.tenant_shard_id.to_index()) + .collect(); + if controller_indices != latest_shards { + tracing::info!(%tenant_id, "Latest shards seen in S3 ({latest_shards:?}) don't match controller state ({controller_indices:?})"); + continue; + } + + if desc.shards.iter().any(|s| s.is_splitting) { + tracing::info!(%tenant_id, "One or more shards is currently splitting"); + continue; + } + + // This shouldn't be too noisy, because we only log this for tenants that have some ancestral refs. + tracing::info!(%tenant_id, "Validated state with controller: {desc:?}"); + } + } + + // GC ancestor shards + for ancestor_shard in ancestor_shards.iter().map(|idx| TenantShardId { + tenant_id, + shard_count: idx.shard_count, + shard_number: idx.shard_number, + }) { + ancestors_to_gc.push(ancestor_shard); + } } + + (ancestors_to_gc, self.ancestor_ref_shards) + } +} + +async fn is_old_enough( + s3_client: &Client, + bucket_config: &BucketConfig, + min_age: &Duration, + key: &str, + summary: &mut GcSummary, +) -> bool { + // Validation: we will only GC indices & layers after a time threshold (e.g. one week) so that during an incident + // it is easier to read old data for analysis, and easier to roll back shard splits without having to un-delete any objects. + let age: Duration = match s3_client + .head_object() + .bucket(&bucket_config.bucket) + .key(key) + .send() + .await + { + Ok(response) => match response.last_modified { + None => { + tracing::warn!("Missing last_modified"); + summary.remote_storage_errors += 1; + return false; + } + Some(last_modified) => match SystemTime::try_from(last_modified).map(|t| t.elapsed()) { + Ok(Ok(e)) => e, + Err(_) | Ok(Err(_)) => { + tracing::warn!("Bad last_modified time: {last_modified:?}"); + return false; + } + }, + }, + Err(e) => { + tracing::warn!("Failed to HEAD {key}: {e}"); + summary.remote_storage_errors += 1; + return false; + } + }; + let old_enough = &age > min_age; + + if !old_enough { + tracing::info!( + "Skipping young object {} < {}", + humantime::format_duration(age), + humantime::format_duration(*min_age) + ); } + + old_enough } async fn maybe_delete_index( @@ -79,45 +329,7 @@ async fn maybe_delete_index( return; } - // Validation: we will only delete indices after one week, so that during incidents we will have - // easy access to recent indices. - let age: Duration = match s3_client - .head_object() - .bucket(&bucket_config.bucket) - .key(key) - .send() - .await - { - Ok(response) => match response.last_modified { - None => { - tracing::warn!("Missing last_modified"); - summary.remote_storage_errors += 1; - return; - } - Some(last_modified) => { - let last_modified = - UNIX_EPOCH + Duration::from_secs_f64(last_modified.as_secs_f64()); - match last_modified.elapsed() { - Ok(e) => e, - Err(_) => { - tracing::warn!("Bad last_modified time: {last_modified:?}"); - return; - } - } - } - }, - Err(e) => { - tracing::warn!("Failed to HEAD {key}: {e}"); - summary.remote_storage_errors += 1; - return; - } - }; - if &age < min_age { - tracing::info!( - "Skipping young object {} < {}", - age.as_secs_f64(), - min_age.as_secs_f64() - ); + if !is_old_enough(s3_client, bucket_config, min_age, key, summary).await { return; } @@ -145,6 +357,108 @@ async fn maybe_delete_index( } } +#[allow(clippy::too_many_arguments)] +async fn gc_ancestor( + s3_client: &Client, + bucket_config: &BucketConfig, + root_target: &RootTarget, + min_age: &Duration, + ancestor: TenantShardId, + refs: &AncestorRefs, + mode: GcMode, + summary: &mut GcSummary, +) -> anyhow::Result<()> { + // Scan timelines in the ancestor + let timelines = stream_tenant_timelines(s3_client, root_target, ancestor).await?; + let mut timelines = std::pin::pin!(timelines); + + // Build a list of keys to retain + + while let Some(ttid) = timelines.next().await { + let ttid = ttid?; + + let data = list_timeline_blobs(s3_client, ttid, root_target).await?; + + let s3_layers = match data.blob_data { + BlobDataParseResult::Parsed { + index_part: _, + index_part_generation: _, + s3_layers, + } => s3_layers, + BlobDataParseResult::Relic => { + // Post-deletion tenant location: don't try and GC it. + continue; + } + BlobDataParseResult::Incorrect(reasons) => { + // Our primary purpose isn't to report on bad data, but log this rather than skipping silently + tracing::warn!( + "Skipping ancestor GC for timeline {ttid}, bad metadata: {reasons:?}" + ); + continue; + } + }; + + let ttid_refs = refs.get_ttid_refcounts(&ttid.as_tenant_timeline_id()); + let ancestor_shard_index = ttid.tenant_shard_id.to_index(); + + for (layer_name, layer_gen) in s3_layers { + let ref_count = ttid_refs + .and_then(|m| m.get(&(ancestor_shard_index, layer_name.clone()))) + .copied() + .unwrap_or(0); + + if ref_count > 0 { + tracing::debug!(%ttid, "Ancestor layer {layer_name} has {ref_count} refs"); + continue; + } + + tracing::info!(%ttid, "Ancestor layer {layer_name} is not referenced"); + + // Build the key for the layer we are considering deleting + let key = root_target.absolute_key(&remote_layer_path( + &ttid.tenant_shard_id.tenant_id, + &ttid.timeline_id, + ancestor_shard_index, + &layer_name, + layer_gen, + )); + + // We apply a time threshold to GCing objects that are un-referenced: this preserves our ability + // to roll back a shard split if we have to, by avoiding deleting ancestor layers right away + if !is_old_enough(s3_client, bucket_config, min_age, &key, summary).await { + continue; + } + + if !matches!(mode, GcMode::Full) { + tracing::info!("Dry run: would delete key {key}"); + continue; + } + + // All validations passed: erase the object + match s3_client + .delete_object() + .bucket(&bucket_config.bucket) + .key(&key) + .send() + .await + { + Ok(_) => { + tracing::info!("Successfully deleted unreferenced ancestor layer {key}"); + summary.ancestor_layers_deleted += 1; + } + Err(e) => { + tracing::warn!("Failed to delete layer {key}: {e}"); + summary.remote_storage_errors += 1; + } + } + } + + // TODO: if all the layers are gone, clean up the whole timeline dir (remove index) + } + + Ok(()) +} + /// Physical garbage collection: removing unused S3 objects. This is distinct from the garbage collection /// done inside the pageserver, which operates at a higher level (keys, layers). This type of garbage collection /// is about removing: @@ -156,22 +470,26 @@ async fn maybe_delete_index( /// make sure that object listings don't get slowed down by large numbers of garbage objects. pub async fn pageserver_physical_gc( bucket_config: BucketConfig, - tenant_ids: Vec, + controller_client_conf: Option, + tenant_shard_ids: Vec, min_age: Duration, mode: GcMode, ) -> anyhow::Result { let (s3_client, target) = init_remote(bucket_config.clone(), NodeKind::Pageserver).await?; - let tenants = if tenant_ids.is_empty() { + let tenants = if tenant_shard_ids.is_empty() { futures::future::Either::Left(stream_tenants(&s3_client, &target)) } else { - futures::future::Either::Right(futures::stream::iter(tenant_ids.into_iter().map(Ok))) + futures::future::Either::Right(futures::stream::iter(tenant_shard_ids.into_iter().map(Ok))) }; // How many tenants to process in parallel. We need to be mindful of pageservers // accessing the same per tenant prefixes, so use a lower setting than pageservers. const CONCURRENCY: usize = 32; + // Accumulate information about each tenant for cross-shard GC step we'll do at the end + let accumulator = Arc::new(std::sync::Mutex::new(TenantRefAccumulator::default())); + // Generate a stream of TenantTimelineId let timelines = tenants.map_ok(|t| stream_tenant_timelines(&s3_client, &target, t)); let timelines = timelines.try_buffered(CONCURRENCY); @@ -185,16 +503,17 @@ pub async fn pageserver_physical_gc( target: &RootTarget, mode: GcMode, ttid: TenantShardTimelineId, + accumulator: &Arc>, ) -> anyhow::Result { let mut summary = GcSummary::default(); let data = list_timeline_blobs(s3_client, ttid, target).await?; - let (latest_gen, candidates) = match &data.blob_data { + let (index_part, latest_gen, candidates) = match &data.blob_data { BlobDataParseResult::Parsed { - index_part: _index_part, + index_part, index_part_generation, s3_layers: _s3_layers, - } => (*index_part_generation, data.unused_index_keys), + } => (index_part, *index_part_generation, data.unused_index_keys), BlobDataParseResult::Relic => { // Post-deletion tenant location: don't try and GC it. return Ok(summary); @@ -206,6 +525,8 @@ pub async fn pageserver_physical_gc( } }; + accumulator.lock().unwrap().update(ttid, index_part); + for key in candidates { maybe_delete_index( s3_client, @@ -222,17 +543,61 @@ pub async fn pageserver_physical_gc( Ok(summary) } - let timelines = timelines - .map_ok(|ttid| gc_timeline(&s3_client, &bucket_config, &min_age, &target, mode, ttid)); - let mut timelines = std::pin::pin!(timelines.try_buffered(CONCURRENCY)); let mut summary = GcSummary::default(); - while let Some(i) = timelines.next().await { - let tl_summary = i?; + // Drain futures for per-shard GC, populating accumulator as a side effect + { + let timelines = timelines.map_ok(|ttid| { + gc_timeline( + &s3_client, + &bucket_config, + &min_age, + &target, + mode, + ttid, + &accumulator, + ) + }); + let mut timelines = std::pin::pin!(timelines.try_buffered(CONCURRENCY)); + + while let Some(i) = timelines.next().await { + summary.merge(i?); + } + } + + // Execute cross-shard GC, using the accumulator's full view of all the shards built in the per-shard GC + let Some(controller_client) = controller_client_conf.as_ref().map(|c| { + let ControllerClientConfig { + controller_api, + controller_jwt, + } = c; + control_api::Client::new(controller_api.clone(), Some(controller_jwt.clone())) + }) else { + tracing::info!("Skipping ancestor layer GC, because no `--controller-api` was specified"); + return Ok(summary); + }; + + let (ancestor_shards, ancestor_refs) = Arc::into_inner(accumulator) + .unwrap() + .into_inner() + .unwrap() + .into_gc_ancestors(&controller_client, &mut summary) + .await; - summary.indices_deleted += tl_summary.indices_deleted; - summary.remote_storage_errors += tl_summary.remote_storage_errors; + for ancestor_shard in ancestor_shards { + gc_ancestor( + &s3_client, + &bucket_config, + &target, + &min_age, + ancestor_shard, + &ancestor_refs, + mode, + &mut summary, + ) + .instrument(info_span!("gc_ancestor", %ancestor_shard)) + .await?; } Ok(summary) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index db7269ad4148..9e39457c066f 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -997,7 +997,7 @@ def __exit__( if self.scrub_on_exit: try: - StorageScrubber(self).scan_metadata() + self.env.storage_scrubber.scan_metadata() except Exception as e: log.error(f"Error during remote storage scrub: {e}") cleanup_error = e @@ -1225,6 +1225,9 @@ def __init__(self, config: NeonEnvBuilder): ) cfg["safekeepers"].append(sk_cfg) + # Scrubber instance for tests that use it, and for use during teardown checks + self.storage_scrubber = StorageScrubber(self, log_dir=config.test_output_dir) + log.info(f"Config: {cfg}") self.neon_cli.init( cfg, @@ -4265,9 +4268,9 @@ def paused(): class StorageScrubber: - def __init__(self, env: NeonEnvBuilder, log_dir: Optional[Path] = None): + def __init__(self, env: NeonEnv, log_dir: Path): self.env = env - self.log_dir = log_dir or env.test_output_dir + self.log_dir = log_dir def scrubber_cli(self, args: list[str], timeout) -> str: assert isinstance(self.env.pageserver_remote_storage, S3Storage) @@ -4284,11 +4287,14 @@ def scrubber_cli(self, args: list[str], timeout) -> str: if s3_storage.endpoint is not None: env.update({"AWS_ENDPOINT_URL": s3_storage.endpoint}) - base_args = [str(self.env.neon_binpath / "storage_scrubber")] + base_args = [ + str(self.env.neon_binpath / "storage_scrubber"), + f"--controller-api={self.env.storage_controller_api}", + ] args = base_args + args (output_path, stdout, status_code) = subprocess_capture( - self.env.test_output_dir, + self.log_dir, args, echo_stderr=True, echo_stdout=True, @@ -4327,7 +4333,10 @@ def tenant_snapshot(self, tenant_id: TenantId, output_path: Path): log.info(f"tenant-snapshot output: {stdout}") def pageserver_physical_gc( - self, min_age_secs: int, tenant_ids: Optional[list[TenantId]] = None + self, + min_age_secs: int, + tenant_ids: Optional[list[TenantId]] = None, + mode: Optional[str] = None, ): args = ["pageserver-physical-gc", "--min-age", f"{min_age_secs}s"] @@ -4337,6 +4346,9 @@ def pageserver_physical_gc( for tenant_id in tenant_ids: args.extend(["--tenant-id", str(tenant_id)]) + if mode is not None: + args.extend(["--mode", mode]) + stdout = self.scrubber_cli( args, timeout=30, diff --git a/test_runner/regress/test_pageserver_generations.py b/test_runner/regress/test_pageserver_generations.py index 7ce38c5c3c82..041942cda33a 100644 --- a/test_runner/regress/test_pageserver_generations.py +++ b/test_runner/regress/test_pageserver_generations.py @@ -22,7 +22,6 @@ NeonEnv, NeonEnvBuilder, PgBin, - StorageScrubber, generate_uploads_and_deletions, ) from fixtures.pageserver.common_types import parse_layer_file_name @@ -215,7 +214,7 @@ def parse_generation_suffix(key): # Having written a mixture of generation-aware and legacy index_part.json, # ensure the scrubber handles the situation as expected. - metadata_summary = StorageScrubber(neon_env_builder).scan_metadata() + metadata_summary = env.storage_scrubber.scan_metadata() assert metadata_summary["tenant_count"] == 1 # Scrubber should have seen our timeline assert metadata_summary["timeline_count"] == 1 assert metadata_summary["timeline_shard_count"] == 1 diff --git a/test_runner/regress/test_pageserver_secondary.py b/test_runner/regress/test_pageserver_secondary.py index 0416078ebc67..58d61eab0de5 100644 --- a/test_runner/regress/test_pageserver_secondary.py +++ b/test_runner/regress/test_pageserver_secondary.py @@ -7,7 +7,7 @@ import pytest from fixtures.common_types import TenantId, TimelineId from fixtures.log_helper import log -from fixtures.neon_fixtures import NeonEnvBuilder, NeonPageserver, StorageScrubber +from fixtures.neon_fixtures import NeonEnvBuilder, NeonPageserver from fixtures.pageserver.common_types import parse_layer_file_name from fixtures.pageserver.utils import ( assert_prefix_empty, @@ -234,7 +234,7 @@ def ignore_notify(request: Request): # Having done a bunch of attach/detach cycles, we will have generated some index garbage: check # that the scrubber sees it and cleans it up. We do this before the final attach+validate pass, # to also validate that the scrubber isn't breaking anything. - gc_summary = StorageScrubber(neon_env_builder).pageserver_physical_gc(min_age_secs=1) + gc_summary = env.storage_scrubber.pageserver_physical_gc(min_age_secs=1) assert gc_summary["remote_storage_errors"] == 0 assert gc_summary["indices_deleted"] > 0 @@ -555,7 +555,7 @@ def test_secondary_downloads(neon_env_builder: NeonEnvBuilder): # Scrub the remote storage # ======================== # This confirms that the scrubber isn't upset by the presence of the heatmap - StorageScrubber(neon_env_builder).scan_metadata() + env.storage_scrubber.scan_metadata() # Detach secondary and delete tenant # =================================== diff --git a/test_runner/regress/test_sharding.py b/test_runner/regress/test_sharding.py index 4471237900b8..90c6e26d012f 100644 --- a/test_runner/regress/test_sharding.py +++ b/test_runner/regress/test_sharding.py @@ -12,7 +12,6 @@ NeonEnv, NeonEnvBuilder, StorageControllerApiException, - StorageScrubber, last_flush_lsn_upload, tenant_get_shards, wait_for_last_flush_lsn, @@ -128,7 +127,7 @@ def get_sizes(): # Check the scrubber isn't confused by sharded content, then disable # it during teardown because we'll have deleted by then - StorageScrubber(neon_env_builder).scan_metadata() + env.storage_scrubber.scan_metadata() neon_env_builder.scrub_on_exit = False env.storage_controller.pageserver_api().tenant_delete(tenant_id) diff --git a/test_runner/regress/test_storage_scrubber.py b/test_runner/regress/test_storage_scrubber.py index 35ae61c380df..635690fc7fba 100644 --- a/test_runner/regress/test_storage_scrubber.py +++ b/test_runner/regress/test_storage_scrubber.py @@ -1,14 +1,19 @@ import os import shutil +import threading +import time +from concurrent.futures import ThreadPoolExecutor from typing import Optional import pytest from fixtures.common_types import TenantId, TenantShardId, TimelineId +from fixtures.log_helper import log from fixtures.neon_fixtures import ( + NeonEnv, NeonEnvBuilder, - StorageScrubber, ) from fixtures.remote_storage import S3Storage, s3_storage +from fixtures.utils import wait_until from fixtures.workload import Workload @@ -60,8 +65,7 @@ def test_scrubber_tenant_snapshot(neon_env_builder: NeonEnvBuilder, shard_count: output_path = neon_env_builder.test_output_dir / "snapshot" os.makedirs(output_path) - scrubber = StorageScrubber(neon_env_builder) - scrubber.tenant_snapshot(tenant_id, output_path) + env.storage_scrubber.tenant_snapshot(tenant_id, output_path) assert len(os.listdir(output_path)) > 0 @@ -111,6 +115,14 @@ def test_scrubber_tenant_snapshot(neon_env_builder: NeonEnvBuilder, shard_count: workload.validate() +def drop_local_state(env: NeonEnv, tenant_id: TenantId): + env.storage_controller.tenant_policy_update(tenant_id, {"placement": "Detached"}) + env.storage_controller.reconcile_until_idle() + + env.storage_controller.tenant_policy_update(tenant_id, {"placement": {"Attached": 0}}) + env.storage_controller.reconcile_until_idle() + + @pytest.mark.parametrize("shard_count", [None, 4]) def test_scrubber_physical_gc(neon_env_builder: NeonEnvBuilder, shard_count: Optional[int]): neon_env_builder.enable_pageserver_remote_storage(s3_storage()) @@ -133,28 +145,231 @@ def test_scrubber_physical_gc(neon_env_builder: NeonEnvBuilder, shard_count: Opt # For each cycle, detach and attach the tenant to bump the generation, and do some writes to generate uploads for _i in range(0, n_cycles): - env.storage_controller.tenant_policy_update(tenant_id, {"placement": "Detached"}) - env.storage_controller.reconcile_until_idle() - - env.storage_controller.tenant_policy_update(tenant_id, {"placement": {"Attached": 0}}) - env.storage_controller.reconcile_until_idle() + drop_local_state(env, tenant_id) # This write includes remote upload, will generate an index in this generation workload.write_rows(1) # With a high min_age, the scrubber should decline to delete anything - gc_summary = StorageScrubber(neon_env_builder).pageserver_physical_gc(min_age_secs=3600) + gc_summary = env.storage_scrubber.pageserver_physical_gc(min_age_secs=3600) assert gc_summary["remote_storage_errors"] == 0 assert gc_summary["indices_deleted"] == 0 # If targeting a different tenant, the scrubber shouldn't do anything - gc_summary = StorageScrubber(neon_env_builder).pageserver_physical_gc( + gc_summary = env.storage_scrubber.pageserver_physical_gc( min_age_secs=1, tenant_ids=[TenantId.generate()] ) assert gc_summary["remote_storage_errors"] == 0 assert gc_summary["indices_deleted"] == 0 # With a low min_age, the scrubber should go ahead and clean up all but the latest 2 generations - gc_summary = StorageScrubber(neon_env_builder).pageserver_physical_gc(min_age_secs=1) + gc_summary = env.storage_scrubber.pageserver_physical_gc(min_age_secs=1) assert gc_summary["remote_storage_errors"] == 0 assert gc_summary["indices_deleted"] == (expect_indices_per_shard - 2) * shard_count + + +@pytest.mark.parametrize("shard_count", [None, 2]) +def test_scrubber_physical_gc_ancestors( + neon_env_builder: NeonEnvBuilder, shard_count: Optional[int] +): + neon_env_builder.enable_pageserver_remote_storage(s3_storage()) + neon_env_builder.num_pageservers = 2 + + env = neon_env_builder.init_configs() + env.start() + + tenant_id = TenantId.generate() + timeline_id = TimelineId.generate() + env.neon_cli.create_tenant( + tenant_id, + timeline_id, + shard_count=shard_count, + conf={ + # Small layers and low compaction thresholds, so that when we split we can expect some to + # be dropped by child shards + "checkpoint_distance": f"{1024 * 1024}", + "compaction_threshold": "1", + "compaction_target_size": f"{1024 * 1024}", + "image_creation_threshold": "2", + "image_layer_creation_check_threshold": "0", + # Disable background compaction, we will do it explicitly + "compaction_period": "0s", + # No PITR, so that as soon as child shards generate an image layer, it covers ancestor deltas + # and makes them GC'able + "pitr_interval": "0s", + }, + ) + + # Make sure the original shard has some layers + workload = Workload(env, tenant_id, timeline_id) + workload.init() + workload.write_rows(100) + + new_shard_count = 4 + assert shard_count is None or new_shard_count > shard_count + shards = env.storage_controller.tenant_shard_split(tenant_id, shard_count=new_shard_count) + + # Make sure child shards have some layers + workload.write_rows(100) + + # Flush deletion queue so that we don't leave any orphan layers in the parent that will confuse subsequent checks: once + # a shard is split, any layers in its prefix that aren't referenced by a child will be considered GC'able, even + # if they were logically deleted before the shard split, just not physically deleted yet because of the queue. + for ps in env.pageservers: + ps.http_client().deletion_queue_flush(execute=True) + + # Before compacting, all the layers in the ancestor should still be referenced by the children: the scrubber + # should not erase any ancestor layers + gc_summary = env.storage_scrubber.pageserver_physical_gc(min_age_secs=1, mode="full") + assert gc_summary["remote_storage_errors"] == 0 + assert gc_summary["indices_deleted"] == 0 + assert gc_summary["ancestor_layers_deleted"] == 0 + + # Write some data and compact: compacting, some ancestor layers should no longer be needed by children + # (the compaction is part of the checkpoint that Workload does for us) + workload.churn_rows(100) + workload.churn_rows(100) + workload.churn_rows(100) + for shard in shards: + ps = env.get_tenant_pageserver(shard) + ps.http_client().timeline_compact(shard, timeline_id) + ps.http_client().timeline_gc(shard, timeline_id, 0) + + # We will use a min_age_secs=1 threshold for deletion, let it pass + time.sleep(2) + + # Our time threshold should be respected: check that with a high threshold we delete nothing + gc_summary = env.storage_scrubber.pageserver_physical_gc(min_age_secs=3600, mode="full") + assert gc_summary["remote_storage_errors"] == 0 + assert gc_summary["indices_deleted"] == 0 + assert gc_summary["ancestor_layers_deleted"] == 0 + + # Now run with a low time threshold: deletions of ancestor layers should be executed + gc_summary = env.storage_scrubber.pageserver_physical_gc(min_age_secs=1, mode="full") + assert gc_summary["remote_storage_errors"] == 0 + assert gc_summary["indices_deleted"] == 0 + assert gc_summary["ancestor_layers_deleted"] > 0 + + # We deleted some layers: now check we didn't corrupt the tenant by doing so. Detach and + # attach it, to drop any local state, then check it's still readable. + workload.stop() + drop_local_state(env, tenant_id) + + workload.validate() + + +def test_scrubber_physical_gc_ancestors_split(neon_env_builder: NeonEnvBuilder): + """ + Exercise ancestor GC while a tenant is partly split: this test ensures that if we have some child shards + which don't reference an ancestor, but some child shards that don't exist yet, then we do not incorrectly + GC any ancestor layers. + """ + neon_env_builder.enable_pageserver_remote_storage(s3_storage()) + neon_env_builder.num_pageservers = 2 + + env = neon_env_builder.init_configs() + env.start() + + tenant_id = TenantId.generate() + timeline_id = TimelineId.generate() + initial_shard_count = 2 + env.neon_cli.create_tenant( + tenant_id, + timeline_id, + shard_count=initial_shard_count, + conf={ + # Small layers and low compaction thresholds, so that when we split we can expect some to + # be dropped by child shards + "checkpoint_distance": f"{1024 * 1024}", + "compaction_threshold": "1", + "compaction_target_size": f"{1024 * 1024}", + "image_creation_threshold": "2", + "image_layer_creation_check_threshold": "0", + # Disable background compaction, we will do it explicitly + "compaction_period": "0s", + # No PITR, so that as soon as child shards generate an image layer, it covers ancestor deltas + # and makes them GC'able + "pitr_interval": "0s", + }, + ) + + unstuck = threading.Event() + + def stuck_split(): + # Pause our shard split after the first shard but before the second, such that when we run + # the scrub, the S3 bucket contains shards 0002, 0101, 0004, 0204 (but not 0104, 0304). + env.storage_controller.configure_failpoints( + ("shard-split-post-remote-sleep", "return(3600000)") + ) + try: + split_response = env.storage_controller.tenant_shard_split(tenant_id, shard_count=4) + except Exception as e: + log.info(f"Split failed with {e}") + else: + if not unstuck.is_set(): + raise RuntimeError(f"Split succeeded unexpectedly ({split_response})") + + with ThreadPoolExecutor(max_workers=1) as threads: + log.info("Starting hung shard split") + stuck_split_fut = threads.submit(stuck_split) + + # Let the controller reach the failpoint + wait_until( + 10, + 1, + lambda: env.storage_controller.assert_log_contains( + 'failpoint "shard-split-post-remote-sleep": sleeping' + ), + ) + + # Run compaction on the new child shards, so that they drop some refs to their parent + child_shards = [ + TenantShardId(tenant_id, 0, 4), + TenantShardId(tenant_id, 2, 4), + ] + log.info("Compacting first two children") + for child in child_shards: + env.get_tenant_pageserver( + TenantShardId(tenant_id, 0, initial_shard_count) + ).http_client().timeline_compact(child, timeline_id) + + # Check that the other child shards weren't created + assert env.get_tenant_pageserver(TenantShardId(tenant_id, 1, 4)) is None + assert env.get_tenant_pageserver(TenantShardId(tenant_id, 3, 4)) is None + + # Run scrubber: it should not incorrectly interpret the **04 shards' lack of refs to all + # ancestor layers as a reason to GC them, because it should realize that a split is in progress. + # (GC requires that controller does not indicate split in progress, and that if we see the highest + # shard count N, then there are N shards present with that shard count). + gc_output = env.storage_scrubber.pageserver_physical_gc(min_age_secs=0, mode="full") + log.info(f"Ran physical GC partway through split: {gc_output}") + assert gc_output["ancestor_layers_deleted"] == 0 + assert gc_output["remote_storage_errors"] == 0 + assert gc_output["controller_api_errors"] == 0 + + # Storage controller shutdown lets our split request client complete + log.info("Stopping storage controller") + unstuck.set() + env.storage_controller.allowed_errors.append(".*Timed out joining HTTP server task.*") + env.storage_controller.stop() + stuck_split_fut.result() + + # Restart the controller and retry the split with the failpoint disabled, this should + # complete successfully and result in an S3 state that allows the scrubber to proceed with removing ancestor layers + log.info("Starting & retrying split") + env.storage_controller.start() + env.storage_controller.tenant_shard_split(tenant_id, shard_count=4) + + # The other child shards exist now, we can compact them to drop refs to ancestor + log.info("Compacting second two children") + for child in [ + TenantShardId(tenant_id, 1, 4), + TenantShardId(tenant_id, 3, 4), + ]: + env.get_tenant_pageserver(child).http_client().timeline_compact(child, timeline_id) + + gc_output = env.storage_scrubber.pageserver_physical_gc(min_age_secs=0, mode="full") + log.info(f"Ran physical GC after split completed: {gc_output}") + assert gc_output["ancestor_layers_deleted"] > 0 + assert gc_output["remote_storage_errors"] == 0 + assert gc_output["controller_api_errors"] == 0 diff --git a/test_runner/regress/test_tenant_delete.py b/test_runner/regress/test_tenant_delete.py index 1d7c8b8e31f0..6d20b3d0de20 100644 --- a/test_runner/regress/test_tenant_delete.py +++ b/test_runner/regress/test_tenant_delete.py @@ -5,7 +5,6 @@ from fixtures.neon_fixtures import ( NeonEnvBuilder, PgBin, - StorageScrubber, wait_for_last_flush_lsn, ) from fixtures.pageserver.http import PageserverApiException @@ -325,7 +324,6 @@ def test_tenant_delete_scrubber(pg_bin: PgBin, neon_env_builder: NeonEnvBuilder) remote_storage_kind = RemoteStorageKind.MOCK_S3 neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) - scrubber = StorageScrubber(neon_env_builder) env = neon_env_builder.init_start(initial_tenant_conf=MANY_SMALL_LAYERS_TENANT_CONFIG) ps_http = env.pageserver.http_client() @@ -340,7 +338,7 @@ def test_tenant_delete_scrubber(pg_bin: PgBin, neon_env_builder: NeonEnvBuilder) wait_for_upload(ps_http, tenant_id, timeline_id, last_flush_lsn) env.stop() - result = scrubber.scan_metadata() + result = env.storage_scrubber.scan_metadata() assert result["with_warnings"] == [] env.start() @@ -348,5 +346,5 @@ def test_tenant_delete_scrubber(pg_bin: PgBin, neon_env_builder: NeonEnvBuilder) ps_http.tenant_delete(tenant_id) env.stop() - scrubber.scan_metadata() + env.storage_scrubber.scan_metadata() assert result["with_warnings"] == [] From 067363fe95678c0d470654f01d6b3177618d09f5 Mon Sep 17 00:00:00 2001 From: Shinya Kato <37682778+shinyaaa@users.noreply.github.com> Date: Sat, 20 Jul 2024 02:10:19 +0900 Subject: [PATCH 190/194] safekeeper: remove unused safekeeper runtimes (#8433) There are unused safekeeper runtimes `WAL_REMOVER_RUNTIME` and `METRICS_SHIFTER_RUNTIME`. `WAL_REMOVER_RUNTIME` was implemented in [#4119](https://github.com/neondatabase/neon/pull/4119) and removed in [#7887](https://github.com/neondatabase/neon/pull/7887). `METRICS_SHIFTER_RUNTIME` was also implemented in [#4119](https://github.com/neondatabase/neon/pull/4119) but has never been used. I removed unused safekeeper runtimes `WAL_REMOVER_RUNTIME` and `METRICS_SHIFTER_RUNTIME`. --- safekeeper/src/lib.rs | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/safekeeper/src/lib.rs b/safekeeper/src/lib.rs index af83feb77fac..8f2920ada39e 100644 --- a/safekeeper/src/lib.rs +++ b/safekeeper/src/lib.rs @@ -173,15 +173,6 @@ pub static BROKER_RUNTIME: Lazy = Lazy::new(|| { .expect("Failed to create broker runtime") }); -pub static WAL_REMOVER_RUNTIME: Lazy = Lazy::new(|| { - tokio::runtime::Builder::new_multi_thread() - .thread_name("WAL remover") - .worker_threads(1) - .enable_all() - .build() - .expect("Failed to create broker runtime") -}); - pub static WAL_BACKUP_RUNTIME: Lazy = Lazy::new(|| { tokio::runtime::Builder::new_multi_thread() .thread_name("WAL backup worker") @@ -189,12 +180,3 @@ pub static WAL_BACKUP_RUNTIME: Lazy = Lazy::new(|| { .build() .expect("Failed to create WAL backup runtime") }); - -pub static METRICS_SHIFTER_RUNTIME: Lazy = Lazy::new(|| { - tokio::runtime::Builder::new_multi_thread() - .thread_name("metric shifter") - .worker_threads(1) - .enable_all() - .build() - .expect("Failed to create broker runtime") -}); From bba062e2626051825437989577142ae97fb2bbe0 Mon Sep 17 00:00:00 2001 From: John Spray Date: Fri, 19 Jul 2024 18:30:28 +0100 Subject: [PATCH 191/194] tests: longer timeouts in test_timeline_deletion_with_files_stuck_in_upload_queue (#8438) ## Problem This test had two locations with 2 second timeouts, which is rather low when we run on a highly contended test machine running lots of tests in parallel. It usually passes, but today I've seen both of these locations time out on separate PRs. Example failure: https://neon-github-public-dev.s3.amazonaws.com/reports/pr-8432/10007868041/index.html#suites/837740b64a53e769572c4ed7b7a7eeeb/6c6a092be083d27c ## Summary of changes - Change 2 second timeouts to 20 second timeouts --- test_runner/regress/test_remote_storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_runner/regress/test_remote_storage.py b/test_runner/regress/test_remote_storage.py index fac7fe9deef6..09f941f582a6 100644 --- a/test_runner/regress/test_remote_storage.py +++ b/test_runner/regress/test_remote_storage.py @@ -577,7 +577,7 @@ def assert_compacted_and_uploads_queued(): > 0 ) - wait_until(20, 0.1, assert_compacted_and_uploads_queued) + wait_until(200, 0.1, assert_compacted_and_uploads_queued) # Regardless, give checkpoint some time to block for good. # Not strictly necessary, but might help uncover failure modes in the future. @@ -619,7 +619,7 @@ def assert_compacted_and_uploads_queued(): ) # timeline deletion should be unblocking checkpoint ops - checkpoint_thread.join(2.0) + checkpoint_thread.join(20.0) assert not checkpoint_thread.is_alive() # Just to be sure, unblock ongoing uploads. If the previous assert was incorrect, or the prometheus metric broken, From f7f9b4aaec21775147dce63100b0cb35b3d90d50 Mon Sep 17 00:00:00 2001 From: Shinya Kato <37682778+shinyaaa@users.noreply.github.com> Date: Sat, 20 Jul 2024 03:20:57 +0900 Subject: [PATCH 192/194] Fix openapi specification (#8273) ## Problem There are some swagger errors in `pageserver/src/http/openapi_spec.yml` ``` Error 431 15000 Object includes not allowed fields Error 569 3100401 should always have a 'required' Error 569 15000 Object includes not allowed fields Error 1111 10037 properties members must be schemas ``` ## Summary of changes Fixed the above errors. --- pageserver/src/http/openapi_spec.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index ae109ec1e75f..4d243ddeb995 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -377,7 +377,7 @@ paths: schema: $ref: "#/components/schemas/ConflictError" - /v1/tenant/{tenant_id}/{timeline_id}/preserve_initdb_archive: + /v1/tenant/{tenant_id}/timeline/{timeline_id}/preserve_initdb_archive: parameters: - name: tenant_id in: path @@ -429,7 +429,9 @@ paths: schema: $ref: "#/components/schemas/SyntheticSizeResponse" text/html: - description: SVG representation of the tenant and it's timelines. + schema: + type: string + description: SVG representation of the tenant and its timelines. "401": description: Unauthorized Error content: @@ -568,7 +570,7 @@ paths: type: string - name: timeline_id in: path - Å•equired: true + required: true schema: type: string @@ -774,15 +776,13 @@ components: TenantCreateRequest: allOf: - $ref: '#/components/schemas/TenantConfig' + - $ref: '#/components/schemas/TenantLoadRequest' - type: object required: - new_tenant_id properties: new_tenant_id: type: string - generation: - type: integer - description: Attachment generation number. TenantLoadRequest: type: object properties: @@ -1106,7 +1106,7 @@ components: reparented_timelines: type: array description: Set of reparented timeline ids - properties: + items: type: string format: hex description: TimelineId From 21b3a191bfd1a1eb1a8df773431a92939033d063 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Fri, 19 Jul 2024 21:01:59 +0200 Subject: [PATCH 193/194] Add archival_config endpoint to pageserver (#8414) This adds an archival_config endpoint to the pageserver. Currently it has no effect, and always "works", but later the intent is that it will make a timeline archived/unarchived. - [x] add yml spec - [x] add endpoint handler Part of https://github.com/neondatabase/neon/issues/8088 --- libs/pageserver_api/src/models.rs | 11 ++++++ pageserver/src/http/openapi_spec.yml | 54 ++++++++++++++++++++++++++++ pageserver/src/http/routes.rs | 44 +++++++++++++++++++++-- pageserver/src/tenant.rs | 9 +++++ 4 files changed, 115 insertions(+), 3 deletions(-) diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index 6abdcb88d0fb..231a604b475b 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -651,6 +651,17 @@ pub struct TenantDetails { pub timelines: Vec, } +#[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Copy, Debug)] +pub enum TimelineArchivalState { + Archived, + Unarchived, +} + +#[derive(Serialize, Deserialize, PartialEq, Eq, Clone)] +pub struct TimelineArchivalConfigRequest { + pub state: TimelineArchivalState, +} + /// This represents the output of the "timeline_detail" and "timeline_list" API calls. #[derive(Debug, Serialize, Deserialize, Clone)] pub struct TimelineInfo { diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index 4d243ddeb995..087d281a0c7b 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -397,6 +397,51 @@ paths: "202": description: Tenant scheduled to load successfully + /v1/tenant/{tenant_shard_id}/timeline/{timeline_id}/archival_config: + parameters: + - name: tenant_shard_id + in: path + required: true + schema: + type: string + - name: timeline_id + in: path + required: true + schema: + type: string + put: + description: | + Either archives or unarchives the given timeline. + An archived timeline may not have any non-archived children. + requestBody: + required: false + content: + application/json: + schema: + $ref: "#/components/schemas/ArchivalConfigRequest" + responses: + "200": + description: Timeline (un)archived successfully + "409": + description: | + The tenant/timeline is already being modified, perhaps by a concurrent call to this API + content: + application/json: + schema: + $ref: "#/components/schemas/ConflictError" + "500": + description: Generic operation error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "503": + description: Temporarily unavailable, please retry. + content: + application/json: + schema: + $ref: "#/components/schemas/ServiceUnavailableError" + /v1/tenant/{tenant_id}/synthetic_size: parameters: - name: tenant_id @@ -846,6 +891,15 @@ components: warm: type: boolean description: Whether to poll remote storage for layers to download. If false, secondary locations don't download anything. + ArchivalConfigRequest: + type: object + required + - state + properties: + state: + description: The archival state of a timeline + type: string + enum: ["Archived", "Unarchived"] TenantConfig: type: object properties: diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index d7ef70477f45..b8063eb5a26b 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -18,14 +18,17 @@ use hyper::StatusCode; use hyper::{Body, Request, Response, Uri}; use metrics::launch_timestamp::LaunchTimestamp; use pageserver_api::models::AuxFilePolicy; +use pageserver_api::models::DownloadRemoteLayersTaskSpawnRequest; use pageserver_api::models::IngestAuxFilesRequest; use pageserver_api::models::ListAuxFilesRequest; use pageserver_api::models::LocationConfig; use pageserver_api::models::LocationConfigListResponse; +use pageserver_api::models::LocationConfigMode; use pageserver_api::models::LsnLease; use pageserver_api::models::LsnLeaseRequest; use pageserver_api::models::ShardParameters; use pageserver_api::models::TenantDetails; +use pageserver_api::models::TenantLocationConfigRequest; use pageserver_api::models::TenantLocationConfigResponse; use pageserver_api::models::TenantScanRemoteStorageResponse; use pageserver_api::models::TenantScanRemoteStorageShard; @@ -33,12 +36,10 @@ use pageserver_api::models::TenantShardLocation; use pageserver_api::models::TenantShardSplitRequest; use pageserver_api::models::TenantShardSplitResponse; use pageserver_api::models::TenantSorting; +use pageserver_api::models::TimelineArchivalConfigRequest; use pageserver_api::models::TopTenantShardItem; use pageserver_api::models::TopTenantShardsRequest; use pageserver_api::models::TopTenantShardsResponse; -use pageserver_api::models::{ - DownloadRemoteLayersTaskSpawnRequest, LocationConfigMode, TenantLocationConfigRequest, -}; use pageserver_api::shard::ShardCount; use pageserver_api::shard::TenantShardId; use remote_storage::DownloadError; @@ -664,6 +665,39 @@ async fn timeline_preserve_initdb_handler( json_response(StatusCode::OK, ()) } +async fn timeline_archival_config_handler( + mut request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { + let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?; + let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; + + let request_data: TimelineArchivalConfigRequest = json_request(&mut request).await?; + check_permission(&request, Some(tenant_shard_id.tenant_id))?; + let state = get_state(&request); + + async { + let tenant = state + .tenant_manager + .get_attached_tenant_shard(tenant_shard_id)?; + + tenant + .apply_timeline_archival_config(timeline_id, request_data.state) + .await + .context("applying archival config") + .map_err(ApiError::InternalServerError)?; + Ok::<_, ApiError>(()) + } + .instrument(info_span!("timeline_archival_config", + tenant_id = %tenant_shard_id.tenant_id, + shard_id = %tenant_shard_id.shard_slug(), + state = ?request_data.state, + %timeline_id)) + .await?; + + json_response(StatusCode::OK, ()) +} + async fn timeline_detail_handler( request: Request, _cancel: CancellationToken, @@ -2789,6 +2823,10 @@ pub fn make_router( "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/preserve_initdb_archive", |r| api_handler(r, timeline_preserve_initdb_handler), ) + .post( + "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/archival_config", + |r| api_handler(r, timeline_archival_config_handler), + ) .get("/v1/tenant/:tenant_shard_id/timeline/:timeline_id", |r| { api_handler(r, timeline_detail_handler) }) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 637051413f16..01f7ac626bbc 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -21,6 +21,7 @@ use futures::FutureExt; use futures::StreamExt; use pageserver_api::models; use pageserver_api::models::AuxFilePolicy; +use pageserver_api::models::TimelineArchivalState; use pageserver_api::models::TimelineState; use pageserver_api::models::TopTenantShardItem; use pageserver_api::models::WalRedoManagerStatus; @@ -1228,6 +1229,14 @@ impl Tenant { Ok(timeline_preloads) } + pub async fn apply_timeline_archival_config( + &self, + _timeline_id: TimelineId, + _config: TimelineArchivalState, + ) -> anyhow::Result<()> { + Ok(()) + } + pub(crate) fn tenant_shard_id(&self) -> TenantShardId { self.tenant_shard_id } From 31bfeaf9343f3c40fa798246dcd5d8c88aaf4aaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Fri, 19 Jul 2024 21:19:30 +0200 Subject: [PATCH 194/194] Use DefaultCredentialsChain AWS authentication in remote_storage (#8440) PR #8299 has switched the storage scrubber to use `DefaultCredentialsChain`. Now we do this for `remote_storage`, as it allows us to use `remote_storage` from inside kubernetes. Most of the diff is due to `GenericRemoteStorage::from_config` becoming `async fn`. --- libs/remote_storage/src/lib.rs | 4 +- libs/remote_storage/src/s3_bucket.rs | 66 ++++------ libs/remote_storage/tests/test_real_azure.rs | 7 +- libs/remote_storage/tests/test_real_s3.rs | 7 +- pageserver/ctl/src/main.rs | 2 +- pageserver/src/bin/pageserver.rs | 6 +- pageserver/src/consumption_metrics.rs | 2 +- pageserver/src/deletion_queue.rs | 20 ++- pageserver/src/pgdatadir_mapping.rs | 2 +- pageserver/src/tenant.rs | 116 +++++++++++------- pageserver/src/tenant/mgr.rs | 4 +- .../src/tenant/remote_timeline_client.rs | 2 +- .../src/tenant/storage_layer/delta_layer.rs | 8 +- .../src/tenant/storage_layer/image_layer.rs | 4 +- .../src/tenant/storage_layer/layer/tests.rs | 25 ++-- .../tenant/storage_layer/merge_iterator.rs | 12 +- pageserver/src/tenant/timeline.rs | 5 +- .../walreceiver/connection_manager.rs | 17 +-- pageserver/src/walingest.rs | 16 ++- proxy/src/context/parquet.rs | 10 +- proxy/src/usage_metrics.rs | 14 ++- safekeeper/src/bin/safekeeper.rs | 2 +- safekeeper/src/wal_backup.rs | 26 ++-- 23 files changed, 220 insertions(+), 157 deletions(-) diff --git a/libs/remote_storage/src/lib.rs b/libs/remote_storage/src/lib.rs index d440c03a0e65..3381c4296f05 100644 --- a/libs/remote_storage/src/lib.rs +++ b/libs/remote_storage/src/lib.rs @@ -443,7 +443,7 @@ impl GenericRemoteStorage> { } impl GenericRemoteStorage { - pub fn from_config(storage_config: &RemoteStorageConfig) -> anyhow::Result { + pub async fn from_config(storage_config: &RemoteStorageConfig) -> anyhow::Result { let timeout = storage_config.timeout; Ok(match &storage_config.storage { RemoteStorageKind::LocalFs { local_path: path } => { @@ -458,7 +458,7 @@ impl GenericRemoteStorage { std::env::var("AWS_ACCESS_KEY_ID").unwrap_or_else(|_| "".into()); info!("Using s3 bucket '{}' in region '{}' as a remote storage, prefix in bucket: '{:?}', bucket endpoint: '{:?}', profile: {profile}, access_key_id: {access_key_id}", s3_config.bucket_name, s3_config.bucket_region, s3_config.prefix_in_bucket, s3_config.endpoint); - Self::AwsS3(Arc::new(S3Bucket::new(s3_config, timeout)?)) + Self::AwsS3(Arc::new(S3Bucket::new(s3_config, timeout).await?)) } RemoteStorageKind::AzureContainer(azure_config) => { let storage_account = azure_config diff --git a/libs/remote_storage/src/s3_bucket.rs b/libs/remote_storage/src/s3_bucket.rs index ef1bd2c04730..b65d8b7e9e7a 100644 --- a/libs/remote_storage/src/s3_bucket.rs +++ b/libs/remote_storage/src/s3_bucket.rs @@ -16,16 +16,10 @@ use std::{ use anyhow::{anyhow, Context as _}; use aws_config::{ - environment::credentials::EnvironmentVariableCredentialsProvider, - imds::credentials::ImdsCredentialsProvider, - meta::credentials::CredentialsProviderChain, - profile::ProfileFileCredentialsProvider, - provider_config::ProviderConfig, + default_provider::credentials::DefaultCredentialsChain, retry::{RetryConfigBuilder, RetryMode}, - web_identity_token::WebIdentityTokenCredentialsProvider, BehaviorVersion, }; -use aws_credential_types::provider::SharedCredentialsProvider; use aws_sdk_s3::{ config::{AsyncSleep, IdentityCache, Region, SharedAsyncSleep}, error::SdkError, @@ -76,40 +70,27 @@ struct GetObjectRequest { } impl S3Bucket { /// Creates the S3 storage, errors if incorrect AWS S3 configuration provided. - pub fn new(remote_storage_config: &S3Config, timeout: Duration) -> anyhow::Result { + pub async fn new(remote_storage_config: &S3Config, timeout: Duration) -> anyhow::Result { tracing::debug!( "Creating s3 remote storage for S3 bucket {}", remote_storage_config.bucket_name ); - let region = Some(Region::new(remote_storage_config.bucket_region.clone())); - - let provider_conf = ProviderConfig::without_region().with_region(region.clone()); - - let credentials_provider = { - // uses "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY" - CredentialsProviderChain::first_try( - "env", - EnvironmentVariableCredentialsProvider::new(), - ) - // uses "AWS_PROFILE" / `aws sso login --profile ` - .or_else( - "profile-sso", - ProfileFileCredentialsProvider::builder() - .configure(&provider_conf) - .build(), - ) - // uses "AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN", "AWS_ROLE_SESSION_NAME" - // needed to access remote extensions bucket - .or_else( - "token", - WebIdentityTokenCredentialsProvider::builder() - .configure(&provider_conf) - .build(), - ) - // uses imds v2 - .or_else("imds", ImdsCredentialsProvider::builder().build()) - }; + let region = Region::new(remote_storage_config.bucket_region.clone()); + let region_opt = Some(region.clone()); + + // https://docs.aws.amazon.com/sdkref/latest/guide/standardized-credentials.html + // https://docs.rs/aws-config/latest/aws_config/default_provider/credentials/struct.DefaultCredentialsChain.html + // Incomplete list of auth methods used by this: + // * "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY" + // * "AWS_PROFILE" / `aws sso login --profile ` + // * "AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN", "AWS_ROLE_SESSION_NAME" + // * http (ECS/EKS) container credentials + // * imds v2 + let credentials_provider = DefaultCredentialsChain::builder() + .region(region) + .build() + .await; // AWS SDK requires us to specify how the RetryConfig should sleep when it wants to back off let sleep_impl: Arc = Arc::new(TokioSleep::new()); @@ -118,9 +99,9 @@ impl S3Bucket { #[allow(deprecated)] /* TODO: https://github.com/neondatabase/neon/issues/7665 */ BehaviorVersion::v2023_11_09(), ) - .region(region) + .region(region_opt) .identity_cache(IdentityCache::lazy().build()) - .credentials_provider(SharedCredentialsProvider::new(credentials_provider)) + .credentials_provider(credentials_provider) .sleep_impl(SharedAsyncSleep::from(sleep_impl)); let sdk_config: aws_config::SdkConfig = std::thread::scope(|s| { @@ -1041,8 +1022,8 @@ mod tests { use crate::{RemotePath, S3Bucket, S3Config}; - #[test] - fn relative_path() { + #[tokio::test] + async fn relative_path() { let all_paths = ["", "some/path", "some/path/"]; let all_paths: Vec = all_paths .iter() @@ -1085,8 +1066,9 @@ mod tests { max_keys_per_list_response: Some(5), upload_storage_class: None, }; - let storage = - S3Bucket::new(&config, std::time::Duration::ZERO).expect("remote storage init"); + let storage = S3Bucket::new(&config, std::time::Duration::ZERO) + .await + .expect("remote storage init"); for (test_path_idx, test_path) in all_paths.iter().enumerate() { let result = storage.relative_path_to_s3_object(test_path); let expected = expected_outputs[prefix_idx][test_path_idx]; diff --git a/libs/remote_storage/tests/test_real_azure.rs b/libs/remote_storage/tests/test_real_azure.rs index 23628dfebecc..3a20649490ba 100644 --- a/libs/remote_storage/tests/test_real_azure.rs +++ b/libs/remote_storage/tests/test_real_azure.rs @@ -31,6 +31,7 @@ struct EnabledAzure { impl EnabledAzure { async fn setup(max_keys_in_list_response: Option) -> Self { let client = create_azure_client(max_keys_in_list_response) + .await .context("Azure client creation") .expect("Azure client creation failed"); @@ -187,7 +188,7 @@ impl AsyncTestContext for MaybeEnabledStorageWithSimpleTestBlobs { } } -fn create_azure_client( +async fn create_azure_client( max_keys_per_list_response: Option, ) -> anyhow::Result> { use rand::Rng; @@ -221,6 +222,8 @@ fn create_azure_client( timeout: Duration::from_secs(120), }; Ok(Arc::new( - GenericRemoteStorage::from_config(&remote_storage_config).context("remote storage init")?, + GenericRemoteStorage::from_config(&remote_storage_config) + .await + .context("remote storage init")?, )) } diff --git a/libs/remote_storage/tests/test_real_s3.rs b/libs/remote_storage/tests/test_real_s3.rs index a273abe867e1..342bc6da0bac 100644 --- a/libs/remote_storage/tests/test_real_s3.rs +++ b/libs/remote_storage/tests/test_real_s3.rs @@ -197,6 +197,7 @@ struct EnabledS3 { impl EnabledS3 { async fn setup(max_keys_in_list_response: Option) -> Self { let client = create_s3_client(max_keys_in_list_response) + .await .context("S3 client creation") .expect("S3 client creation failed"); @@ -352,7 +353,7 @@ impl AsyncTestContext for MaybeEnabledStorageWithSimpleTestBlobs { } } -fn create_s3_client( +async fn create_s3_client( max_keys_per_list_response: Option, ) -> anyhow::Result> { use rand::Rng; @@ -385,7 +386,9 @@ fn create_s3_client( timeout: RemoteStorageConfig::DEFAULT_TIMEOUT, }; Ok(Arc::new( - GenericRemoteStorage::from_config(&remote_storage_config).context("remote storage init")?, + GenericRemoteStorage::from_config(&remote_storage_config) + .await + .context("remote storage init")?, )) } diff --git a/pageserver/ctl/src/main.rs b/pageserver/ctl/src/main.rs index ea09a011e5cf..3fabf629875e 100644 --- a/pageserver/ctl/src/main.rs +++ b/pageserver/ctl/src/main.rs @@ -179,7 +179,7 @@ async fn main() -> anyhow::Result<()> { .get("remote_storage") .expect("need remote_storage"); let config = RemoteStorageConfig::from_toml(toml_item)?; - let storage = remote_storage::GenericRemoteStorage::from_config(&config); + let storage = remote_storage::GenericRemoteStorage::from_config(&config).await; let cancel = CancellationToken::new(); storage .unwrap() diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index fceddfb7575c..ec1ceb54ce0f 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -385,7 +385,7 @@ fn start_pageserver( let shutdown_pageserver = tokio_util::sync::CancellationToken::new(); // Set up remote storage client - let remote_storage = create_remote_storage_client(conf)?; + let remote_storage = BACKGROUND_RUNTIME.block_on(create_remote_storage_client(conf))?; // Set up deletion queue let (deletion_queue, deletion_workers) = DeletionQueue::new( @@ -701,7 +701,7 @@ fn start_pageserver( } } -fn create_remote_storage_client( +async fn create_remote_storage_client( conf: &'static PageServerConf, ) -> anyhow::Result { let config = if let Some(config) = &conf.remote_storage_config { @@ -711,7 +711,7 @@ fn create_remote_storage_client( }; // Create the client - let mut remote_storage = GenericRemoteStorage::from_config(config)?; + let mut remote_storage = GenericRemoteStorage::from_config(config).await?; // If `test_remote_failures` is non-zero, wrap the client with a // wrapper that simulates failures. diff --git a/pageserver/src/consumption_metrics.rs b/pageserver/src/consumption_metrics.rs index 6861adad2c24..9104da60729c 100644 --- a/pageserver/src/consumption_metrics.rs +++ b/pageserver/src/consumption_metrics.rs @@ -96,7 +96,7 @@ pub async fn collect_metrics( .expect("Failed to create http client with timeout"); let bucket_client = if let Some(bucket_config) = metric_collection_bucket { - match GenericRemoteStorage::from_config(bucket_config) { + match GenericRemoteStorage::from_config(bucket_config).await { Ok(client) => Some(client), Err(e) => { // Non-fatal error: if we were given an invalid config, we will proceed diff --git a/pageserver/src/deletion_queue.rs b/pageserver/src/deletion_queue.rs index 3e48552ace44..22f7d5b8242d 100644 --- a/pageserver/src/deletion_queue.rs +++ b/pageserver/src/deletion_queue.rs @@ -828,9 +828,9 @@ mod test { } } - fn setup(test_name: &str) -> anyhow::Result { + async fn setup(test_name: &str) -> anyhow::Result { let test_name = Box::leak(Box::new(format!("deletion_queue__{test_name}"))); - let harness = TenantHarness::create(test_name)?; + let harness = TenantHarness::create(test_name).await?; // We do not load() the harness: we only need its config and remote_storage @@ -844,7 +844,9 @@ mod test { }, timeout: RemoteStorageConfig::DEFAULT_TIMEOUT, }; - let storage = GenericRemoteStorage::from_config(&storage_config).unwrap(); + let storage = GenericRemoteStorage::from_config(&storage_config) + .await + .unwrap(); let mock_control_plane = MockControlPlane::new(); @@ -922,7 +924,9 @@ mod test { #[tokio::test] async fn deletion_queue_smoke() -> anyhow::Result<()> { // Basic test that the deletion queue processes the deletions we pass into it - let ctx = setup("deletion_queue_smoke").expect("Failed test setup"); + let ctx = setup("deletion_queue_smoke") + .await + .expect("Failed test setup"); let client = ctx.deletion_queue.new_client(); client.recover(HashMap::new())?; @@ -992,7 +996,9 @@ mod test { #[tokio::test] async fn deletion_queue_validation() -> anyhow::Result<()> { - let ctx = setup("deletion_queue_validation").expect("Failed test setup"); + let ctx = setup("deletion_queue_validation") + .await + .expect("Failed test setup"); let client = ctx.deletion_queue.new_client(); client.recover(HashMap::new())?; @@ -1051,7 +1057,9 @@ mod test { #[tokio::test] async fn deletion_queue_recovery() -> anyhow::Result<()> { // Basic test that the deletion queue processes the deletions we pass into it - let mut ctx = setup("deletion_queue_recovery").expect("Failed test setup"); + let mut ctx = setup("deletion_queue_recovery") + .await + .expect("Failed test setup"); let client = ctx.deletion_queue.new_client(); client.recover(HashMap::new())?; diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index a821b824d0c3..3bbd084ab498 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -2031,7 +2031,7 @@ mod tests { #[tokio::test] async fn aux_files_round_trip() -> anyhow::Result<()> { let name = "aux_files_round_trip"; - let harness = TenantHarness::create(name)?; + let harness = TenantHarness::create(name).await?; pub const TIMELINE_ID: TimelineId = TimelineId::from_array(hex!("11223344556677881122334455667788")); diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 01f7ac626bbc..6d597526068f 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -3797,7 +3797,7 @@ pub(crate) mod harness { } impl TenantHarness { - pub fn create_custom( + pub async fn create_custom( test_name: &'static str, tenant_conf: TenantConf, tenant_id: TenantId, @@ -3833,7 +3833,7 @@ pub(crate) mod harness { }, timeout: RemoteStorageConfig::DEFAULT_TIMEOUT, }; - let remote_storage = GenericRemoteStorage::from_config(&config).unwrap(); + let remote_storage = GenericRemoteStorage::from_config(&config).await.unwrap(); let deletion_queue = MockDeletionQueue::new(Some(remote_storage.clone())); Ok(Self { @@ -3848,7 +3848,7 @@ pub(crate) mod harness { }) } - pub fn create(test_name: &'static str) -> anyhow::Result { + pub async fn create(test_name: &'static str) -> anyhow::Result { // Disable automatic GC and compaction to make the unit tests more deterministic. // The tests perform them manually if needed. let tenant_conf = TenantConf { @@ -3865,6 +3865,7 @@ pub(crate) mod harness { shard, Generation::new(0xdeadbeef), ) + .await } pub fn span(&self) -> tracing::Span { @@ -4001,7 +4002,7 @@ mod tests { #[tokio::test] async fn test_basic() -> anyhow::Result<()> { - let (tenant, ctx) = TenantHarness::create("test_basic")?.load().await; + let (tenant, ctx) = TenantHarness::create("test_basic").await?.load().await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(0x08), DEFAULT_PG_VERSION, &ctx) .await?; @@ -4048,7 +4049,8 @@ mod tests { #[tokio::test] async fn no_duplicate_timelines() -> anyhow::Result<()> { - let (tenant, ctx) = TenantHarness::create("no_duplicate_timelines")? + let (tenant, ctx) = TenantHarness::create("no_duplicate_timelines") + .await? .load() .await; let _ = tenant @@ -4080,7 +4082,7 @@ mod tests { async fn test_branch() -> anyhow::Result<()> { use std::str::from_utf8; - let (tenant, ctx) = TenantHarness::create("test_branch")?.load().await; + let (tenant, ctx) = TenantHarness::create("test_branch").await?.load().await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) .await?; @@ -4202,7 +4204,8 @@ mod tests { #[tokio::test] async fn test_prohibit_branch_creation_on_garbage_collected_data() -> anyhow::Result<()> { let (tenant, ctx) = - TenantHarness::create("test_prohibit_branch_creation_on_garbage_collected_data")? + TenantHarness::create("test_prohibit_branch_creation_on_garbage_collected_data") + .await? .load() .await; let tline = tenant @@ -4249,7 +4252,8 @@ mod tests { #[tokio::test] async fn test_prohibit_branch_creation_on_pre_initdb_lsn() -> anyhow::Result<()> { let (tenant, ctx) = - TenantHarness::create("test_prohibit_branch_creation_on_pre_initdb_lsn")? + TenantHarness::create("test_prohibit_branch_creation_on_pre_initdb_lsn") + .await? .load() .await; @@ -4304,7 +4308,8 @@ mod tests { #[tokio::test] async fn test_get_branchpoints_from_an_inactive_timeline() -> anyhow::Result<()> { let (tenant, ctx) = - TenantHarness::create("test_get_branchpoints_from_an_inactive_timeline")? + TenantHarness::create("test_get_branchpoints_from_an_inactive_timeline") + .await? .load() .await; let tline = tenant @@ -4361,7 +4366,8 @@ mod tests { #[tokio::test] async fn test_retain_data_in_parent_which_is_needed_for_child() -> anyhow::Result<()> { let (tenant, ctx) = - TenantHarness::create("test_retain_data_in_parent_which_is_needed_for_child")? + TenantHarness::create("test_retain_data_in_parent_which_is_needed_for_child") + .await? .load() .await; let tline = tenant @@ -4391,10 +4397,10 @@ mod tests { } #[tokio::test] async fn test_parent_keeps_data_forever_after_branching() -> anyhow::Result<()> { - let (tenant, ctx) = - TenantHarness::create("test_parent_keeps_data_forever_after_branching")? - .load() - .await; + let (tenant, ctx) = TenantHarness::create("test_parent_keeps_data_forever_after_branching") + .await? + .load() + .await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) .await?; @@ -4432,7 +4438,7 @@ mod tests { #[tokio::test] async fn timeline_load() -> anyhow::Result<()> { const TEST_NAME: &str = "timeline_load"; - let harness = TenantHarness::create(TEST_NAME)?; + let harness = TenantHarness::create(TEST_NAME).await?; { let (tenant, ctx) = harness.load().await; let tline = tenant @@ -4459,7 +4465,7 @@ mod tests { #[tokio::test] async fn timeline_load_with_ancestor() -> anyhow::Result<()> { const TEST_NAME: &str = "timeline_load_with_ancestor"; - let harness = TenantHarness::create(TEST_NAME)?; + let harness = TenantHarness::create(TEST_NAME).await?; // create two timelines { let (tenant, ctx) = harness.load().await; @@ -4507,7 +4513,10 @@ mod tests { #[tokio::test] async fn delta_layer_dumping() -> anyhow::Result<()> { use storage_layer::AsLayerDesc; - let (tenant, ctx) = TenantHarness::create("test_layer_dumping")?.load().await; + let (tenant, ctx) = TenantHarness::create("test_layer_dumping") + .await? + .load() + .await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) .await?; @@ -4534,7 +4543,7 @@ mod tests { #[tokio::test] async fn test_images() -> anyhow::Result<()> { - let (tenant, ctx) = TenantHarness::create("test_images")?.load().await; + let (tenant, ctx) = TenantHarness::create("test_images").await?.load().await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(0x08), DEFAULT_PG_VERSION, &ctx) .await?; @@ -4705,7 +4714,7 @@ mod tests { // #[tokio::test] async fn test_bulk_insert() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_bulk_insert")?; + let harness = TenantHarness::create("test_bulk_insert").await?; let (tenant, ctx) = harness.load().await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(0x08), DEFAULT_PG_VERSION, &ctx) @@ -4736,7 +4745,7 @@ mod tests { // so the search can stop at the first delta layer and doesn't traverse any deeper. #[tokio::test] async fn test_get_vectored() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_get_vectored")?; + let harness = TenantHarness::create("test_get_vectored").await?; let (tenant, ctx) = harness.load().await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(0x08), DEFAULT_PG_VERSION, &ctx) @@ -4814,7 +4823,7 @@ mod tests { #[tokio::test] async fn test_get_vectored_aux_files() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_get_vectored_aux_files")?; + let harness = TenantHarness::create("test_get_vectored_aux_files").await?; let (tenant, ctx) = harness.load().await; let tline = tenant @@ -4900,7 +4909,8 @@ mod tests { TenantId::generate(), ShardIdentity::unsharded(), Generation::new(0xdeadbeef), - )?; + ) + .await?; let (tenant, ctx) = harness.load().await; let mut current_key = Key::from_hex("010000000033333333444444445500000000").unwrap(); @@ -5043,7 +5053,7 @@ mod tests { // ``` #[tokio::test] async fn test_get_vectored_ancestor_descent() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_get_vectored_on_lsn_axis")?; + let harness = TenantHarness::create("test_get_vectored_on_lsn_axis").await?; let (tenant, ctx) = harness.load().await; let start_key = Key::from_hex("010000000033333333444444445500000000").unwrap(); @@ -5192,7 +5202,7 @@ mod tests { name: &'static str, compaction_algorithm: CompactionAlgorithm, ) -> anyhow::Result<()> { - let mut harness = TenantHarness::create(name)?; + let mut harness = TenantHarness::create(name).await?; harness.tenant_conf.compaction_algorithm = CompactionAlgorithmSettings { kind: compaction_algorithm, }; @@ -5276,7 +5286,8 @@ mod tests { #[tokio::test] async fn test_traverse_branches() -> anyhow::Result<()> { - let (tenant, ctx) = TenantHarness::create("test_traverse_branches")? + let (tenant, ctx) = TenantHarness::create("test_traverse_branches") + .await? .load() .await; let mut tline = tenant @@ -5366,7 +5377,8 @@ mod tests { #[tokio::test] async fn test_traverse_ancestors() -> anyhow::Result<()> { - let (tenant, ctx) = TenantHarness::create("test_traverse_ancestors")? + let (tenant, ctx) = TenantHarness::create("test_traverse_ancestors") + .await? .load() .await; let mut tline = tenant @@ -5432,7 +5444,8 @@ mod tests { #[tokio::test] async fn test_write_at_initdb_lsn_takes_optimization_code_path() -> anyhow::Result<()> { - let (tenant, ctx) = TenantHarness::create("test_empty_test_timeline_is_usable")? + let (tenant, ctx) = TenantHarness::create("test_empty_test_timeline_is_usable") + .await? .load() .await; @@ -5501,7 +5514,7 @@ mod tests { #[tokio::test] async fn test_create_guard_crash() -> anyhow::Result<()> { let name = "test_create_guard_crash"; - let harness = TenantHarness::create(name)?; + let harness = TenantHarness::create(name).await?; { let (tenant, ctx) = harness.load().await; let tline = tenant @@ -5554,7 +5567,7 @@ mod tests { name: &'static str, compaction_algorithm: CompactionAlgorithm, ) -> anyhow::Result<()> { - let mut harness = TenantHarness::create(name)?; + let mut harness = TenantHarness::create(name).await?; harness.tenant_conf.compaction_algorithm = CompactionAlgorithmSettings { kind: compaction_algorithm, }; @@ -5578,7 +5591,7 @@ mod tests { #[tokio::test] async fn test_metadata_scan() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_metadata_scan")?; + let harness = TenantHarness::create("test_metadata_scan").await?; let (tenant, ctx) = harness.load().await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) @@ -5697,7 +5710,7 @@ mod tests { #[tokio::test] async fn test_metadata_compaction_trigger() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_metadata_compaction_trigger")?; + let harness = TenantHarness::create("test_metadata_compaction_trigger").await?; let (tenant, ctx) = harness.load().await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) @@ -5756,7 +5769,9 @@ mod tests { #[tokio::test] async fn test_branch_copies_dirty_aux_file_flag() { - let harness = TenantHarness::create("test_branch_copies_dirty_aux_file_flag").unwrap(); + let harness = TenantHarness::create("test_branch_copies_dirty_aux_file_flag") + .await + .unwrap(); // the default aux file policy to switch is v1 if not set by the admins assert_eq!( @@ -5858,7 +5873,9 @@ mod tests { #[tokio::test] async fn aux_file_policy_switch() { - let mut harness = TenantHarness::create("aux_file_policy_switch").unwrap(); + let mut harness = TenantHarness::create("aux_file_policy_switch") + .await + .unwrap(); harness.tenant_conf.switch_aux_file_policy = AuxFilePolicy::CrossValidation; // set to cross-validation mode let (tenant, ctx) = harness.load().await; @@ -6032,7 +6049,9 @@ mod tests { #[tokio::test] async fn aux_file_policy_force_switch() { - let mut harness = TenantHarness::create("aux_file_policy_force_switch").unwrap(); + let mut harness = TenantHarness::create("aux_file_policy_force_switch") + .await + .unwrap(); harness.tenant_conf.switch_aux_file_policy = AuxFilePolicy::V1; let (tenant, ctx) = harness.load().await; @@ -6093,7 +6112,9 @@ mod tests { #[tokio::test] async fn aux_file_policy_auto_detect() { - let mut harness = TenantHarness::create("aux_file_policy_auto_detect").unwrap(); + let mut harness = TenantHarness::create("aux_file_policy_auto_detect") + .await + .unwrap(); harness.tenant_conf.switch_aux_file_policy = AuxFilePolicy::V2; // set to cross-validation mode let (tenant, ctx) = harness.load().await; @@ -6156,7 +6177,7 @@ mod tests { #[tokio::test] async fn test_metadata_image_creation() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_metadata_image_creation")?; + let harness = TenantHarness::create("test_metadata_image_creation").await?; let (tenant, ctx) = harness.load().await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) @@ -6255,7 +6276,7 @@ mod tests { #[tokio::test] async fn test_vectored_missing_data_key_reads() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_vectored_missing_data_key_reads")?; + let harness = TenantHarness::create("test_vectored_missing_data_key_reads").await?; let (tenant, ctx) = harness.load().await; let base_key = Key::from_hex("000000000033333333444444445500000000").unwrap(); @@ -6327,7 +6348,7 @@ mod tests { #[tokio::test] async fn test_vectored_missing_metadata_key_reads() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_vectored_missing_metadata_key_reads")?; + let harness = TenantHarness::create("test_vectored_missing_metadata_key_reads").await?; let (tenant, ctx) = harness.load().await; let base_key = Key::from_hex("620000000033333333444444445500000000").unwrap(); @@ -6419,7 +6440,7 @@ mod tests { #[tokio::test] async fn test_metadata_tombstone_reads() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_metadata_tombstone_reads")?; + let harness = TenantHarness::create("test_metadata_tombstone_reads").await?; let (tenant, ctx) = harness.load().await; let key0 = Key::from_hex("620000000033333333444444445500000000").unwrap(); let key1 = Key::from_hex("620000000033333333444444445500000001").unwrap(); @@ -6499,7 +6520,9 @@ mod tests { #[tokio::test] async fn test_metadata_tombstone_image_creation() { - let harness = TenantHarness::create("test_metadata_tombstone_image_creation").unwrap(); + let harness = TenantHarness::create("test_metadata_tombstone_image_creation") + .await + .unwrap(); let (tenant, ctx) = harness.load().await; let key0 = Key::from_hex("620000000033333333444444445500000000").unwrap(); @@ -6571,8 +6594,9 @@ mod tests { #[tokio::test] async fn test_metadata_tombstone_empty_image_creation() { - let harness = - TenantHarness::create("test_metadata_tombstone_empty_image_creation").unwrap(); + let harness = TenantHarness::create("test_metadata_tombstone_empty_image_creation") + .await + .unwrap(); let (tenant, ctx) = harness.load().await; let key1 = Key::from_hex("620000000033333333444444445500000001").unwrap(); @@ -6635,7 +6659,7 @@ mod tests { #[tokio::test] async fn test_simple_bottom_most_compaction_images() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_simple_bottom_most_compaction_images")?; + let harness = TenantHarness::create("test_simple_bottom_most_compaction_images").await?; let (tenant, ctx) = harness.load().await; fn get_key(id: u32) -> Key { @@ -6843,7 +6867,7 @@ mod tests { #[tokio::test] async fn test_neon_test_record() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_neon_test_record")?; + let harness = TenantHarness::create("test_neon_test_record").await?; let (tenant, ctx) = harness.load().await; fn get_key(id: u32) -> Key { @@ -6924,7 +6948,7 @@ mod tests { #[tokio::test] async fn test_lsn_lease() -> anyhow::Result<()> { - let (tenant, ctx) = TenantHarness::create("test_lsn_lease")?.load().await; + let (tenant, ctx) = TenantHarness::create("test_lsn_lease").await?.load().await; let key = Key::from_hex("010000000033333333444444445500000000").unwrap(); let end_lsn = Lsn(0x100); @@ -7013,7 +7037,7 @@ mod tests { #[tokio::test] async fn test_simple_bottom_most_compaction_deltas() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_simple_bottom_most_compaction_deltas")?; + let harness = TenantHarness::create("test_simple_bottom_most_compaction_deltas").await?; let (tenant, ctx) = harness.load().await; fn get_key(id: u32) -> Key { diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index b0159e22bfc0..49126086772b 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -2698,7 +2698,9 @@ mod tests { // Test that if an InProgress tenant is in the map during shutdown, the shutdown will gracefully // wait for it to complete before proceeding. - let h = TenantHarness::create("shutdown_awaits_in_progress_tenant").unwrap(); + let h = TenantHarness::create("shutdown_awaits_in_progress_tenant") + .await + .unwrap(); let (t, _ctx) = h.load().await; // harness loads it to active, which is forced and nothing is running on the tenant diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index 66b759c8e0d8..bb42fbeebf78 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -2128,7 +2128,7 @@ mod tests { impl TestSetup { async fn new(test_name: &str) -> anyhow::Result { let test_name = Box::leak(Box::new(format!("remote_timeline_client__{test_name}"))); - let harness = TenantHarness::create(test_name)?; + let harness = TenantHarness::create(test_name).await?; let (tenant, ctx) = harness.load().await; let timeline = tenant diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index c34923320aee..512e9e86fac1 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -1934,7 +1934,7 @@ pub(crate) mod test { #[tokio::test] async fn test_delta_layer_vectored_read_end_to_end() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_delta_layer_oversized_vectored_read")?; + let harness = TenantHarness::create("test_delta_layer_oversized_vectored_read").await?; let (tenant, ctx) = harness.load().await; let timeline_id = TimelineId::generate(); @@ -2034,7 +2034,9 @@ pub(crate) mod test { use crate::walrecord::NeonWalRecord; use bytes::Bytes; - let h = crate::tenant::harness::TenantHarness::create("truncate_delta_smoke").unwrap(); + let h = crate::tenant::harness::TenantHarness::create("truncate_delta_smoke") + .await + .unwrap(); let (tenant, ctx) = h.load().await; let ctx = &ctx; let timeline = tenant @@ -2312,7 +2314,7 @@ pub(crate) mod test { #[tokio::test] async fn delta_layer_iterator() { - let harness = TenantHarness::create("delta_layer_iterator").unwrap(); + let harness = TenantHarness::create("delta_layer_iterator").await.unwrap(); let (tenant, ctx) = harness.load().await; let tline = tenant diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index 45b47bb62b0c..19e4e9e2e9ca 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -1111,6 +1111,7 @@ mod test { ShardIdentity::unsharded(), get_next_gen(), ) + .await .unwrap(); let (tenant, ctx) = harness.load().await; let timeline = tenant @@ -1177,6 +1178,7 @@ mod test { // But here, all we care about is that the gen number is unique. get_next_gen(), ) + .await .unwrap(); let (tenant, ctx) = harness.load().await; let timeline = tenant @@ -1308,7 +1310,7 @@ mod test { #[tokio::test] async fn image_layer_iterator() { - let harness = TenantHarness::create("image_layer_iterator").unwrap(); + let harness = TenantHarness::create("image_layer_iterator").await.unwrap(); let (tenant, ctx) = harness.load().await; let tline = tenant diff --git a/pageserver/src/tenant/storage_layer/layer/tests.rs b/pageserver/src/tenant/storage_layer/layer/tests.rs index 3a7aca7a6cc4..8a3737f8a760 100644 --- a/pageserver/src/tenant/storage_layer/layer/tests.rs +++ b/pageserver/src/tenant/storage_layer/layer/tests.rs @@ -22,7 +22,7 @@ const FOREVER: std::time::Duration = std::time::Duration::from_secs(ADVANCE.as_s async fn smoke_test() { let handle = tokio::runtime::Handle::current(); - let h = TenantHarness::create("smoke_test").unwrap(); + let h = TenantHarness::create("smoke_test").await.unwrap(); let span = h.span(); let download_span = span.in_scope(|| tracing::info_span!("downloading", timeline_id = 1)); let (tenant, _) = h.load().await; @@ -176,7 +176,9 @@ async fn evict_and_wait_on_wanted_deleted() { // this is the runtime on which Layer spawns the blocking tasks on let handle = tokio::runtime::Handle::current(); - let h = TenantHarness::create("evict_and_wait_on_wanted_deleted").unwrap(); + let h = TenantHarness::create("evict_and_wait_on_wanted_deleted") + .await + .unwrap(); utils::logging::replace_panic_hook_with_tracing_panic_hook().forget(); let (tenant, ctx) = h.load().await; @@ -258,7 +260,9 @@ fn read_wins_pending_eviction() { rt.block_on(async move { // this is the runtime on which Layer spawns the blocking tasks on let handle = tokio::runtime::Handle::current(); - let h = TenantHarness::create("read_wins_pending_eviction").unwrap(); + let h = TenantHarness::create("read_wins_pending_eviction") + .await + .unwrap(); let (tenant, ctx) = h.load().await; let span = h.span(); let download_span = span.in_scope(|| tracing::info_span!("downloading", timeline_id = 1)); @@ -390,7 +394,7 @@ fn multiple_pending_evictions_scenario(name: &'static str, in_order: bool) { rt.block_on(async move { // this is the runtime on which Layer spawns the blocking tasks on let handle = tokio::runtime::Handle::current(); - let h = TenantHarness::create(name).unwrap(); + let h = TenantHarness::create(name).await.unwrap(); let (tenant, ctx) = h.load().await; let span = h.span(); let download_span = span.in_scope(|| tracing::info_span!("downloading", timeline_id = 1)); @@ -559,8 +563,9 @@ fn multiple_pending_evictions_scenario(name: &'static str, in_order: bool) { #[tokio::test(start_paused = true)] async fn cancelled_get_or_maybe_download_does_not_cancel_eviction() { let handle = tokio::runtime::Handle::current(); - let h = - TenantHarness::create("cancelled_get_or_maybe_download_does_not_cancel_eviction").unwrap(); + let h = TenantHarness::create("cancelled_get_or_maybe_download_does_not_cancel_eviction") + .await + .unwrap(); let (tenant, ctx) = h.load().await; let timeline = tenant @@ -636,7 +641,9 @@ async fn cancelled_get_or_maybe_download_does_not_cancel_eviction() { #[tokio::test(start_paused = true)] async fn evict_and_wait_does_not_wait_for_download() { // let handle = tokio::runtime::Handle::current(); - let h = TenantHarness::create("evict_and_wait_does_not_wait_for_download").unwrap(); + let h = TenantHarness::create("evict_and_wait_does_not_wait_for_download") + .await + .unwrap(); let (tenant, ctx) = h.load().await; let span = h.span(); let download_span = span.in_scope(|| tracing::info_span!("downloading", timeline_id = 1)); @@ -733,7 +740,9 @@ async fn eviction_cancellation_on_drop() { // this is the runtime on which Layer spawns the blocking tasks on let handle = tokio::runtime::Handle::current(); - let h = TenantHarness::create("eviction_cancellation_on_drop").unwrap(); + let h = TenantHarness::create("eviction_cancellation_on_drop") + .await + .unwrap(); utils::logging::replace_panic_hook_with_tracing_panic_hook().forget(); let (tenant, ctx) = h.load().await; diff --git a/pageserver/src/tenant/storage_layer/merge_iterator.rs b/pageserver/src/tenant/storage_layer/merge_iterator.rs index 6f59b2fd7765..eb4a1f28a11c 100644 --- a/pageserver/src/tenant/storage_layer/merge_iterator.rs +++ b/pageserver/src/tenant/storage_layer/merge_iterator.rs @@ -293,7 +293,9 @@ mod tests { use crate::repository::Value; use bytes::Bytes; - let harness = TenantHarness::create("merge_iterator_merge_in_between").unwrap(); + let harness = TenantHarness::create("merge_iterator_merge_in_between") + .await + .unwrap(); let (tenant, ctx) = harness.load().await; let tline = tenant @@ -356,7 +358,9 @@ mod tests { use crate::repository::Value; use bytes::Bytes; - let harness = TenantHarness::create("merge_iterator_delta_merge").unwrap(); + let harness = TenantHarness::create("merge_iterator_delta_merge") + .await + .unwrap(); let (tenant, ctx) = harness.load().await; let tline = tenant @@ -430,7 +434,9 @@ mod tests { use crate::repository::Value; use bytes::Bytes; - let harness = TenantHarness::create("merge_iterator_delta_image_mixed_merge").unwrap(); + let harness = TenantHarness::create("merge_iterator_delta_image_mixed_merge") + .await + .unwrap(); let (tenant, ctx) = harness.load().await; let tline = tenant diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 3d3d3ac34de1..19b13969811c 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -6046,8 +6046,9 @@ mod tests { #[tokio::test] async fn two_layer_eviction_attempts_at_the_same_time() { - let harness = - TenantHarness::create("two_layer_eviction_attempts_at_the_same_time").unwrap(); + let harness = TenantHarness::create("two_layer_eviction_attempts_at_the_same_time") + .await + .unwrap(); let (tenant, ctx) = harness.load().await; let timeline = tenant diff --git a/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs b/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs index 1d2ffec08fb5..de50f217d80e 100644 --- a/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs +++ b/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs @@ -1118,7 +1118,7 @@ mod tests { #[tokio::test] async fn no_connection_no_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::create("no_connection_no_candidate")?; + let harness = TenantHarness::create("no_connection_no_candidate").await?; let mut state = dummy_state(&harness).await; let now = Utc::now().naive_utc(); @@ -1151,7 +1151,7 @@ mod tests { #[tokio::test] async fn connection_no_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::create("connection_no_candidate")?; + let harness = TenantHarness::create("connection_no_candidate").await?; let mut state = dummy_state(&harness).await; let now = Utc::now().naive_utc(); @@ -1216,7 +1216,7 @@ mod tests { #[tokio::test] async fn no_connection_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::create("no_connection_candidate")?; + let harness = TenantHarness::create("no_connection_candidate").await?; let mut state = dummy_state(&harness).await; let now = Utc::now().naive_utc(); @@ -1279,7 +1279,7 @@ mod tests { #[tokio::test] async fn candidate_with_many_connection_failures() -> anyhow::Result<()> { - let harness = TenantHarness::create("candidate_with_many_connection_failures")?; + let harness = TenantHarness::create("candidate_with_many_connection_failures").await?; let mut state = dummy_state(&harness).await; let now = Utc::now().naive_utc(); @@ -1319,7 +1319,7 @@ mod tests { #[tokio::test] async fn lsn_wal_over_threshold_current_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::create("lsn_wal_over_threshcurrent_candidate")?; + let harness = TenantHarness::create("lsn_wal_over_threshcurrent_candidate").await?; let mut state = dummy_state(&harness).await; let current_lsn = Lsn(100_000).align(); let now = Utc::now().naive_utc(); @@ -1385,7 +1385,8 @@ mod tests { #[tokio::test] async fn timeout_connection_threshold_current_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::create("timeout_connection_threshold_current_candidate")?; + let harness = + TenantHarness::create("timeout_connection_threshold_current_candidate").await?; let mut state = dummy_state(&harness).await; let current_lsn = Lsn(100_000).align(); let now = Utc::now().naive_utc(); @@ -1448,7 +1449,7 @@ mod tests { #[tokio::test] async fn timeout_wal_over_threshold_current_candidate() -> anyhow::Result<()> { - let harness = TenantHarness::create("timeout_wal_over_threshold_current_candidate")?; + let harness = TenantHarness::create("timeout_wal_over_threshold_current_candidate").await?; let mut state = dummy_state(&harness).await; let current_lsn = Lsn(100_000).align(); let new_lsn = Lsn(100_100).align(); @@ -1550,7 +1551,7 @@ mod tests { // and pageserver should prefer to connect to it. let test_az = Some("test_az".to_owned()); - let harness = TenantHarness::create("switch_to_same_availability_zone")?; + let harness = TenantHarness::create("switch_to_same_availability_zone").await?; let mut state = dummy_state(&harness).await; state.conf.availability_zone.clone_from(&test_az); let current_lsn = Lsn(100_000).align(); diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index 07c90385e654..dff3a8f52da4 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -1754,7 +1754,7 @@ mod tests { #[tokio::test] async fn test_relsize() -> Result<()> { - let (tenant, ctx) = TenantHarness::create("test_relsize")?.load().await; + let (tenant, ctx) = TenantHarness::create("test_relsize").await?.load().await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(8), DEFAULT_PG_VERSION, &ctx) .await?; @@ -1975,7 +1975,10 @@ mod tests { // and then created it again within the same layer. #[tokio::test] async fn test_drop_extend() -> Result<()> { - let (tenant, ctx) = TenantHarness::create("test_drop_extend")?.load().await; + let (tenant, ctx) = TenantHarness::create("test_drop_extend") + .await? + .load() + .await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(8), DEFAULT_PG_VERSION, &ctx) .await?; @@ -2046,7 +2049,10 @@ mod tests { // and then extended it again within the same layer. #[tokio::test] async fn test_truncate_extend() -> Result<()> { - let (tenant, ctx) = TenantHarness::create("test_truncate_extend")?.load().await; + let (tenant, ctx) = TenantHarness::create("test_truncate_extend") + .await? + .load() + .await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(8), DEFAULT_PG_VERSION, &ctx) .await?; @@ -2188,7 +2194,7 @@ mod tests { /// split into multiple 1 GB segments in Postgres. #[tokio::test] async fn test_large_rel() -> Result<()> { - let (tenant, ctx) = TenantHarness::create("test_large_rel")?.load().await; + let (tenant, ctx) = TenantHarness::create("test_large_rel").await?.load().await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(8), DEFAULT_PG_VERSION, &ctx) .await?; @@ -2296,7 +2302,7 @@ mod tests { let startpoint = Lsn::from_hex("14AEC08").unwrap(); let _endpoint = Lsn::from_hex("1FFFF98").unwrap(); - let harness = TenantHarness::create("test_ingest_real_wal").unwrap(); + let harness = TenantHarness::create("test_ingest_real_wal").await.unwrap(); let (tenant, ctx) = harness.load().await; let remote_initdb_path = diff --git a/proxy/src/context/parquet.rs b/proxy/src/context/parquet.rs index cfc1f8e89e3f..543a45827400 100644 --- a/proxy/src/context/parquet.rs +++ b/proxy/src/context/parquet.rs @@ -181,8 +181,9 @@ pub async fn worker( let rx = futures::stream::poll_fn(move |cx| rx.poll_recv(cx)); let rx = rx.map(RequestData::from); - let storage = - GenericRemoteStorage::from_config(&remote_storage_config).context("remote storage init")?; + let storage = GenericRemoteStorage::from_config(&remote_storage_config) + .await + .context("remote storage init")?; let properties = WriterProperties::builder() .set_data_page_size_limit(config.parquet_upload_page_size) @@ -217,6 +218,7 @@ pub async fn worker( let storage_disconnect = GenericRemoteStorage::from_config(&disconnect_events_storage_config) + .await .context("remote storage for disconnect events init")?; let parquet_config_disconnect = parquet_config.clone(); tokio::try_join!( @@ -545,7 +547,9 @@ mod tests { }, timeout: std::time::Duration::from_secs(120), }; - let storage = GenericRemoteStorage::from_config(&remote_storage_config).unwrap(); + let storage = GenericRemoteStorage::from_config(&remote_storage_config) + .await + .unwrap(); worker_inner(storage, rx, config).await.unwrap(); diff --git a/proxy/src/usage_metrics.rs b/proxy/src/usage_metrics.rs index 56ed2145dc25..a8735fe0bbda 100644 --- a/proxy/src/usage_metrics.rs +++ b/proxy/src/usage_metrics.rs @@ -357,11 +357,15 @@ pub async fn task_backup( info!("metrics backup has shut down"); } // Even if the remote storage is not configured, we still want to clear the metrics. - let storage = backup_config - .remote_storage_config - .as_ref() - .map(|config| GenericRemoteStorage::from_config(config).context("remote storage init")) - .transpose()?; + let storage = if let Some(config) = backup_config.remote_storage_config.as_ref() { + Some( + GenericRemoteStorage::from_config(config) + .await + .context("remote storage init")?, + ) + } else { + None + }; let mut ticker = tokio::time::interval(backup_config.interval); let mut prev = Utc::now(); let hostname = hostname::get()?.as_os_str().to_string_lossy().into_owned(); diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index 9eb6546d6bae..2365fd05871f 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -418,7 +418,7 @@ async fn start_safekeeper(conf: SafeKeeperConf) -> Result<()> { let timeline_collector = safekeeper::metrics::TimelineCollector::new(); metrics::register_internal(Box::new(timeline_collector))?; - wal_backup::init_remote_storage(&conf); + wal_backup::init_remote_storage(&conf).await; // Keep handles to main tasks to die if any of them disappears. let mut tasks_handles: FuturesUnordered> = diff --git a/safekeeper/src/wal_backup.rs b/safekeeper/src/wal_backup.rs index 5a590689c374..7ecee178f3b4 100644 --- a/safekeeper/src/wal_backup.rs +++ b/safekeeper/src/wal_backup.rs @@ -22,7 +22,7 @@ use tokio::fs::File; use tokio::select; use tokio::sync::mpsc::{self, Receiver, Sender}; -use tokio::sync::watch; +use tokio::sync::{watch, OnceCell}; use tokio::time::sleep; use tracing::*; @@ -33,8 +33,6 @@ use crate::timeline::{PeerInfo, WalResidentTimeline}; use crate::timeline_manager::{Manager, StateSnapshot}; use crate::{SafeKeeperConf, WAL_BACKUP_RUNTIME}; -use once_cell::sync::OnceCell; - const UPLOAD_FAILURE_RETRY_MIN_MS: u64 = 10; const UPLOAD_FAILURE_RETRY_MAX_MS: u64 = 5000; @@ -167,7 +165,7 @@ fn determine_offloader( } } -static REMOTE_STORAGE: OnceCell> = OnceCell::new(); +static REMOTE_STORAGE: OnceCell> = OnceCell::const_new(); // Storage must be configured and initialized when this is called. fn get_configured_remote_storage() -> &'static GenericRemoteStorage { @@ -178,14 +176,22 @@ fn get_configured_remote_storage() -> &'static GenericRemoteStorage { .unwrap() } -pub fn init_remote_storage(conf: &SafeKeeperConf) { +pub async fn init_remote_storage(conf: &SafeKeeperConf) { // TODO: refactor REMOTE_STORAGE to avoid using global variables, and provide // dependencies to all tasks instead. - REMOTE_STORAGE.get_or_init(|| { - conf.remote_storage - .as_ref() - .map(|c| GenericRemoteStorage::from_config(c).expect("failed to create remote storage")) - }); + REMOTE_STORAGE + .get_or_init(|| async { + if let Some(conf) = conf.remote_storage.as_ref() { + Some( + GenericRemoteStorage::from_config(conf) + .await + .expect("failed to create remote storage"), + ) + } else { + None + } + }) + .await; } struct WalBackupTask {