Merge remote-tracking branch 'origin/main' into changeTabletTagsRPC

Signed-off-by: Tim Vaillancourt <[email protected]>

changelog/21.0/21.0.0/summary.md

@@ -21,6 +21,10 @@
     - **[Errant GTID Detection on VTTablets](#errant-gtid-vttablet)**
     - **[Automatically Replace MySQL auto_increment Clauses with Vitess Sequences](#auto-replace-mysql-autoinc-with-seq)**
     - **[vtctldclient ChangeTabletTags](#vtctldclient-changetablettags)**
+    - **[Experimental MySQL 8.4 support](#experimental-mysql-84)**
+    - **[Curreny Errant GTIDs Count Metric](#errant-gtid-metric)**
+    - **[vtctldclient ChangeTabletTags](#vtctldclient-changetablettags)**
+
 
 ## <a id="major-changes"/>Major Changes</a>
 
@@ -220,6 +224,14 @@ work automatically during the [`MoveTables`](https://vitess.io/docs/reference/vr
 [`--sharded-auto-increment-handling` flag](https://vitess.io/docs/21.0/reference/programs/vtctldclient/vtctldclient_movetables/vtctldclient_movetables_create/) instead. Please see the new
 [`MoveTables` Auto Increment Handling](https://vitess.io/docs/21.0/reference/vreplication/movetables/#auto-increment-handling) documentation for additional details.
 
+### <a id="experimental-mysql-84"/>Experimental MySQL 8.4 support
+
+We have added experimental support for MySQL 8.4. It passes the Vitess test suite, but it is otherwise not yet tested. We are looking for feedback from the community to improve this to move support out of the experimental phase in a future release.
+
+### <a id="errant-gtid-metric"/>Current Errant GTIDs Count Metric
+A new metric called `CurrentErrantGTIDCount` has been added to the `VTOrc` component. 
+This metric shows the current count of the errant GTIDs in the tablets.
+
 ### <a id="vtctldclient-changetablettags"/>`vtctldclient ChangeTabletTags` command and RPCs
 
 The `vtctldclient` command `ChangeTabletTags` was added to allow the tags of a tablet to be changed dynamically.

go/flags/endtoend/mysqlctld.txt

@@ -61,6 +61,7 @@ Flags:
       --db_tls_min_version string                                        Configures the minimal TLS version negotiated when SSL is enabled. Defaults to TLSv1.2. Options: TLSv1.0, TLSv1.1, TLSv1.2, TLSv1.3.
       --dba_idle_timeout duration                                        Idle timeout for dba connections (default 1m0s)
       --dba_pool_size int                                                Size of the connection pool for dba connections (default 20)
+      --grpc-dial-concurrency-limit int                                  Maximum concurrency of grpc dial operations. This should be less than the golang max thread limit of 10000. (default 1024)
       --grpc_auth_mode string                                            Which auth plugin implementation to use (eg: static)
       --grpc_auth_mtls_allowed_substrings string                         List of substrings of at least one of the client certificate names (separated by colon).
       --grpc_auth_static_client_creds string                             When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server.

go/flags/endtoend/vtbackup.txt

@@ -128,6 +128,7 @@ Flags:
       --file_backup_storage_root string                             Root directory for the file backup storage.
       --gcs_backup_storage_bucket string                            Google Cloud Storage bucket to use for backups.
       --gcs_backup_storage_root string                              Root prefix for all backup-related object names.
+      --grpc-dial-concurrency-limit int                             Maximum concurrency of grpc dial operations. This should be less than the golang max thread limit of 10000. (default 1024)
       --grpc_auth_static_client_creds string                        When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server.
       --grpc_compression string                                     Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy
       --grpc_enable_tracing                                         Enable gRPC tracing.

go/flags/endtoend/vtcombo.txt

@@ -167,7 +167,6 @@ Flags:
       --grpc_server_keepalive_timeout duration                           After having pinged for keepalive check, the server waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s)
       --grpc_use_effective_callerid                                      If set, and SSL is not used, will set the immediate caller id from the effective caller id's principal.
       --health_check_interval duration                                   Interval between health checks (default 20s)
-      --healthcheck-dial-concurrency int                                 Maximum concurrency of new healthcheck connections. This should be less than the golang max thread limit of 10000. (default 1024)
       --healthcheck_retry_delay duration                                 health check retry delay (default 2ms)
       --healthcheck_timeout duration                                     the health check timeout period (default 1m0s)
       --heartbeat_enable                                                 If true, vttablet records (if master) or checks (if replica) the current time of a replication heartbeat in the sidecar database's heartbeat table. The result is used to inform the serving state of the vttablet via healthchecks.

go/flags/endtoend/vtctlclient.txt

@@ -9,6 +9,7 @@ Usage of vtctlclient:
       --config-type string                                          Config file type (omit to infer config type from file extension).
       --datadog-agent-host string                                   host to send spans to. if empty, no tracing will be done
       --datadog-agent-port string                                   port to send spans to. if empty, no tracing will be done
+      --grpc-dial-concurrency-limit int                             Maximum concurrency of grpc dial operations. This should be less than the golang max thread limit of 10000. (default 1024)
       --grpc_auth_static_client_creds string                        When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server.
       --grpc_compression string                                     Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy
       --grpc_enable_tracing                                         Enable gRPC tracing.

go/flags/endtoend/vtctld.txt

@@ -57,6 +57,7 @@ Flags:
       --file_backup_storage_root string                                  Root directory for the file backup storage.
       --gcs_backup_storage_bucket string                                 Google Cloud Storage bucket to use for backups.
       --gcs_backup_storage_root string                                   Root prefix for all backup-related object names.
+      --grpc-dial-concurrency-limit int                                  Maximum concurrency of grpc dial operations. This should be less than the golang max thread limit of 10000. (default 1024)
       --grpc_auth_mode string                                            Which auth plugin implementation to use (eg: static)
       --grpc_auth_mtls_allowed_substrings string                         List of substrings of at least one of the client certificate names (separated by colon).
       --grpc_auth_static_client_creds string                             When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server.
@@ -85,7 +86,6 @@ Flags:
       --grpc_server_keepalive_enforcement_policy_permit_without_stream   gRPC server permit client keepalive pings even when there are no active streams (RPCs)
       --grpc_server_keepalive_time duration                              After a duration of this time, if the server doesn't see any activity, it pings the client to see if the transport is still alive. (default 10s)
       --grpc_server_keepalive_timeout duration                           After having pinged for keepalive check, the server waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s)
-      --healthcheck-dial-concurrency int                                 Maximum concurrency of new healthcheck connections. This should be less than the golang max thread limit of 10000. (default 1024)
   -h, --help                                                             help for vtctld
       --jaeger-agent-host string                                         host and port to send spans to. if empty, no tracing will be done
       --keep_logs duration                                               keep logs for this long (using ctime) (zero to keep forever)

go/flags/endtoend/vtgate.txt

@@ -67,6 +67,7 @@ Flags:
       --foreign_key_mode string                                          This is to provide how to handle foreign key constraint in create/alter table. Valid values are: allow, disallow (default "allow")
       --gate_query_cache_memory int                                      gate server query cache size in bytes, maximum amount of memory to be cached. vtgate analyzes every incoming query and generate a query plan, these plans are being cached in a lru cache. This config controls the capacity of the lru cache. (default 33554432)
       --gateway_initial_tablet_timeout duration                          At startup, the tabletGateway will wait up to this duration to get at least one tablet per keyspace/shard/tablet type (default 30s)
+      --grpc-dial-concurrency-limit int                                  Maximum concurrency of grpc dial operations. This should be less than the golang max thread limit of 10000. (default 1024)
       --grpc-send-session-in-streaming                                   If set, will send the session as last packet in streaming api to support transactions in streaming
       --grpc-use-effective-groups                                        If set, and SSL is not used, will set the immediate caller's security groups from the effective caller id's groups.
       --grpc-use-static-authentication-callerid                          If set, will set the immediate caller id to the username authenticated by the static auth plugin.
@@ -99,7 +100,6 @@ Flags:
       --grpc_server_keepalive_time duration                              After a duration of this time, if the server doesn't see any activity, it pings the client to see if the transport is still alive. (default 10s)
       --grpc_server_keepalive_timeout duration                           After having pinged for keepalive check, the server waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s)
       --grpc_use_effective_callerid                                      If set, and SSL is not used, will set the immediate caller id from the effective caller id's principal.
-      --healthcheck-dial-concurrency int                                 Maximum concurrency of new healthcheck connections. This should be less than the golang max thread limit of 10000. (default 1024)
       --healthcheck_retry_delay duration                                 health check retry delay (default 2ms)
       --healthcheck_timeout duration                                     the health check timeout period (default 1m0s)
   -h, --help                                                             help for vtgate

go/flags/endtoend/vtgateclienttest.txt

@@ -14,6 +14,7 @@ Flags:
       --config-persistence-min-interval duration                         minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s)
       --config-type string                                               Config file type (omit to infer config type from file extension).
       --default_tablet_type topodatapb.TabletType                        The default tablet type to set for queries, when one is not explicitly selected. (default PRIMARY)
+      --grpc-dial-concurrency-limit int                                  Maximum concurrency of grpc dial operations. This should be less than the golang max thread limit of 10000. (default 1024)
       --grpc_auth_mode string                                            Which auth plugin implementation to use (eg: static)
       --grpc_auth_mtls_allowed_substrings string                         List of substrings of at least one of the client certificate names (separated by colon).
       --grpc_auth_static_client_creds string                             When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server.

go/flags/endtoend/vtorc.txt

@@ -34,6 +34,7 @@ Flags:
       --config-type string                                          Config file type (omit to infer config type from file extension).
       --consul_auth_static_file string                              JSON File to read the topos/tokens from.
       --emit_stats                                                  If set, emit stats to push-based monitoring and stats backends
+      --grpc-dial-concurrency-limit int                             Maximum concurrency of grpc dial operations. This should be less than the golang max thread limit of 10000. (default 1024)
       --grpc_auth_static_client_creds string                        When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server.
       --grpc_compression string                                     Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy
       --grpc_enable_tracing                                         Enable gRPC tracing.

go/flags/endtoend/vttablet.txt

@@ -166,6 +166,7 @@ Flags:
       --gcs_backup_storage_bucket string                                 Google Cloud Storage bucket to use for backups.
       --gcs_backup_storage_root string                                   Root prefix for all backup-related object names.
       --gh-ost-path string                                               override default gh-ost binary full path (default "gh-ost")
+      --grpc-dial-concurrency-limit int                                  Maximum concurrency of grpc dial operations. This should be less than the golang max thread limit of 10000. (default 1024)
       --grpc_auth_mode string                                            Which auth plugin implementation to use (eg: static)
       --grpc_auth_mtls_allowed_substrings string                         List of substrings of at least one of the client certificate names (separated by colon).
       --grpc_auth_static_client_creds string                             When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server.

go/flags/endtoend/vttestserver.txt

@@ -43,6 +43,7 @@ Flags:
       --external_topo_implementation string                              the topology implementation to use for vtcombo process
       --extra_my_cnf string                                              extra files to add to the config, separated by ':'
       --foreign_key_mode string                                          This is to provide how to handle foreign key constraint in create/alter table. Valid values are: allow, disallow (default "allow")
+      --grpc-dial-concurrency-limit int                                  Maximum concurrency of grpc dial operations. This should be less than the golang max thread limit of 10000. (default 1024)
       --grpc_auth_mode string                                            Which auth plugin implementation to use (eg: static)
       --grpc_auth_mtls_allowed_substrings string                         List of substrings of at least one of the client certificate names (separated by colon).
       --grpc_auth_static_client_creds string                             When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server.

go/mysql/replication/mysql56_gtid_set.go

@@ -715,3 +715,18 @@ func Subtract(lhs, rhs string) (string, error) {
 	diffSet := lhsSet.Difference(rhsSet)
 	return diffSet.String(), nil
 }
+
+// GTIDCount returns the number of GTIDs in a GTID set.
+func GTIDCount(gtidStr string) (int64, error) {
+	gtidSet, err := ParseMysql56GTIDSet(gtidStr)
+	if err != nil {
+		return 0, err
+	}
+	var count int64
+	for _, intervals := range gtidSet {
+		for _, intvl := range intervals {
+			count = count + intvl.end - intvl.start + 1
+		}
+	}
+	return count, nil
+}

go/mysql/replication/mysql56_gtid_set_test.go

@@ -705,6 +705,52 @@ func BenchmarkMySQL56GTIDParsing(b *testing.B) {
 	}
 }
 
+func TestGTIDCount(t *testing.T) {
+	tests := []struct {
+		name      string
+		gtidStr   string
+		wantCount int64
+		wantErr   string
+	}{
+		{
+			name:      "Empty GTID String",
+			gtidStr:   "",
+			wantCount: 0,
+		}, {
+			name:      "Single GTID",
+			gtidStr:   "00010203-0405-0607-0809-0a0b0c0d0e0f:12",
+			wantCount: 1,
+		}, {
+			name:      "Single GTID Interval",
+			gtidStr:   "00010203-0405-0607-0809-0a0b0c0d0e0f:1-5",
+			wantCount: 5,
+		}, {
+			name:      "Single UUID",
+			gtidStr:   "00010203-0405-0607-0809-0a0b0c0d0e0f:1-5:11-20",
+			wantCount: 15,
+		}, {
+			name:      "Multiple UUIDs",
+			gtidStr:   "00010203-0405-0607-0809-0a0b0c0d0e0f:1-5:10-20,00010203-0405-0607-0809-0a0b0c0d0eff:1-5:50",
+			wantCount: 22,
+		}, {
+			name:    "Parsing error",
+			gtidStr: "incorrect set",
+			wantErr: "invalid MySQL 5.6 GTID set",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			count, err := GTIDCount(tt.gtidStr)
+			require.EqualValues(t, tt.wantCount, count)
+			if tt.wantErr != "" {
+				require.ErrorContains(t, err, tt.wantErr)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}
+
 func TestErrantGTIDsOnReplica(t *testing.T) {
 	tests := []struct {
 		name             string

go/mysql/sqlerror/constants.go

@@ -123,6 +123,7 @@ const (
 	ErSPNotVarArg                   = ErrorCode(1414)
 	ERRowIsReferenced2              = ErrorCode(1451)
 	ErNoReferencedRow2              = ErrorCode(1452)
+	ERInnodbIndexCorrupt            = ErrorCode(1817)
 	ERDupIndex                      = ErrorCode(1831)
 	ERInnodbReadOnly                = ErrorCode(1874)
 

go/pools/smartconnpool/waitlist.go

@@ -88,11 +88,14 @@ func (wl *waitlist[C]) expire(force bool) {
 	// or remove everything if force is true
 	for e := wl.list.Front(); e != nil; e = e.Next() {
 		if force || e.Value.ctx.Err() != nil {
-			wl.list.Remove(e)
 			expired = append(expired, e)
 			continue
 		}
 	}
+	// remove the expired waiters from the waitlist after traversing it
+	for _, e := range expired {
+		wl.list.Remove(e)
+	}
 	wl.mu.Unlock()
 
 	// once all the expired waiters have been removed from the waitlist, wake them up one by one

go/pools/smartconnpool/waitlist_test.go

@@ -0,0 +1,68 @@
+/*
+Copyright 2024 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package smartconnpool
+
+import (
+	"context"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestWaitlistExpireWithMultipleWaiters(t *testing.T) {
+	wait := waitlist[*TestConn]{}
+	wait.init()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
+	defer cancel()
+
+	waiterCount := 2
+	expireCount := atomic.Int32{}
+
+	for i := 0; i < waiterCount; i++ {
+		go func() {
+			_, err := wait.waitForConn(ctx, nil)
+			if err != nil {
+				expireCount.Add(1)
+			}
+		}()
+	}
+
+	// Wait for the context to expire
+	<-ctx.Done()
+
+	// Expire the waiters
+	wait.expire(false)
+
+	// Wait for the notified goroutines to finish
+	timeout := time.After(1 * time.Second)
+	ticker := time.NewTicker(10 * time.Millisecond)
+	defer ticker.Stop()
+	for expireCount.Load() != int32(waiterCount) {
+		select {
+		case <-timeout:
+			require.Failf(t, "Timed out waiting for all waiters to expire", "Wanted %d, got %d", waiterCount, expireCount.Load())
+		case <-ticker.C:
+			// try again
+		}
+	}
+
+	assert.Equal(t, int32(waiterCount), expireCount.Load())
+}