From 2c5959358246e2a7d7c561d974f9e1b07307135d Mon Sep 17 00:00:00 2001 From: "vitess-bot[bot]" <108069721+vitess-bot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 19:32:16 +0000 Subject: [PATCH] prevent vtctld from creating tons of S3 connections (#15296) Signed-off-by: Renan Rangel Signed-off-by: 'Renan Rangel' --- go/vt/mysqlctl/s3backupstorage/s3.go | 25 +++++++++++++++-------- go/vt/mysqlctl/s3backupstorage/s3_test.go | 10 +++++++++ 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/go/vt/mysqlctl/s3backupstorage/s3.go b/go/vt/mysqlctl/s3backupstorage/s3.go index ef3bfc37b31..6f5f3737719 100644 --- a/go/vt/mysqlctl/s3backupstorage/s3.go +++ b/go/vt/mysqlctl/s3backupstorage/s3.go @@ -281,10 +281,21 @@ func (s3ServerSideEncryption *S3ServerSideEncryption) reset() { // S3BackupStorage implements the backupstorage.BackupStorage interface. type S3BackupStorage struct { - _client *s3.S3 - mu sync.Mutex - s3SSE S3ServerSideEncryption - params backupstorage.Params + _client *s3.S3 + mu sync.Mutex + s3SSE S3ServerSideEncryption + params backupstorage.Params + transport *http.Transport +} + +func newS3BackupStorage() *S3BackupStorage { + // This initialises a new transport based off http.DefaultTransport the first time and returns the same + // transport on subsequent calls so connections can be reused as part of the same transport. + tlsClientConf := &tls.Config{InsecureSkipVerify: tlsSkipVerifyCert} + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = tlsClientConf + + return &S3BackupStorage{params: backupstorage.NoParams(), transport: transport} } // ListBackups is part of the backupstorage.BackupStorage interface. @@ -445,9 +456,7 @@ func (bs *S3BackupStorage) client() (*s3.S3, error) { if bs._client == nil { logLevel := getLogLevel() - tlsClientConf := &tls.Config{InsecureSkipVerify: tlsSkipVerifyCert} - httpTransport := &http.Transport{TLSClientConfig: tlsClientConf} - httpClient := &http.Client{Transport: httpTransport} + httpClient := &http.Client{Transport: bs.transport} session, err := session.NewSession() if err != nil { @@ -497,7 +506,7 @@ func objName(parts ...string) *string { } func init() { - backupstorage.BackupStorageMap["s3"] = &S3BackupStorage{params: backupstorage.NoParams()} + backupstorage.BackupStorageMap["s3"] = newS3BackupStorage() logNameMap = logNameToLogLevel{ "LogOff": aws.LogOff, diff --git a/go/vt/mysqlctl/s3backupstorage/s3_test.go b/go/vt/mysqlctl/s3backupstorage/s3_test.go index a10432b78c2..ed97555e64b 100644 --- a/go/vt/mysqlctl/s3backupstorage/s3_test.go +++ b/go/vt/mysqlctl/s3backupstorage/s3_test.go @@ -276,3 +276,13 @@ func TestSSECustomerFileBase64Key(t *testing.T) { assert.Nil(t, sseData.customerKey, "customerKey expected to be nil") assert.Nil(t, sseData.customerMd5, "customerMd5 expected to be nil") } + +func TestNewS3Transport(t *testing.T) { + s3 := newS3BackupStorage() + + // checking some of the values are present in the returned transport and match the http.DefaultTransport. + assert.Equal(t, http.DefaultTransport.(*http.Transport).IdleConnTimeout, s3.transport.IdleConnTimeout) + assert.Equal(t, http.DefaultTransport.(*http.Transport).MaxIdleConns, s3.transport.MaxIdleConns) + assert.NotNil(t, s3.transport.DialContext) + assert.NotNil(t, s3.transport.Proxy) +}