Skip to content

Commit

Permalink
[release-18.0] Flaky test fixes (#16940) (#16957)
Browse files Browse the repository at this point in the history 
Signed-off-by: Manan Gupta <[email protected]>
Co-authored-by: Manan Gupta <[email protected]>
  • Loading branch information
@vitess-bot @GuptaManan100
vitess-bot[bot] and GuptaManan100 authored Oct 15, 2024
1 parent a0b1861 commit f7c229c
Show file tree
Hide file tree
Showing 9 changed files with 106 additions and 79 deletions.
20 changes: 19 additions & 1 deletion go/mysql/server_flaky_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1429,7 +1429,7 @@ func TestListenerShutdown(t *testing.T) {

l.Shutdown()

assert.EqualValues(t, 1, connRefuse.Get(), "connRefuse")
waitForConnRefuse(t, 1)

err = conn.Ping()
require.EqualError(t, err, "Server shutdown in progress (errno 1053) (sqlstate 08S01)")
Expand All @@ -1441,6 +1441,24 @@ func TestListenerShutdown(t *testing.T) {
require.Equal(t, "Server shutdown in progress", sqlErr.Message)
}

func waitForConnRefuse(t *testing.T, valWanted int64) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
tick := time.NewTicker(100 * time.Millisecond)
defer tick.Stop()

for {
select {
case <-ctx.Done():
require.FailNow(t, "connRefuse did not reach %v", valWanted)
case <-tick.C:
if connRefuse.Get() == valWanted {
return
}
}
}
}

func TestParseConnAttrs(t *testing.T) {
expected := map[string]string{
"_client_version": "8.0.11",
Expand Down
25 changes: 17 additions & 8 deletions go/vt/mysqlctl/fakemysqldaemon.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,13 +277,6 @@ func (fmd *FakeMysqlDaemon) GetServerUUID(ctx context.Context) (string, error) {
return "000000", nil
}

// CurrentPrimaryPositionLocked is thread-safe
func (fmd *FakeMysqlDaemon) CurrentPrimaryPositionLocked(pos replication.Position) {
fmd.mu.Lock()
defer fmd.mu.Unlock()
fmd.CurrentPrimaryPosition = pos
}

// ReplicationStatus is part of the MysqlDaemon interface
func (fmd *FakeMysqlDaemon) ReplicationStatus() (replication.ReplicationStatus, error) {
if fmd.ReplicationStatusError != nil {
Expand All @@ -307,6 +300,8 @@ func (fmd *FakeMysqlDaemon) ReplicationStatus() (replication.ReplicationStatus,

// PrimaryStatus is part of the MysqlDaemon interface
func (fmd *FakeMysqlDaemon) PrimaryStatus(ctx context.Context) (replication.PrimaryStatus, error) {
fmd.mu.Lock()
defer fmd.mu.Unlock()
if fmd.PrimaryStatusError != nil {
return replication.PrimaryStatus{}, fmd.PrimaryStatusError
}
Expand Down Expand Up @@ -372,7 +367,21 @@ func (fmd *FakeMysqlDaemon) GetPreviousGTIDs(ctx context.Context, binlog string)

// PrimaryPosition is part of the MysqlDaemon interface
func (fmd *FakeMysqlDaemon) PrimaryPosition() (replication.Position, error) {
return fmd.CurrentPrimaryPosition, nil
return fmd.GetPrimaryPositionLocked(), nil
}

// GetPrimaryPositionLocked gets the primary position while holding the lock.
func (fmd *FakeMysqlDaemon) GetPrimaryPositionLocked() replication.Position {
fmd.mu.Lock()
defer fmd.mu.Unlock()
return fmd.CurrentPrimaryPosition
}

// SetPrimaryPositionLocked is thread-safe.
func (fmd *FakeMysqlDaemon) SetPrimaryPositionLocked(pos replication.Position) {
fmd.mu.Lock()
defer fmd.mu.Unlock()
fmd.CurrentPrimaryPosition = pos
}

// IsReadOnly is part of the MysqlDaemon interface
Expand Down
2 changes: 1 addition & 1 deletion go/vt/vtctl/grpcvtctldserver/server_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12030,7 +12030,7 @@ func TestTabletExternallyReparented(t *testing.T) {
defer func() {
topofactory.SetError(nil)

ctx, cancel := context.WithTimeout(ctx, time.Millisecond*10)
ctx, cancel := context.WithTimeout(ctx, time.Second*10)
defer cancel()

resp, err := vtctld.GetTablets(ctx, &vtctldatapb.GetTabletsRequest{})
Expand Down
4 changes: 2 additions & 2 deletions go/vt/vttablet/tabletmanager/framework_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,8 @@ func newTestEnv(t *testing.T, ctx context.Context, sourceKeyspace string, source
tmclienttest.SetProtocol(fmt.Sprintf("go.vt.vttablet.tabletmanager.framework_test_%s", t.Name()), tenv.protoName)

tenv.mysqld = mysqlctl.NewFakeMysqlDaemon(fakesqldb.New(t))
var err error
tenv.mysqld.CurrentPrimaryPosition, err = replication.ParsePosition(gtidFlavor, gtidPosition)
curPosition, err := replication.ParsePosition(gtidFlavor, gtidPosition)
tenv.mysqld.SetPrimaryPositionLocked(curPosition)
require.NoError(t, err)

return tenv
Expand Down
66 changes: 33 additions & 33 deletions go/vt/wrangler/testlib/backup_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,15 +148,15 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error {
primary := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_PRIMARY, db)
primary.FakeMysqlDaemon.ReadOnly = false
primary.FakeMysqlDaemon.Replicating = false
primary.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
primary.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
Server: 123,
Sequence: 457,
},
},
}
})

// start primary so that replica can fetch primary position from it
primary.StartActionLoop(t, wr)
Expand All @@ -168,15 +168,15 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error {
sourceTablet.FakeMysqlDaemon.ReadOnly = true
sourceTablet.FakeMysqlDaemon.Replicating = true
sourceTablet.FakeMysqlDaemon.SetReplicationSourceInputs = []string{fmt.Sprintf("%s:%d", primary.Tablet.MysqlHostname, primary.Tablet.MysqlPort)}
sourceTablet.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
sourceTablet.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
Server: 123,
Sequence: 457,
},
},
}
})
sourceTablet.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
// These 3 statements come from tablet startup
"STOP SLAVE",
Expand Down Expand Up @@ -221,15 +221,15 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error {
destTablet := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, db)
destTablet.FakeMysqlDaemon.ReadOnly = true
destTablet.FakeMysqlDaemon.Replicating = true
destTablet.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
destTablet.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
Server: 123,
Sequence: 457,
},
},
}
})
destTablet.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
// These 3 statements come from tablet startup
"STOP SLAVE",
Expand All @@ -247,7 +247,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error {
"RESET MASTER": {},
"SET GLOBAL gtid_purged": {},
}
destTablet.FakeMysqlDaemon.SetReplicationPositionPos = sourceTablet.FakeMysqlDaemon.CurrentPrimaryPosition
destTablet.FakeMysqlDaemon.SetReplicationPositionPos = sourceTablet.FakeMysqlDaemon.GetPrimaryPositionLocked()
destTablet.FakeMysqlDaemon.SetReplicationSourceInputs = append(destTablet.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(primary.Tablet))

destTablet.StartActionLoop(t, wr)
Expand Down Expand Up @@ -299,7 +299,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error {
"START SLAVE",
}

primary.FakeMysqlDaemon.SetReplicationPositionPos = primary.FakeMysqlDaemon.CurrentPrimaryPosition
primary.FakeMysqlDaemon.SetReplicationPositionPos = primary.FakeMysqlDaemon.GetPrimaryPositionLocked()

// restore primary from latest backup
require.NoError(t, primary.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* restoreFromBackupTs */, time.Time{} /* restoreToTimestamp */, ""),
Expand Down Expand Up @@ -386,15 +386,15 @@ func TestBackupRestoreLagged(t *testing.T) {
primary := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_PRIMARY, db)
primary.FakeMysqlDaemon.ReadOnly = false
primary.FakeMysqlDaemon.Replicating = false
primary.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
primary.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
Server: 123,
Sequence: 457,
},
},
}
})

// start primary so that replica can fetch primary position from it
primary.StartActionLoop(t, wr)
Expand All @@ -405,15 +405,15 @@ func TestBackupRestoreLagged(t *testing.T) {
sourceTablet := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, db)
sourceTablet.FakeMysqlDaemon.ReadOnly = true
sourceTablet.FakeMysqlDaemon.Replicating = true
sourceTablet.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
sourceTablet.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
Server: 123,
Sequence: 456,
},
},
}
})
sourceTablet.FakeMysqlDaemon.SetReplicationSourceInputs = []string{fmt.Sprintf("%s:%d", primary.Tablet.MysqlHostname, primary.Tablet.MysqlPort)}
sourceTablet.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
// These 3 statements come from tablet startup
Expand Down Expand Up @@ -447,7 +447,7 @@ func TestBackupRestoreLagged(t *testing.T) {

timer := time.NewTicker(1 * time.Second)
<-timer.C
sourceTablet.FakeMysqlDaemon.CurrentPrimaryPositionLocked(replication.Position{
sourceTablet.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
Expand All @@ -466,7 +466,7 @@ func TestBackupRestoreLagged(t *testing.T) {
require.NoError(t, sourceTablet.FakeMysqlDaemon.CheckSuperQueryList())
assert.True(t, sourceTablet.FakeMysqlDaemon.Replicating)
assert.True(t, sourceTablet.FakeMysqlDaemon.Running)
assert.Equal(t, primary.FakeMysqlDaemon.CurrentPrimaryPosition, sourceTablet.FakeMysqlDaemon.CurrentPrimaryPosition)
assert.Equal(t, primary.FakeMysqlDaemon.GetPrimaryPositionLocked(), sourceTablet.FakeMysqlDaemon.GetPrimaryPositionLocked())
case <-timer2.C:
require.FailNow(t, "Backup timed out")
}
Expand All @@ -475,15 +475,15 @@ func TestBackupRestoreLagged(t *testing.T) {
destTablet := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, db)
destTablet.FakeMysqlDaemon.ReadOnly = true
destTablet.FakeMysqlDaemon.Replicating = true
destTablet.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
destTablet.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
Server: 123,
Sequence: 456,
},
},
}
})
destTablet.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
// These 3 statements come from tablet startup
"STOP SLAVE",
Expand All @@ -501,7 +501,7 @@ func TestBackupRestoreLagged(t *testing.T) {
"RESET MASTER": {},
"SET GLOBAL gtid_purged": {},
}
destTablet.FakeMysqlDaemon.SetReplicationPositionPos = destTablet.FakeMysqlDaemon.CurrentPrimaryPosition
destTablet.FakeMysqlDaemon.SetReplicationPositionPos = destTablet.FakeMysqlDaemon.GetPrimaryPositionLocked()
destTablet.FakeMysqlDaemon.SetReplicationSourceInputs = append(destTablet.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(primary.Tablet))

destTablet.StartActionLoop(t, wr)
Expand All @@ -524,7 +524,7 @@ func TestBackupRestoreLagged(t *testing.T) {

timer = time.NewTicker(1 * time.Second)
<-timer.C
destTablet.FakeMysqlDaemon.CurrentPrimaryPositionLocked(replication.Position{
destTablet.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
Expand All @@ -542,7 +542,7 @@ func TestBackupRestoreLagged(t *testing.T) {
require.NoError(t, destTablet.FakeMysqlDaemon.CheckSuperQueryList(), "destTablet.FakeMysqlDaemon.CheckSuperQueryList failed")
assert.True(t, destTablet.FakeMysqlDaemon.Replicating)
assert.True(t, destTablet.FakeMysqlDaemon.Running)
assert.Equal(t, primary.FakeMysqlDaemon.CurrentPrimaryPosition, destTablet.FakeMysqlDaemon.CurrentPrimaryPosition)
assert.Equal(t, primary.FakeMysqlDaemon.GetPrimaryPositionLocked(), destTablet.FakeMysqlDaemon.GetPrimaryPositionLocked())
case <-timer2.C:
require.FailNow(t, "Restore timed out")
}
Expand Down Expand Up @@ -605,15 +605,15 @@ func TestRestoreUnreachablePrimary(t *testing.T) {
primary := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_PRIMARY, db)
primary.FakeMysqlDaemon.ReadOnly = false
primary.FakeMysqlDaemon.Replicating = false
primary.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
primary.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
Server: 123,
Sequence: 457,
},
},
}
})

// start primary so that replica can fetch primary position from it
primary.StartActionLoop(t, wr)
Expand All @@ -623,15 +623,15 @@ func TestRestoreUnreachablePrimary(t *testing.T) {
sourceTablet := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, db)
sourceTablet.FakeMysqlDaemon.ReadOnly = true
sourceTablet.FakeMysqlDaemon.Replicating = true
sourceTablet.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
sourceTablet.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
Server: 123,
Sequence: 457,
},
},
}
})
sourceTablet.FakeMysqlDaemon.SetReplicationSourceInputs = []string{fmt.Sprintf("%s:%d", primary.Tablet.MysqlHostname, primary.Tablet.MysqlPort)}
sourceTablet.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
// These 3 statements come from tablet startup
Expand Down Expand Up @@ -665,15 +665,15 @@ func TestRestoreUnreachablePrimary(t *testing.T) {
destTablet := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, db)
destTablet.FakeMysqlDaemon.ReadOnly = true
destTablet.FakeMysqlDaemon.Replicating = true
destTablet.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
destTablet.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
Server: 123,
Sequence: 457,
},
},
}
})
destTablet.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
// These 3 statements come from tablet startup
"STOP SLAVE",
Expand All @@ -691,7 +691,7 @@ func TestRestoreUnreachablePrimary(t *testing.T) {
"RESET MASTER": {},
"SET GLOBAL gtid_purged": {},
}
destTablet.FakeMysqlDaemon.SetReplicationPositionPos = sourceTablet.FakeMysqlDaemon.CurrentPrimaryPosition
destTablet.FakeMysqlDaemon.SetReplicationPositionPos = sourceTablet.FakeMysqlDaemon.GetPrimaryPositionLocked()
destTablet.FakeMysqlDaemon.SetReplicationSourceInputs = append(destTablet.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(primary.Tablet))

destTablet.StartActionLoop(t, wr)
Expand Down Expand Up @@ -780,15 +780,15 @@ func TestDisableActiveReparents(t *testing.T) {
primary := NewFakeTablet(t, wr, "cell1", 0, topodatapb.TabletType_PRIMARY, db)
primary.FakeMysqlDaemon.ReadOnly = false
primary.FakeMysqlDaemon.Replicating = false
primary.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
primary.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
Server: 123,
Sequence: 457,
},
},
}
})

// start primary so that replica can fetch primary position from it
primary.StartActionLoop(t, wr)
Expand All @@ -799,15 +799,15 @@ func TestDisableActiveReparents(t *testing.T) {
sourceTablet := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_REPLICA, db)
sourceTablet.FakeMysqlDaemon.ReadOnly = true
sourceTablet.FakeMysqlDaemon.Replicating = true
sourceTablet.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
sourceTablet.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
Server: 123,
Sequence: 457,
},
},
}
})
sourceTablet.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"STOP SLAVE",
}
Expand All @@ -832,15 +832,15 @@ func TestDisableActiveReparents(t *testing.T) {
destTablet := NewFakeTablet(t, wr, "cell1", 2, topodatapb.TabletType_REPLICA, db)
destTablet.FakeMysqlDaemon.ReadOnly = true
destTablet.FakeMysqlDaemon.Replicating = true
destTablet.FakeMysqlDaemon.CurrentPrimaryPosition = replication.Position{
destTablet.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{
GTIDSet: replication.MariadbGTIDSet{
2: replication.MariadbGTID{
Domain: 2,
Server: 123,
Sequence: 457,
},
},
}
})
destTablet.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{
"STOP SLAVE",
"RESET SLAVE ALL",
Expand All @@ -851,7 +851,7 @@ func TestDisableActiveReparents(t *testing.T) {
"RESET MASTER": {},
"SET GLOBAL gtid_purged": {},
}
destTablet.FakeMysqlDaemon.SetReplicationPositionPos = sourceTablet.FakeMysqlDaemon.CurrentPrimaryPosition
destTablet.FakeMysqlDaemon.SetReplicationPositionPos = sourceTablet.FakeMysqlDaemon.GetPrimaryPositionLocked()
destTablet.FakeMysqlDaemon.SetReplicationSourceInputs = append(destTablet.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(primary.Tablet))

destTablet.StartActionLoop(t, wr)
Expand Down
Loading

0 comments on commit f7c229c

Please sign in to comment.