From 551a5f7e28c6a6060bbee9d0295732a37a9ec13a Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Tue, 3 Dec 2024 09:48:44 -0500 Subject: [PATCH] VReplication: Optimize replication on target tablets (#17166) Signed-off-by: Matt Lord --- examples/common/scripts/vtctld-up.sh | 1 + examples/common/scripts/vtgate-up.sh | 1 + examples/common/scripts/vttablet-up.sh | 1 + go/flags/endtoend/vtcombo.txt | 2 +- go/flags/endtoend/vttablet.txt | 2 +- .../onlineddl/flow/onlineddl_flow_test.go | 4 - .../onlineddl_vrepl_mini_stress_test.go | 4 - .../onlineddl_vrepl_stress_suite_test.go | 4 - go/test/endtoend/vreplication/cluster_test.go | 13 -- go/test/endtoend/vreplication/fk_test.go | 4 - go/test/endtoend/vreplication/vdiff2_test.go | 4 - .../vreplication/vdiff_helper_test.go | 7 +- .../vreplication/vreplication_test.go | 3 - go/vt/vttablet/common/flags.go | 3 +- .../vreplication/replicator_plan.go | 63 ++++---- .../vreplication/replicator_plan_test.go | 139 +++++++++++++++++- .../tabletmanager/vreplication/vdbclient.go | 2 +- proto/binlogdata.proto | 7 + 18 files changed, 184 insertions(+), 80 deletions(-) diff --git a/examples/common/scripts/vtctld-up.sh b/examples/common/scripts/vtctld-up.sh index 6902a851997..4a4b2587c4f 100755 --- a/examples/common/scripts/vtctld-up.sh +++ b/examples/common/scripts/vtctld-up.sh @@ -33,6 +33,7 @@ vtctld \ --port $vtctld_web_port \ --grpc_port $grpc_port \ --pid_file $VTDATAROOT/tmp/vtctld.pid \ + --pprof-http \ > $VTDATAROOT/tmp/vtctld.out 2>&1 & for _ in {0..300}; do diff --git a/examples/common/scripts/vtgate-up.sh b/examples/common/scripts/vtgate-up.sh index dbaaad02367..fd7860cf6ba 100755 --- a/examples/common/scripts/vtgate-up.sh +++ b/examples/common/scripts/vtgate-up.sh @@ -41,6 +41,7 @@ vtgate \ --pid_file $VTDATAROOT/tmp/vtgate.pid \ --enable_buffer \ --mysql_auth_server_impl none \ + --pprof-http \ > $VTDATAROOT/tmp/vtgate.out 2>&1 & # Block waiting for vtgate to be listening diff --git a/examples/common/scripts/vttablet-up.sh b/examples/common/scripts/vttablet-up.sh index daa40aee894..282cd0553ea 100755 --- a/examples/common/scripts/vttablet-up.sh +++ b/examples/common/scripts/vttablet-up.sh @@ -54,6 +54,7 @@ vttablet \ --service_map 'grpc-queryservice,grpc-tabletmanager,grpc-updatestream' \ --pid_file $VTDATAROOT/$tablet_dir/vttablet.pid \ --heartbeat_on_demand_duration=5s \ + --pprof-http \ > $VTDATAROOT/$tablet_dir/vttablet.out 2>&1 & # Block waiting for the tablet to be listening diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index 01a391d0cad..c10d045ecbd 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -414,7 +414,7 @@ Flags: --vreplication_copy_phase_duration duration Duration for each copy phase loop (before running the next catchup: default 1h) (default 1h0m0s) --vreplication_copy_phase_max_innodb_history_list_length int The maximum InnoDB transaction history that can exist on a vstreamer (source) before starting another round of copying rows. This helps to limit the impact on the source tablet. (default 1000000) --vreplication_copy_phase_max_mysql_replication_lag int The maximum MySQL replication lag (in seconds) that can exist on a vstreamer (source) before starting another round of copying rows. This helps to limit the impact on the source tablet. (default 43200) - --vreplication_experimental_flags int (Bitmask) of experimental features in vreplication to enable (default 3) + --vreplication_experimental_flags int (Bitmask) of experimental features in vreplication to enable (default 7) --vreplication_heartbeat_update_interval int Frequency (in seconds, default 1, max 60) at which the time_updated column of a vreplication stream when idling (default 1) --vreplication_max_time_to_retry_on_error duration stop automatically retrying when we've had consecutive failures with the same error for this long after the first occurrence --vreplication_net_read_timeout int Session value of net_read_timeout for vreplication, in seconds (default 300) diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index 8be7b620469..f79db05f327 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -414,7 +414,7 @@ Flags: --vreplication_copy_phase_duration duration Duration for each copy phase loop (before running the next catchup: default 1h) (default 1h0m0s) --vreplication_copy_phase_max_innodb_history_list_length int The maximum InnoDB transaction history that can exist on a vstreamer (source) before starting another round of copying rows. This helps to limit the impact on the source tablet. (default 1000000) --vreplication_copy_phase_max_mysql_replication_lag int The maximum MySQL replication lag (in seconds) that can exist on a vstreamer (source) before starting another round of copying rows. This helps to limit the impact on the source tablet. (default 43200) - --vreplication_experimental_flags int (Bitmask) of experimental features in vreplication to enable (default 3) + --vreplication_experimental_flags int (Bitmask) of experimental features in vreplication to enable (default 7) --vreplication_heartbeat_update_interval int Frequency (in seconds, default 1, max 60) at which the time_updated column of a vreplication stream when idling (default 1) --vreplication_max_time_to_retry_on_error duration stop automatically retrying when we've had consecutive failures with the same error for this long after the first occurrence --vreplication_net_read_timeout int Session value of net_read_timeout for vreplication, in seconds (default 300) diff --git a/go/test/endtoend/onlineddl/flow/onlineddl_flow_test.go b/go/test/endtoend/onlineddl/flow/onlineddl_flow_test.go index c442c042f8a..035789e4b87 100644 --- a/go/test/endtoend/onlineddl/flow/onlineddl_flow_test.go +++ b/go/test/endtoend/onlineddl/flow/onlineddl_flow_test.go @@ -63,7 +63,6 @@ import ( "vitess.io/vitess/go/test/endtoend/throttler" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/schema" - vttablet "vitess.io/vitess/go/vt/vttablet/common" throttlebase "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base" "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/throttlerapp" ) @@ -145,9 +144,6 @@ func TestMain(m *testing.M) { "--heartbeat_on_demand_duration", "5s", "--migration_check_interval", "2s", "--watch_replication_stream", - // Test VPlayer batching mode. - fmt.Sprintf("--vreplication_experimental_flags=%d", - vttablet.VReplicationExperimentalFlagAllowNoBlobBinlogRowImage|vttablet.VReplicationExperimentalFlagOptimizeInserts|vttablet.VReplicationExperimentalFlagVPlayerBatching), } clusterInstance.VtGateExtraArgs = []string{ "--ddl_strategy", "online", diff --git a/go/test/endtoend/onlineddl/vrepl_stress/onlineddl_vrepl_mini_stress_test.go b/go/test/endtoend/onlineddl/vrepl_stress/onlineddl_vrepl_mini_stress_test.go index e0dd9701cf8..88c145dc40c 100644 --- a/go/test/endtoend/onlineddl/vrepl_stress/onlineddl_vrepl_mini_stress_test.go +++ b/go/test/endtoend/onlineddl/vrepl_stress/onlineddl_vrepl_mini_stress_test.go @@ -38,7 +38,6 @@ import ( "vitess.io/vitess/go/test/endtoend/throttler" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/schema" - vttablet "vitess.io/vitess/go/vt/vttablet/common" ) type WriteMetrics struct { @@ -184,9 +183,6 @@ func TestMain(m *testing.M) { "--heartbeat_on_demand_duration", "5s", "--migration_check_interval", "5s", "--watch_replication_stream", - // Test VPlayer batching mode. - fmt.Sprintf("--vreplication_experimental_flags=%d", - vttablet.VReplicationExperimentalFlagAllowNoBlobBinlogRowImage|vttablet.VReplicationExperimentalFlagOptimizeInserts|vttablet.VReplicationExperimentalFlagVPlayerBatching), } clusterInstance.VtGateExtraArgs = []string{ "--ddl_strategy", "online", diff --git a/go/test/endtoend/onlineddl/vrepl_stress_suite/onlineddl_vrepl_stress_suite_test.go b/go/test/endtoend/onlineddl/vrepl_stress_suite/onlineddl_vrepl_stress_suite_test.go index 440b921f9ba..85b3585beb4 100644 --- a/go/test/endtoend/onlineddl/vrepl_stress_suite/onlineddl_vrepl_stress_suite_test.go +++ b/go/test/endtoend/onlineddl/vrepl_stress_suite/onlineddl_vrepl_stress_suite_test.go @@ -51,7 +51,6 @@ import ( "vitess.io/vitess/go/timer" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/schema" - vttablet "vitess.io/vitess/go/vt/vttablet/common" ) type testcase struct { @@ -436,9 +435,6 @@ func TestMain(m *testing.M) { "--migration_check_interval", "5s", "--vstream_packet_size", "4096", // Keep this value small and below 10k to ensure multilple vstream iterations "--watch_replication_stream", - // Test VPlayer batching mode. - fmt.Sprintf("--vreplication_experimental_flags=%d", - vttablet.VReplicationExperimentalFlagAllowNoBlobBinlogRowImage|vttablet.VReplicationExperimentalFlagOptimizeInserts|vttablet.VReplicationExperimentalFlagVPlayerBatching), } clusterInstance.VtGateExtraArgs = []string{ "--ddl_strategy", "online", diff --git a/go/test/endtoend/vreplication/cluster_test.go b/go/test/endtoend/vreplication/cluster_test.go index 119843651bc..dc5a72e5e88 100644 --- a/go/test/endtoend/vreplication/cluster_test.go +++ b/go/test/endtoend/vreplication/cluster_test.go @@ -39,7 +39,6 @@ import ( "vitess.io/vitess/go/vt/mysqlctl" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" - vttablet "vitess.io/vitess/go/vt/vttablet/common" vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" ) @@ -101,18 +100,6 @@ func (cc *ClusterConfig) enableGTIDCompression() func() { } } -// setAllVTTabletExperimentalFlags sets all the experimental flags for vttablet and returns a function -// that can be used to reset them in a defer. -func setAllVTTabletExperimentalFlags() func() { - experimentalArgs := fmt.Sprintf("--vreplication_experimental_flags=%d", - vttablet.VReplicationExperimentalFlagAllowNoBlobBinlogRowImage|vttablet.VReplicationExperimentalFlagOptimizeInserts|vttablet.VReplicationExperimentalFlagVPlayerBatching) - oldArgs := extraVTTabletArgs - extraVTTabletArgs = append(extraVTTabletArgs, experimentalArgs) - return func() { - extraVTTabletArgs = oldArgs - } -} - // VitessCluster represents all components within the test cluster type VitessCluster struct { t *testing.T diff --git a/go/test/endtoend/vreplication/fk_test.go b/go/test/endtoend/vreplication/fk_test.go index 34881cbcd1a..f977d5a74cd 100644 --- a/go/test/endtoend/vreplication/fk_test.go +++ b/go/test/endtoend/vreplication/fk_test.go @@ -29,7 +29,6 @@ import ( "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/log" - vttablet "vitess.io/vitess/go/vt/vttablet/common" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" ) @@ -43,9 +42,6 @@ func TestFKWorkflow(t *testing.T) { extraVTTabletArgs = []string{ // Ensure that there are multiple copy phase cycles per table. "--vstream_packet_size=256", - // Test VPlayer batching mode. - fmt.Sprintf("--vreplication_experimental_flags=%d", - vttablet.VReplicationExperimentalFlagAllowNoBlobBinlogRowImage|vttablet.VReplicationExperimentalFlagOptimizeInserts|vttablet.VReplicationExperimentalFlagVPlayerBatching), } defer func() { extraVTTabletArgs = nil }() diff --git a/go/test/endtoend/vreplication/vdiff2_test.go b/go/test/endtoend/vreplication/vdiff2_test.go index aaf4cae5375..612ba00236b 100644 --- a/go/test/endtoend/vreplication/vdiff2_test.go +++ b/go/test/endtoend/vreplication/vdiff2_test.go @@ -36,7 +36,6 @@ import ( "vitess.io/vitess/go/test/endtoend/cluster" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/sqlparser" - vttablet "vitess.io/vitess/go/vt/vttablet/common" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" @@ -140,9 +139,6 @@ func TestVDiff2(t *testing.T) { extraVTTabletArgs = []string{ // This forces us to use multiple vstream packets even with small test tables. "--vstream_packet_size=1", - // Test VPlayer batching mode. - fmt.Sprintf("--vreplication_experimental_flags=%d", - vttablet.VReplicationExperimentalFlagAllowNoBlobBinlogRowImage|vttablet.VReplicationExperimentalFlagOptimizeInserts|vttablet.VReplicationExperimentalFlagVPlayerBatching), } vc = NewVitessCluster(t, &clusterOptions{cells: strings.Split(cellNames, ",")}) diff --git a/go/test/endtoend/vreplication/vdiff_helper_test.go b/go/test/endtoend/vreplication/vdiff_helper_test.go index 561edfe8b7e..fd223d78082 100644 --- a/go/test/endtoend/vreplication/vdiff_helper_test.go +++ b/go/test/endtoend/vreplication/vdiff_helper_test.go @@ -35,7 +35,7 @@ import ( ) const ( - vdiffTimeout = 120 * time.Second // We can leverage auto retry on error with this longer-than-usual timeout + vdiffTimeout = 180 * time.Second // We can leverage auto retry on error with this longer-than-usual timeout vdiffRetryTimeout = 30 * time.Second vdiffStatusCheckInterval = 5 * time.Second vdiffRetryInterval = 5 * time.Second @@ -71,7 +71,8 @@ func doVtctlclientVDiff(t *testing.T, keyspace, workflow, cells string, want *ex ksWorkflow := fmt.Sprintf("%s.%s", keyspace, workflow) t.Run(fmt.Sprintf("vtctlclient vdiff %s", ksWorkflow), func(t *testing.T) { // update-table-stats is needed in order to test progress reports. - uuid, _ := performVDiff2Action(t, true, ksWorkflow, cells, "create", "", false, "--auto-retry", "--update-table-stats") + uuid, _ := performVDiff2Action(t, true, ksWorkflow, cells, "create", "", false, "--auto-retry", + "--update-table-stats", fmt.Sprintf("--filtered_replication_wait_time=%v", vdiffTimeout/2)) info := waitForVDiff2ToComplete(t, true, ksWorkflow, cells, uuid, time.Time{}) require.NotNil(t, info) require.Equal(t, workflow, info.Workflow) @@ -164,7 +165,7 @@ func doVtctldclientVDiff(t *testing.T, keyspace, workflow, cells string, want *e ksWorkflow := fmt.Sprintf("%s.%s", keyspace, workflow) t.Run(fmt.Sprintf("vtctldclient vdiff %s", ksWorkflow), func(t *testing.T) { // update-table-stats is needed in order to test progress reports. - flags := []string{"--auto-retry", "--update-table-stats"} + flags := []string{"--auto-retry", "--update-table-stats", fmt.Sprintf("--filtered-replication-wait-time=%v", vdiffTimeout/2)} if len(extraFlags) > 0 { flags = append(flags, extraFlags...) } diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index 04a5eabc33b..d3193298a0c 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -293,7 +293,6 @@ func TestVreplicationCopyThrottling(t *testing.T) { } func TestBasicVreplicationWorkflow(t *testing.T) { - defer setAllVTTabletExperimentalFlags() sourceKsOpts["DBTypeVersion"] = "mysql-8.0" targetKsOpts["DBTypeVersion"] = "mysql-8.0" testBasicVreplicationWorkflow(t, "noblob") @@ -595,8 +594,6 @@ func TestCellAliasVreplicationWorkflow(t *testing.T) { cells := []string{"zone1", "zone2"} resetCompression := mainClusterConfig.enableGTIDCompression() defer resetCompression() - resetExperimentalFlags := setAllVTTabletExperimentalFlags() - defer resetExperimentalFlags() vc = NewVitessCluster(t, &clusterOptions{cells: cells}) defer vc.TearDown() diff --git a/go/vt/vttablet/common/flags.go b/go/vt/vttablet/common/flags.go index 3c6141d62eb..75e8e58982f 100644 --- a/go/vt/vttablet/common/flags.go +++ b/go/vt/vttablet/common/flags.go @@ -33,8 +33,7 @@ const ( ) var ( - // Default flags: currently VReplicationExperimentalFlagVPlayerBatching is not enabled by default. - vreplicationExperimentalFlags = VReplicationExperimentalFlagOptimizeInserts | VReplicationExperimentalFlagAllowNoBlobBinlogRowImage + vreplicationExperimentalFlags = VReplicationExperimentalFlagOptimizeInserts | VReplicationExperimentalFlagAllowNoBlobBinlogRowImage | VReplicationExperimentalFlagVPlayerBatching vreplicationNetReadTimeout = 300 vreplicationNetWriteTimeout = 600 vreplicationCopyPhaseDuration = 1 * time.Hour diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index 6a416cb4414..62d6166b5ca 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -618,49 +618,40 @@ func valsEqual(v1, v2 sqltypes.Value) bool { func (tp *TablePlan) appendFromRow(buf *bytes2.Buffer, row *querypb.Row) error { bindLocations := tp.BulkInsertValues.BindLocations() if len(tp.Fields) < len(bindLocations) { - return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "wrong number of fields: got %d fields for %d bind locations ", + return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "wrong number of fields: got %d fields for %d bind locations", len(tp.Fields), len(bindLocations)) } - type colInfo struct { - typ querypb.Type - length int64 - offset int64 - field *querypb.Field - } - rowInfo := make([]*colInfo, 0) - - offset := int64(0) - for i, field := range tp.Fields { // collect info required for fields to be bound - length := row.Lengths[i] - if !tp.FieldsToSkip[strings.ToLower(field.Name)] { - rowInfo = append(rowInfo, &colInfo{ - typ: field.Type, - length: length, - offset: offset, - field: field, - }) - } - if length > 0 { - offset += row.Lengths[i] + // Bind field values to locations. + var ( + offset int64 + offsetQuery int + fieldsIndex int + field *querypb.Field + ) + for i, loc := range bindLocations { + field = tp.Fields[fieldsIndex] + length := row.Lengths[fieldsIndex] + for tp.FieldsToSkip[strings.ToLower(field.Name)] { + if length > 0 { + offset += length + } + fieldsIndex++ + field = tp.Fields[fieldsIndex] + length = row.Lengths[fieldsIndex] } - } - // bind field values to locations - var offsetQuery int - for i, loc := range bindLocations { - col := rowInfo[i] buf.WriteString(tp.BulkInsertValues.Query[offsetQuery:loc.Offset]) - typ := col.typ + typ := field.Type switch typ { case querypb.Type_TUPLE: return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "unexpected Type_TUPLE for value %d", i) case querypb.Type_JSON: - if col.length < 0 { // An SQL NULL and not an actual JSON value + if length < 0 { // An SQL NULL and not an actual JSON value buf.WriteString(sqltypes.NullStr) } else { // A JSON value (which may be a JSON null literal value) - buf2 := row.Values[col.offset : col.offset+col.length] + buf2 := row.Values[offset : offset+length] vv, err := vjson.MarshalSQLValue(buf2) if err != nil { return err @@ -668,16 +659,16 @@ func (tp *TablePlan) appendFromRow(buf *bytes2.Buffer, row *querypb.Row) error { buf.WriteString(vv.RawStr()) } default: - if col.length < 0 { + if length < 0 { // -1 means a null variable; serialize it directly buf.WriteString(sqltypes.NullStr) } else { - raw := row.Values[col.offset : col.offset+col.length] + raw := row.Values[offset : offset+length] var vv sqltypes.Value - if conversion, ok := tp.ConvertCharset[col.field.Name]; ok && col.length > 0 { + if conversion, ok := tp.ConvertCharset[field.Name]; ok && length > 0 { // Non-null string value, for which we have a charset conversion instruction - out, err := tp.convertStringCharset(raw, conversion, col.field.Name) + out, err := tp.convertStringCharset(raw, conversion, field.Name) if err != nil { return err } @@ -690,6 +681,10 @@ func (tp *TablePlan) appendFromRow(buf *bytes2.Buffer, row *querypb.Row) error { } } offsetQuery = loc.Offset + loc.Length + if length > 0 { + offset += length + } + fieldsIndex++ } buf.WriteString(tp.BulkInsertValues.Query[offsetQuery:]) return nil diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go index 644b4585914..09ace916f11 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan_test.go @@ -21,17 +21,18 @@ import ( "strings" "testing" - vttablet "vitess.io/vitess/go/vt/vttablet/common" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/bytes2" "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog/binlogplayer" "vitess.io/vitess/go/vt/sqlparser" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" + querypb "vitess.io/vitess/go/vt/proto/query" + vttablet "vitess.io/vitess/go/vt/vttablet/common" ) type TestReplicatorPlan struct { @@ -829,3 +830,137 @@ func TestBuildPlayerPlanExclude(t *testing.T) { wantPlan, _ := json.Marshal(want) assert.Equal(t, string(gotPlan), string(wantPlan)) } + +func TestAppendFromRow(t *testing.T) { + testCases := []struct { + name string + tp *TablePlan + row *querypb.Row + want string + wantErr string + }{ + { + name: "simple", + tp: &TablePlan{ + BulkInsertValues: sqlparser.BuildParsedQuery("values (%a, %a, %a)", + ":c1", ":c2", ":c3", + ), + Fields: []*querypb.Field{ + {Name: "c1", Type: querypb.Type_INT32}, + {Name: "c2", Type: querypb.Type_INT32}, + {Name: "c3", Type: querypb.Type_INT32}, + }, + }, + row: sqltypes.RowToProto3( + []sqltypes.Value{ + sqltypes.NewInt64(1), + sqltypes.NewInt64(2), + sqltypes.NewInt64(3), + }, + ), + want: "values (1, 2, 3)", + }, + { + name: "too few fields", + tp: &TablePlan{ + BulkInsertValues: sqlparser.BuildParsedQuery("values (%a, %a, %a)", + ":c1", ":c2", ":c3", + ), + Fields: []*querypb.Field{ + {Name: "c1", Type: querypb.Type_INT32}, + {Name: "c2", Type: querypb.Type_INT32}, + }, + }, + wantErr: "wrong number of fields: got 2 fields for 3 bind locations", + }, + { + name: "skip half", + tp: &TablePlan{ + BulkInsertValues: sqlparser.BuildParsedQuery("values (%a, %a, %a, %a)", + ":c1", ":c2", ":c4", ":c8", + ), + Fields: []*querypb.Field{ + {Name: "c1", Type: querypb.Type_INT32}, + {Name: "c2", Type: querypb.Type_INT32}, + {Name: "c3", Type: querypb.Type_INT32}, + {Name: "c4", Type: querypb.Type_INT32}, + {Name: "c5", Type: querypb.Type_INT32}, + {Name: "c6", Type: querypb.Type_INT32}, + {Name: "c7", Type: querypb.Type_INT32}, + {Name: "c8", Type: querypb.Type_INT32}, + }, + FieldsToSkip: map[string]bool{ + "c3": true, + "c5": true, + "c6": true, + "c7": true, + }, + }, + row: sqltypes.RowToProto3( + []sqltypes.Value{ + sqltypes.NewInt64(1), + sqltypes.NewInt64(2), + sqltypes.NewInt64(3), + sqltypes.NewInt64(4), + sqltypes.NewInt64(5), + sqltypes.NewInt64(6), + sqltypes.NewInt64(7), + sqltypes.NewInt64(8), + }, + ), + want: "values (1, 2, 4, 8)", + }, + { + name: "skip all but one", + tp: &TablePlan{ + BulkInsertValues: sqlparser.BuildParsedQuery("values (%a)", + ":c4", + ), + Fields: []*querypb.Field{ + {Name: "c1", Type: querypb.Type_INT32}, + {Name: "c2", Type: querypb.Type_INT32}, + {Name: "c3", Type: querypb.Type_INT32}, + {Name: "c4", Type: querypb.Type_INT32}, + {Name: "c5", Type: querypb.Type_INT32}, + {Name: "c6", Type: querypb.Type_INT32}, + {Name: "c7", Type: querypb.Type_INT32}, + {Name: "c8", Type: querypb.Type_INT32}, + }, + FieldsToSkip: map[string]bool{ + "c1": true, + "c2": true, + "c3": true, + "c5": true, + "c6": true, + "c7": true, + "c8": true, + }, + }, + row: sqltypes.RowToProto3( + []sqltypes.Value{ + sqltypes.NewInt64(1), + sqltypes.NewInt64(2), + sqltypes.NewInt64(3), + sqltypes.NewInt64(4), + sqltypes.NewInt64(5), + sqltypes.NewInt64(6), + sqltypes.NewInt64(7), + sqltypes.NewInt64(8), + }, + ), + want: "values (4)", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + bb := &bytes2.Buffer{} + err := tc.tp.appendFromRow(bb, tc.row) + if tc.wantErr != "" { + require.EqualError(t, err, tc.wantErr) + } else { + require.NoError(t, err) + require.Equal(t, tc.want, bb.String()) + } + }) + } +} diff --git a/go/vt/vttablet/tabletmanager/vreplication/vdbclient.go b/go/vt/vttablet/tabletmanager/vreplication/vdbclient.go index b8339cdf874..8a4409db06c 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vdbclient.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vdbclient.go @@ -171,7 +171,7 @@ func (vc *vdbClient) Execute(query string) (*sqltypes.Result, error) { func (vc *vdbClient) ExecuteWithRetry(ctx context.Context, query string) (*sqltypes.Result, error) { qr, err := vc.Execute(query) for err != nil { - if sqlErr, ok := err.(*sqlerror.SQLError); ok && sqlErr.Number() == sqlerror.ERLockDeadlock || sqlErr.Number() == sqlerror.ERLockWaitTimeout { + if sqlErr, ok := err.(*sqlerror.SQLError); ok && (sqlErr.Number() == sqlerror.ERLockDeadlock || sqlErr.Number() == sqlerror.ERLockWaitTimeout) { log.Infof("retryable error: %v, waiting for %v and retrying", sqlErr, dbLockRetryDelay) if err := vc.Rollback(); err != nil { return nil, err diff --git a/proto/binlogdata.proto b/proto/binlogdata.proto index 595760dcd52..e1df792776b 100644 --- a/proto/binlogdata.proto +++ b/proto/binlogdata.proto @@ -353,6 +353,10 @@ message FieldEvent { repeated query.Field fields = 2; string keyspace = 3; string shard = 4; + + // Field numbers in the gap between shard (4) and enum_set_string_values + // (25) are NOT reserved and can be used. + // Are ENUM and SET field values already mapped to strings in the ROW // events? This allows us to transition VTGate VStream consumers from // the pre v20 behavior of having to do this mapping themselves to the @@ -362,6 +366,9 @@ message FieldEvent { // vstreams managed by the vstreamManager. bool enum_set_string_values = 25; bool is_internal_table = 26; // set for sidecardb tables + + // Add new members in the field number gap between shard (4) and + // enum_set_string_values (25). } // ShardGtid contains the GTID position for one shard.