From fb4abd5dd8bfbf3bb911727105a15b2dd01d1739 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Tue, 27 Feb 2024 21:30:55 +0530 Subject: [PATCH] Multi Target Delete Support (#15294) Signed-off-by: Harshit Gangal --- .../endtoend/vtgate/queries/dml/dml_test.go | 75 +++ go/vt/vtgate/engine/cached_size.go | 22 +- go/vt/vtgate/engine/dml_with_input.go | 41 +- go/vt/vtgate/engine/dml_with_input_test.go | 78 ++- go/vt/vtgate/evalengine/cached_size.go | 2 +- go/vt/vtgate/evalengine/expr_tuple_bvar.go | 6 - go/vt/vtgate/planbuilder/delete.go | 21 +- go/vt/vtgate/planbuilder/dml_with_input.go | 11 +- .../planbuilder/operator_transformers.go | 12 +- go/vt/vtgate/planbuilder/operators/delete.go | 135 +++-- .../planbuilder/operators/dml_planning.go | 22 + .../planbuilder/operators/dml_with_input.go | 40 +- go/vt/vtgate/planbuilder/operators/phases.go | 5 +- go/vt/vtgate/planbuilder/operators/update.go | 19 +- go/vt/vtgate/planbuilder/plan_test.go | 13 + .../planbuilder/testdata/dml_cases.json | 501 +++++++++++++++++- .../testdata/foreignkey_cases.json | 25 +- .../testdata/foreignkey_checks_on_cases.json | 8 +- .../planbuilder/testdata/show_cases.json | 2 +- .../testdata/unknown_schema_cases.json | 10 + .../testdata/unsupported_cases.json | 25 - .../planbuilder/testdata/vschemas/schema.json | 47 ++ 22 files changed, 941 insertions(+), 179 deletions(-) diff --git a/go/test/endtoend/vtgate/queries/dml/dml_test.go b/go/test/endtoend/vtgate/queries/dml/dml_test.go index 561d73f44d5..deca3f01caf 100644 --- a/go/test/endtoend/vtgate/queries/dml/dml_test.go +++ b/go/test/endtoend/vtgate/queries/dml/dml_test.go @@ -293,3 +293,78 @@ func TestDeleteWithSubquery(t *testing.T) { utils.AssertMatches(t, mcmp.VtConn, `select region_id, oid, cust_no from order_tbl order by oid`, `[[INT64(1) INT64(1) INT64(4)] [INT64(1) INT64(2) INT64(2)]]`) } + +// TestMultiTargetDelete executed multi-target delete queries +func TestMultiTargetDelete(t *testing.T) { + utils.SkipIfBinaryIsBelowVersion(t, 20, "vtgate") + + mcmp, closer := start(t) + defer closer() + + // initial rows + mcmp.Exec("insert into order_tbl(region_id, oid, cust_no) values (1,1,4), (1,2,2), (2,3,5), (2,4,55)") + mcmp.Exec("insert into oevent_tbl(oid, ename) values (1,'a'), (2,'b'), (3,'a'), (2,'c')") + + // check rows + mcmp.AssertMatches(`select region_id, oid, cust_no from order_tbl order by oid`, + `[[INT64(1) INT64(1) INT64(4)] [INT64(1) INT64(2) INT64(2)] [INT64(2) INT64(3) INT64(5)] [INT64(2) INT64(4) INT64(55)]]`) + mcmp.AssertMatches(`select oid, ename from oevent_tbl order by oid`, + `[[INT64(1) VARCHAR("a")] [INT64(2) VARCHAR("b")] [INT64(2) VARCHAR("c")] [INT64(3) VARCHAR("a")]]`) + + // multi table delete + qr := mcmp.Exec(`delete o, ev from order_tbl o join oevent_tbl ev where o.oid = ev.oid and ev.ename = 'a'`) + assert.EqualValues(t, 4, qr.RowsAffected) + + // check rows + mcmp.AssertMatches(`select region_id, oid, cust_no from order_tbl order by oid`, + `[[INT64(1) INT64(2) INT64(2)] [INT64(2) INT64(4) INT64(55)]]`) + mcmp.AssertMatches(`select oid, ename from oevent_tbl order by oid`, + `[[INT64(2) VARCHAR("b")] [INT64(2) VARCHAR("c")]]`) + + qr = mcmp.Exec(`delete o, ev from order_tbl o join oevent_tbl ev where o.cust_no = ev.oid`) + assert.EqualValues(t, 3, qr.RowsAffected) + + // check rows + mcmp.AssertMatches(`select region_id, oid, cust_no from order_tbl order by oid`, + `[[INT64(2) INT64(4) INT64(55)]]`) + mcmp.AssertMatches(`select oid, ename from oevent_tbl order by oid`, + `[]`) +} + +// TestMultiTargetDeleteMore executed multi-target delete queries with additional cases +func TestMultiTargetDeleteMore(t *testing.T) { + utils.SkipIfBinaryIsBelowVersion(t, 20, "vtgate") + + mcmp, closer := start(t) + defer closer() + + // multi table delete on empty table. + qr := mcmp.Exec(`delete o, ev from order_tbl o join oevent_tbl ev on o.oid = ev.oid`) + assert.EqualValues(t, 0, qr.RowsAffected) + + // initial rows + mcmp.Exec("insert into order_tbl(region_id, oid, cust_no) values (1,1,4), (1,2,2), (2,3,5), (2,4,55)") + mcmp.Exec("insert into oevent_tbl(oid, ename) values (1,'a'), (2,'b'), (3,'a'), (2,'c')") + + // multi table delete on non-existent data. + qr = mcmp.Exec(`delete o, ev from order_tbl o join oevent_tbl ev on o.oid = ev.oid where ev.oid = 10`) + assert.EqualValues(t, 0, qr.RowsAffected) + + // check rows + mcmp.AssertMatches(`select region_id, oid, cust_no from order_tbl order by oid`, + `[[INT64(1) INT64(1) INT64(4)] [INT64(1) INT64(2) INT64(2)] [INT64(2) INT64(3) INT64(5)] [INT64(2) INT64(4) INT64(55)]]`) + mcmp.AssertMatches(`select oid, ename from oevent_tbl order by oid`, + `[[INT64(1) VARCHAR("a")] [INT64(2) VARCHAR("b")] [INT64(2) VARCHAR("c")] [INT64(3) VARCHAR("a")]]`) + + // multi table delete with rollback + mcmp.Exec(`begin`) + qr = mcmp.Exec(`delete o, ev from order_tbl o join oevent_tbl ev on o.oid = ev.oid where o.cust_no != 4`) + assert.EqualValues(t, 5, qr.RowsAffected) + mcmp.Exec(`rollback`) + + // check rows + mcmp.AssertMatches(`select region_id, oid, cust_no from order_tbl order by oid`, + `[[INT64(1) INT64(1) INT64(4)] [INT64(1) INT64(2) INT64(2)] [INT64(2) INT64(3) INT64(5)] [INT64(2) INT64(4) INT64(55)]]`) + mcmp.AssertMatches(`select oid, ename from oevent_tbl order by oid`, + `[[INT64(1) VARCHAR("a")] [INT64(2) VARCHAR("b")] [INT64(2) VARCHAR("c")] [INT64(3) VARCHAR("a")]]`) +} diff --git a/go/vt/vtgate/engine/cached_size.go b/go/vt/vtgate/engine/cached_size.go index 781c5904044..32ecd9ff6a9 100644 --- a/go/vt/vtgate/engine/cached_size.go +++ b/go/vt/vtgate/engine/cached_size.go @@ -185,17 +185,27 @@ func (cached *DMLWithInput) CachedSize(alloc bool) int64 { if alloc { size += int64(64) } - // field DML vitess.io/vitess/go/vt/vtgate/engine.Primitive - if cc, ok := cached.DML.(cachedObject); ok { - size += cc.CachedSize(true) - } // field Input vitess.io/vitess/go/vt/vtgate/engine.Primitive if cc, ok := cached.Input.(cachedObject); ok { size += cc.CachedSize(true) } - // field OutputCols []int + // field DMLs []vitess.io/vitess/go/vt/vtgate/engine.Primitive + { + size += hack.RuntimeAllocSize(int64(cap(cached.DMLs)) * int64(16)) + for _, elem := range cached.DMLs { + if cc, ok := elem.(cachedObject); ok { + size += cc.CachedSize(true) + } + } + } + // field OutputCols [][]int { - size += hack.RuntimeAllocSize(int64(cap(cached.OutputCols)) * int64(8)) + size += hack.RuntimeAllocSize(int64(cap(cached.OutputCols)) * int64(24)) + for _, elem := range cached.OutputCols { + { + size += hack.RuntimeAllocSize(int64(cap(elem)) * int64(8)) + } + } } return size } diff --git a/go/vt/vtgate/engine/dml_with_input.go b/go/vt/vtgate/engine/dml_with_input.go index 87d7c1d9826..28b306511df 100644 --- a/go/vt/vtgate/engine/dml_with_input.go +++ b/go/vt/vtgate/engine/dml_with_input.go @@ -18,6 +18,7 @@ package engine import ( "context" + "fmt" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/vterrors" @@ -34,10 +35,10 @@ const DmlVals = "dml_vals" type DMLWithInput struct { txNeeded - DML Primitive Input Primitive - OutputCols []int + DMLs []Primitive + OutputCols [][]int } func (dml *DMLWithInput) RouteType() string { @@ -53,7 +54,7 @@ func (dml *DMLWithInput) GetTableName() string { } func (dml *DMLWithInput) Inputs() ([]Primitive, []map[string]any) { - return []Primitive{dml.Input, dml.DML}, nil + return append([]Primitive{dml.Input}, dml.DMLs...), nil } // TryExecute performs a non-streaming exec. @@ -66,15 +67,27 @@ func (dml *DMLWithInput) TryExecute(ctx context.Context, vcursor VCursor, bindVa return &sqltypes.Result{}, nil } - var bv *querypb.BindVariable - if len(dml.OutputCols) == 1 { - bv = getBVSingle(inputRes, dml.OutputCols[0]) - } else { - bv = getBVMulti(inputRes, dml.OutputCols) - } + var res *sqltypes.Result + for idx, prim := range dml.DMLs { + var bv *querypb.BindVariable + if len(dml.OutputCols[idx]) == 1 { + bv = getBVSingle(inputRes, dml.OutputCols[idx][0]) + } else { + bv = getBVMulti(inputRes, dml.OutputCols[idx]) + } - bindVars[DmlVals] = bv - return vcursor.ExecutePrimitive(ctx, dml.DML, bindVars, false) + bindVars[DmlVals] = bv + qr, err := vcursor.ExecutePrimitive(ctx, prim, bindVars, false) + if err != nil { + return nil, err + } + if res == nil { + res = qr + } else { + res.RowsAffected += qr.RowsAffected + } + } + return res, nil } func getBVSingle(res *sqltypes.Result, offset int) *querypb.BindVariable { @@ -113,8 +126,12 @@ func (dml *DMLWithInput) GetFields(context.Context, VCursor, map[string]*querypb } func (dml *DMLWithInput) description() PrimitiveDescription { + var offsets []string + for idx, offset := range dml.OutputCols { + offsets = append(offsets, fmt.Sprintf("%d:%v", idx, offset)) + } other := map[string]any{ - "Offset": dml.OutputCols, + "Offset": offsets, } return PrimitiveDescription{ OperatorType: "DMLWithInput", diff --git a/go/vt/vtgate/engine/dml_with_input_test.go b/go/vt/vtgate/engine/dml_with_input_test.go index fb75ae70f1d..e43bc2b151f 100644 --- a/go/vt/vtgate/engine/dml_with_input_test.go +++ b/go/vt/vtgate/engine/dml_with_input_test.go @@ -24,6 +24,7 @@ import ( "vitess.io/vitess/go/sqltypes" querypb "vitess.io/vitess/go/vt/proto/query" + "vitess.io/vitess/go/vt/vtgate/evalengine" "vitess.io/vitess/go/vt/vtgate/vindexes" ) @@ -34,7 +35,7 @@ func TestDeleteWithInputSingleOffset(t *testing.T) { del := &DMLWithInput{ Input: input, - DML: &Delete{ + DMLs: []Primitive{&Delete{ DML: &DML{ RoutingParameters: &RoutingParameters{ Opcode: Scatter, @@ -45,8 +46,8 @@ func TestDeleteWithInputSingleOffset(t *testing.T) { }, Query: "dummy_delete", }, - }, - OutputCols: []int{0}, + }}, + OutputCols: [][]int{{0}}, } vc := newDMLTestVCursor("-20", "20-") @@ -78,7 +79,7 @@ func TestDeleteWithInputMultiOffset(t *testing.T) { del := &DMLWithInput{ Input: input, - DML: &Delete{ + DMLs: []Primitive{&Delete{ DML: &DML{ RoutingParameters: &RoutingParameters{ Opcode: Scatter, @@ -89,8 +90,8 @@ func TestDeleteWithInputMultiOffset(t *testing.T) { }, Query: "dummy_delete", }, - }, - OutputCols: []int{1, 0}, + }}, + OutputCols: [][]int{{1, 0}}, } vc := newDMLTestVCursor("-20", "20-") @@ -114,3 +115,68 @@ func TestDeleteWithInputMultiOffset(t *testing.T) { `ks.20-: dummy_delete {dml_vals: type:TUPLE values:{type:TUPLE value:"\x950\x01a\x89\x02\x011"} values:{type:TUPLE value:"\x950\x01b\x89\x02\x012"} values:{type:TUPLE value:"\x950\x01c\x89\x02\x013"}} true false`, }) } + +func TestDeleteWithMultiTarget(t *testing.T) { + input := &fakePrimitive{results: []*sqltypes.Result{ + sqltypes.MakeTestResult( + sqltypes.MakeTestFields("id|id|user_id", "int64|int64|int64"), + "1|100|1", "2|100|2", "3|200|3"), + }} + + vindex, _ := vindexes.CreateVindex("hash", "", nil) + + del1 := &Delete{ + DML: &DML{ + RoutingParameters: &RoutingParameters{ + Opcode: IN, + Keyspace: &vindexes.Keyspace{Name: "ks", Sharded: true}, + Vindex: vindex, + Values: []evalengine.Expr{ + &evalengine.BindVariable{Key: "dml_vals", Type: sqltypes.Tuple}, + }, + }, + Query: "dummy_delete_1", + }, + } + + del2 := &Delete{ + DML: &DML{ + RoutingParameters: &RoutingParameters{ + Opcode: MultiEqual, + Keyspace: &vindexes.Keyspace{Name: "ks", Sharded: true}, + Vindex: vindex, + Values: []evalengine.Expr{ + &evalengine.TupleBindVariable{Key: "dml_vals", Index: 1}, + }, + }, + Query: "dummy_delete_2", + }, + } + + del := &DMLWithInput{ + Input: input, + DMLs: []Primitive{del1, del2}, + OutputCols: [][]int{{0}, {1, 2}}, + } + + vc := newDMLTestVCursor("-20", "20-") + _, err := del.TryExecute(context.Background(), vc, map[string]*querypb.BindVariable{}, false) + require.NoError(t, err) + vc.ExpectLog(t, []string{ + `ResolveDestinations ks [type:INT64 value:"1" type:INT64 value:"2" type:INT64 value:"3"] Destinations:DestinationKeyspaceID(166b40b44aba4bd6),DestinationKeyspaceID(06e7ea22ce92708f),DestinationKeyspaceID(4eb190c9a2fa169c)`, + `ExecuteMultiShard ks.-20: dummy_delete_1 {dml_vals: type:TUPLE values:{type:INT64 value:"1"} values:{type:INT64 value:"2"} values:{type:INT64 value:"3"}} true true`, + `ResolveDestinations ks [type:INT64 value:"1" type:INT64 value:"2" type:INT64 value:"3"] Destinations:DestinationKeyspaceID(166b40b44aba4bd6),DestinationKeyspaceID(06e7ea22ce92708f),DestinationKeyspaceID(4eb190c9a2fa169c)`, + `ExecuteMultiShard ks.-20: dummy_delete_2 {dml_vals: type:TUPLE values:{type:TUPLE value:"\x89\x02\x03100\x89\x02\x011"} values:{type:TUPLE value:"\x89\x02\x03100\x89\x02\x012"} values:{type:TUPLE value:"\x89\x02\x03200\x89\x02\x013"}} true true`, + }) + + vc.Rewind() + input.rewind() + err = del.TryStreamExecute(context.Background(), vc, map[string]*querypb.BindVariable{}, false, func(result *sqltypes.Result) error { return nil }) + require.NoError(t, err) + vc.ExpectLog(t, []string{ + `ResolveDestinations ks [type:INT64 value:"1" type:INT64 value:"2" type:INT64 value:"3"] Destinations:DestinationKeyspaceID(166b40b44aba4bd6),DestinationKeyspaceID(06e7ea22ce92708f),DestinationKeyspaceID(4eb190c9a2fa169c)`, + `ExecuteMultiShard ks.-20: dummy_delete_1 {dml_vals: type:TUPLE values:{type:INT64 value:"1"} values:{type:INT64 value:"2"} values:{type:INT64 value:"3"}} true true`, + `ResolveDestinations ks [type:INT64 value:"1" type:INT64 value:"2" type:INT64 value:"3"] Destinations:DestinationKeyspaceID(166b40b44aba4bd6),DestinationKeyspaceID(06e7ea22ce92708f),DestinationKeyspaceID(4eb190c9a2fa169c)`, + `ExecuteMultiShard ks.-20: dummy_delete_2 {dml_vals: type:TUPLE values:{type:TUPLE value:"\x89\x02\x03100\x89\x02\x011"} values:{type:TUPLE value:"\x89\x02\x03100\x89\x02\x012"} values:{type:TUPLE value:"\x89\x02\x03200\x89\x02\x013"}} true true`, + }) +} diff --git a/go/vt/vtgate/evalengine/cached_size.go b/go/vt/vtgate/evalengine/cached_size.go index b386d3dc915..72bfbafed73 100644 --- a/go/vt/vtgate/evalengine/cached_size.go +++ b/go/vt/vtgate/evalengine/cached_size.go @@ -373,7 +373,7 @@ func (cached *TupleBindVariable) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(48) + size += int64(32) } // field Key string size += hack.RuntimeAllocSize(int64(len(cached.Key))) diff --git a/go/vt/vtgate/evalengine/expr_tuple_bvar.go b/go/vt/vtgate/evalengine/expr_tuple_bvar.go index b8e506eaff5..3b2553f25ba 100644 --- a/go/vt/vtgate/evalengine/expr_tuple_bvar.go +++ b/go/vt/vtgate/evalengine/expr_tuple_bvar.go @@ -32,12 +32,6 @@ type ( Index int Type sqltypes.Type Collation collations.ID - - // dynamicTypeOffset is set when the type of this bind variable cannot be calculated - // at translation time. Since expressions with dynamic types cannot be compiled ahead of time, - // compilation will be delayed until the expression is first executed with the bind variables - // sent by the user. See: UntypedExpr - dynamicTypeOffset int } ) diff --git a/go/vt/vtgate/planbuilder/delete.go b/go/vt/vtgate/planbuilder/delete.go index a613c158ab7..6d56a41c6df 100644 --- a/go/vt/vtgate/planbuilder/delete.go +++ b/go/vt/vtgate/planbuilder/delete.go @@ -23,7 +23,6 @@ import ( "vitess.io/vitess/go/vt/vtgate/engine" "vitess.io/vitess/go/vt/vtgate/planbuilder/operators" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" - "vitess.io/vitess/go/vt/vtgate/semantics" "vitess.io/vitess/go/vt/vtgate/vindexes" ) @@ -68,8 +67,10 @@ func gen4DeleteStmtPlanner( } } - if err := checkIfDeleteSupported(deleteStmt, ctx.SemTable); err != nil { - return nil, err + // error out here if delete query cannot bypass the planner and + // planner cannot plan such query due to different reason like missing full information, etc. + if ctx.SemTable.NotUnshardedErr != nil { + return nil, ctx.SemTable.NotUnshardedErr } op, err := operators.PlanQuery(ctx, deleteStmt) @@ -132,17 +133,3 @@ func deleteUnshardedShortcut(stmt *sqlparser.Delete, ks *vindexes.Keyspace, tabl } return &primitiveWrapper{prim: &engine.Delete{DML: edml}} } - -// checkIfDeleteSupported checks if the delete query is supported or we must return an error. -func checkIfDeleteSupported(del *sqlparser.Delete, semTable *semantics.SemTable) error { - if semTable.NotUnshardedErr != nil { - return semTable.NotUnshardedErr - } - - // Delete is only supported for single Target. - if len(del.Targets) > 1 { - return vterrors.VT12001("multi-table DELETE statement with multi-target") - } - - return nil -} diff --git a/go/vt/vtgate/planbuilder/dml_with_input.go b/go/vt/vtgate/planbuilder/dml_with_input.go index 1b6d1c0835a..729314e0fc9 100644 --- a/go/vt/vtgate/planbuilder/dml_with_input.go +++ b/go/vt/vtgate/planbuilder/dml_with_input.go @@ -22,9 +22,9 @@ import ( type dmlWithInput struct { input logicalPlan - dml logicalPlan + dmls []logicalPlan - outputCols []int + outputCols [][]int } var _ logicalPlan = (*dmlWithInput)(nil) @@ -32,9 +32,12 @@ var _ logicalPlan = (*dmlWithInput)(nil) // Primitive implements the logicalPlan interface func (d *dmlWithInput) Primitive() engine.Primitive { inp := d.input.Primitive() - del := d.dml.Primitive() + var dels []engine.Primitive + for _, dml := range d.dmls { + dels = append(dels, dml.Primitive()) + } return &engine.DMLWithInput{ - DML: del, + DMLs: dels, Input: inp, OutputCols: d.outputCols, } diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index 577e585351d..4d5a5642dac 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -87,13 +87,17 @@ func transformDMLWithInput(ctx *plancontext.PlanningContext, op *operators.DMLWi return nil, err } - del, err := transformToLogicalPlan(ctx, op.DML) - if err != nil { - return nil, err + var dmls []logicalPlan + for _, dml := range op.DML { + del, err := transformToLogicalPlan(ctx, dml) + if err != nil { + return nil, err + } + dmls = append(dmls, del) } return &dmlWithInput{ input: input, - dml: del, + dmls: dmls, outputCols: op.Offsets, }, nil } diff --git a/go/vt/vtgate/planbuilder/operators/delete.go b/go/vt/vtgate/planbuilder/operators/delete.go index c22da14c35f..7ce321378d0 100644 --- a/go/vt/vtgate/planbuilder/operators/delete.go +++ b/go/vt/vtgate/planbuilder/operators/delete.go @@ -17,8 +17,9 @@ limitations under the License. package operators import ( - "fmt" + "sort" + "vitess.io/vitess/go/slice" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/engine" @@ -34,15 +35,11 @@ type Delete struct { noPredicates } -type TargetTable struct { - ID semantics.TableSet - VTable *vindexes.Table - Name sqlparser.TableName -} - -// Introduces implements the PhysicalOperator interface -func (d *Delete) introducesTableID() semantics.TableSet { - return d.Target.ID +// delOp stores intermediary value for Delete Operator with the vindexes.Table for ordering. +type delOp struct { + op Operator + vTbl *vindexes.Table + cols []*sqlparser.ColName } // Clone implements the Operator interface @@ -63,28 +60,16 @@ func (d *Delete) SetInputs(inputs []Operator) { d.Source = inputs[0] } -func (d *Delete) TablesUsed() []string { - return SingleQualifiedIdentifier(d.Target.VTable.Keyspace, d.Target.VTable.Name) -} - func (d *Delete) GetOrdering(*plancontext.PlanningContext) []OrderBy { return nil } +func (d *Delete) TablesUsed() []string { + return SingleQualifiedIdentifier(d.Target.VTable.Keyspace, d.Target.VTable.Name) +} + func (d *Delete) ShortDescription() string { - ovq := "" - if d.OwnedVindexQuery != nil { - var cols, orderby, limit string - cols = fmt.Sprintf("COLUMNS: [%s]", sqlparser.String(d.OwnedVindexQuery.SelectExprs)) - if len(d.OwnedVindexQuery.OrderBy) > 0 { - orderby = fmt.Sprintf(" ORDERBY: [%s]", sqlparser.String(d.OwnedVindexQuery.OrderBy)) - } - if d.OwnedVindexQuery.Limit != nil { - limit = fmt.Sprintf(" LIMIT: [%s]", sqlparser.String(d.OwnedVindexQuery.Limit)) - } - ovq = fmt.Sprintf(" vindexQuery(%s%s%s)", cols, orderby, limit) - } - return fmt.Sprintf("%s.%s%s", d.Target.VTable.Keyspace.Name, d.Target.VTable.Name.String(), ovq) + return shortDesc(d.Target, d.OwnedVindexQuery) } func createOperatorFromDelete(ctx *plancontext.PlanningContext, deleteStmt *sqlparser.Delete) (op Operator) { @@ -94,7 +79,7 @@ func createOperatorFromDelete(ctx *plancontext.PlanningContext, deleteStmt *sqlp // slower, because it does a selection and then creates a delete statement wherein we have to // list all the primary key values. if deleteWithInputPlanningRequired(childFks, deleteStmt) { - return deleteWithInputPlanningForFk(ctx, deleteStmt) + return createDeleteWithInputOp(ctx, deleteStmt) } delClone := sqlparser.CloneRefOfDelete(deleteStmt) @@ -117,6 +102,12 @@ func createOperatorFromDelete(ctx *plancontext.PlanningContext, deleteStmt *sqlp } func deleteWithInputPlanningRequired(childFks []vindexes.ChildFKInfo, deleteStmt *sqlparser.Delete) bool { + if len(deleteStmt.Targets) > 1 { + if len(childFks) > 0 { + panic(vterrors.VT12001("multi table delete with foreign keys")) + } + return true + } // If there are no foreign keys, we don't need to use delete with input. if len(childFks) == 0 { return false @@ -132,7 +123,7 @@ func deleteWithInputPlanningRequired(childFks []vindexes.ChildFKInfo, deleteStmt return !deleteStmt.IsSingleAliasExpr() } -func deleteWithInputPlanningForFk(ctx *plancontext.PlanningContext, del *sqlparser.Delete) Operator { +func createDeleteWithInputOp(ctx *plancontext.PlanningContext, del *sqlparser.Delete) (op Operator) { delClone := ctx.SemTable.Clone(del).(*sqlparser.Delete) del.Limit = nil del.OrderBy = nil @@ -144,18 +135,81 @@ func deleteWithInputPlanningForFk(ctx *plancontext.PlanningContext, del *sqlpars Limit: delClone.Limit, Lock: sqlparser.ForUpdateLock, } - ts := ctx.SemTable.Targets[del.Targets[0].Name] + + var delOps []delOp + for _, target := range del.Targets { + op := createDeleteOpWithTarget(ctx, target) + delOps = append(delOps, op) + } + + // sort the operator based on sharding vindex type. + // Unsharded < Lookup Vindex < Any + // This is needed to ensure all the rows are deleted from unowned sharding tables first. + // Otherwise, those table rows will be missed from getting deleted as + // the owned table row won't have matching values. + sort.Slice(delOps, func(i, j int) bool { + a, b := delOps[i], delOps[j] + // Get the first Vindex of a and b, if available + aVdx, bVdx := getFirstVindex(a.vTbl), getFirstVindex(b.vTbl) + + // Sort nil Vindexes to the start + if aVdx == nil || bVdx == nil { + return aVdx != nil // true if bVdx is nil and aVdx is not nil + } + + // Among non-nil Vindexes, those that need VCursor come first + return aVdx.NeedsVCursor() && !bVdx.NeedsVCursor() + }) + + // now map the operator and column list. + var colsList [][]*sqlparser.ColName + dmls := slice.Map(delOps, func(from delOp) Operator { + colsList = append(colsList, from.cols) + for _, col := range from.cols { + selectStmt.SelectExprs = append(selectStmt.SelectExprs, aeWrap(col)) + } + return from.op + }) + + op = &DMLWithInput{ + DML: dmls, + Source: createOperatorFromSelect(ctx, selectStmt), + cols: colsList, + } + + if del.Comments != nil { + op = &LockAndComment{ + Source: op, + Comments: del.Comments, + } + } + return op +} + +// getFirstVindex returns the first Vindex, if available +func getFirstVindex(vTbl *vindexes.Table) vindexes.Vindex { + if len(vTbl.ColumnVindexes) > 0 { + return vTbl.ColumnVindexes[0].Vindex + } + return nil +} + +func createDeleteOpWithTarget(ctx *plancontext.PlanningContext, target sqlparser.TableName) delOp { + ts := ctx.SemTable.Targets[target.Name] ti, err := ctx.SemTable.TableInfoFor(ts) if err != nil { panic(vterrors.VT13001(err.Error())) } + vTbl := ti.GetVindexTable() + if len(vTbl.PrimaryKey) == 0 { + panic(vterrors.VT09015()) + } var leftComp sqlparser.ValTuple cols := make([]*sqlparser.ColName, 0, len(vTbl.PrimaryKey)) for _, col := range vTbl.PrimaryKey { - colName := sqlparser.NewColNameWithQualifier(col.String(), vTbl.GetTableName()) - selectStmt.SelectExprs = append(selectStmt.SelectExprs, aeWrap(colName)) + colName := sqlparser.NewColNameWithQualifier(col.String(), target) cols = append(cols, colName) leftComp = append(leftComp, colName) ctx.SemTable.Recursive[colName] = ts @@ -167,14 +221,15 @@ func deleteWithInputPlanningForFk(ctx *plancontext.PlanningContext, del *sqlpars } compExpr := sqlparser.NewComparisonExpr(sqlparser.InOp, lhs, sqlparser.ListArg(engine.DmlVals), nil) - del.Targets = sqlparser.TableNames{del.Targets[0]} - del.TableExprs = sqlparser.TableExprs{ti.GetAliasedTableExpr()} - del.Where = sqlparser.NewWhere(sqlparser.WhereClause, compExpr) - - return &DMLWithInput{ - DML: createOperatorFromDelete(ctx, del), - Source: createOperatorFromSelect(ctx, selectStmt), - cols: cols, + del := &sqlparser.Delete{ + TableExprs: sqlparser.TableExprs{ti.GetAliasedTableExpr()}, + Targets: sqlparser.TableNames{target}, + Where: sqlparser.NewWhere(sqlparser.WhereClause, compExpr), + } + return delOp{ + createOperatorFromDelete(ctx, del), + vTbl, + cols, } } diff --git a/go/vt/vtgate/planbuilder/operators/dml_planning.go b/go/vt/vtgate/planbuilder/operators/dml_planning.go index 6f71b41162e..561cfde4c1a 100644 --- a/go/vt/vtgate/planbuilder/operators/dml_planning.go +++ b/go/vt/vtgate/planbuilder/operators/dml_planning.go @@ -35,6 +35,28 @@ type DMLCommon struct { Source Operator } +type TargetTable struct { + ID semantics.TableSet + VTable *vindexes.Table + Name sqlparser.TableName +} + +func shortDesc(target TargetTable, ovq *sqlparser.Select) string { + ovqString := "" + if ovq != nil { + var cols, orderby, limit string + cols = fmt.Sprintf("COLUMNS: [%s]", sqlparser.String(ovq.SelectExprs)) + if len(ovq.OrderBy) > 0 { + orderby = fmt.Sprintf(" ORDERBY: [%s]", sqlparser.String(ovq.OrderBy)) + } + if ovq.Limit != nil { + limit = fmt.Sprintf(" LIMIT: [%s]", sqlparser.String(ovq.Limit)) + } + ovqString = fmt.Sprintf(" vindexQuery(%s%s%s)", cols, orderby, limit) + } + return fmt.Sprintf("%s.%s%s", target.VTable.Keyspace.Name, target.VTable.Name.String(), ovqString) +} + // getVindexInformation returns the vindex and VindexPlusPredicates for the DML, // If it cannot find a unique vindex match, it returns an error. func getVindexInformation(id semantics.TableSet, table *vindexes.Table) ( diff --git a/go/vt/vtgate/planbuilder/operators/dml_with_input.go b/go/vt/vtgate/planbuilder/operators/dml_with_input.go index e15a1042a47..848941b4468 100644 --- a/go/vt/vtgate/planbuilder/operators/dml_with_input.go +++ b/go/vt/vtgate/planbuilder/operators/dml_with_input.go @@ -27,10 +27,10 @@ import ( // DMLWithInput is used to represent a DML Operator taking input from a Source Operator type DMLWithInput struct { Source Operator - DML Operator - cols []*sqlparser.ColName - Offsets []int + DML []Operator + cols [][]*sqlparser.ColName + Offsets [][]int noColumns noPredicates @@ -43,26 +43,38 @@ func (d *DMLWithInput) Clone(inputs []Operator) Operator { } func (d *DMLWithInput) Inputs() []Operator { - return []Operator{d.Source, d.DML} + return append([]Operator{d.Source}, d.DML...) } func (d *DMLWithInput) SetInputs(inputs []Operator) { - if len(inputs) != 2 { + if len(inputs) < 2 { panic("unexpected number of inputs for DMLWithInput operator") } d.Source = inputs[0] - d.DML = inputs[1] + d.DML = inputs[1:] } func (d *DMLWithInput) ShortDescription() string { - colStrings := slice.Map(d.cols, func(from *sqlparser.ColName) string { + colStrings := "" + for idx, columns := range d.cols { + var offsets []int + if len(d.Offsets) > idx { + offsets = d.Offsets[idx] + } + colStrings += fmt.Sprintf("[%s]", getShortDesc(columns, offsets)) + } + return colStrings +} + +func getShortDesc(cols []*sqlparser.ColName, offsets []int) string { + colStrings := slice.Map(cols, func(from *sqlparser.ColName) string { return sqlparser.String(from) }) out := "" for idx, colString := range colStrings { out += colString - if len(d.Offsets) > idx { - out += fmt.Sprintf(":%d", d.Offsets[idx]) + if len(offsets) > idx { + out += fmt.Sprintf(":%d", offsets[idx]) } out += " " } @@ -74,10 +86,14 @@ func (d *DMLWithInput) GetOrdering(ctx *plancontext.PlanningContext) []OrderBy { } func (d *DMLWithInput) planOffsets(ctx *plancontext.PlanningContext) Operator { - for _, col := range d.cols { - offset := d.Source.AddColumn(ctx, true, false, aeWrap(col)) - d.Offsets = append(d.Offsets, offset) + offsets := make([][]int, len(d.cols)) + for idx, columns := range d.cols { + for _, col := range columns { + offset := d.Source.AddColumn(ctx, true, false, aeWrap(col)) + offsets[idx] = append(offsets[idx], offset) + } } + d.Offsets = offsets return d } diff --git a/go/vt/vtgate/planbuilder/operators/phases.go b/go/vt/vtgate/planbuilder/operators/phases.go index 3937105c494..2fc3a5a044f 100644 --- a/go/vt/vtgate/planbuilder/operators/phases.go +++ b/go/vt/vtgate/planbuilder/operators/phases.go @@ -144,10 +144,11 @@ func createDMLWithInput(ctx *plancontext.PlanningContext, op, src Operator, in * dm := &DMLWithInput{} var leftComp sqlparser.ValTuple proj := newAliasedProjection(src) + dm.cols = make([][]*sqlparser.ColName, 1) for _, col := range in.Target.VTable.PrimaryKey { colName := sqlparser.NewColNameWithQualifier(col.String(), in.Target.Name) proj.AddColumn(ctx, true, false, aeWrap(colName)) - dm.cols = append(dm.cols, colName) + dm.cols[0] = append(dm.cols[0], colName) leftComp = append(leftComp, colName) ctx.SemTable.Recursive[colName] = in.Target.ID } @@ -189,7 +190,7 @@ func createDMLWithInput(ctx *plancontext.PlanningContext, op, src Operator, in * in.OwnedVindexQuery.OrderBy = nil in.OwnedVindexQuery.Limit = nil } - dm.DML = op + dm.DML = append(dm.DML, op) return dm, Rewrote("changed Delete to DMLWithInput") } diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index cf55f91fddd..311064fd3e6 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -72,11 +72,6 @@ func (se SetExpr) String() string { return fmt.Sprintf("%s = %s", sqlparser.String(se.Name), sqlparser.String(se.Expr.EvalExpr)) } -// Introduces implements the PhysicalOperator interface -func (u *Update) introducesTableID() semantics.TableSet { - return u.Target.ID -} - // Clone implements the Operator interface func (u *Update) Clone(inputs []Operator) Operator { upd := *u @@ -95,19 +90,7 @@ func (u *Update) TablesUsed() []string { } func (u *Update) ShortDescription() string { - ovq := "" - if u.OwnedVindexQuery != nil { - var cols, orderby, limit string - cols = fmt.Sprintf("COLUMNS: [%s]", sqlparser.String(u.OwnedVindexQuery.SelectExprs)) - if len(u.OwnedVindexQuery.OrderBy) > 0 { - orderby = fmt.Sprintf(" ORDERBY: [%s]", sqlparser.String(u.OwnedVindexQuery.OrderBy)) - } - if u.OwnedVindexQuery.Limit != nil { - limit = fmt.Sprintf(" LIMIT: [%s]", sqlparser.String(u.OwnedVindexQuery.Limit)) - } - ovq = fmt.Sprintf(" vindexQuery(%s%s%s)", cols, orderby, limit) - } - return fmt.Sprintf("%s.%s%s", u.Target.VTable.Keyspace.Name, u.Target.VTable.Name.String(), ovq) + return shortDesc(u.Target, u.OwnedVindexQuery) } func createOperatorFromUpdate(ctx *plancontext.PlanningContext, updStmt *sqlparser.Update) (op Operator) { diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index e7c18af1d4c..2ce1d04b4c5 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -64,6 +64,9 @@ func TestPlan(t *testing.T) { } testOutputTempDir := makeTestOutput(t) addPKs(t, vschemaWrapper.V, "user", []string{"user", "music"}) + addPKsProvided(t, vschemaWrapper.V, "user", []string{"user_extra"}, []string{"id", "user_id"}) + addPKsProvided(t, vschemaWrapper.V, "ordering", []string{"order"}, []string{"oid", "region_id"}) + addPKsProvided(t, vschemaWrapper.V, "ordering", []string{"order_event"}, []string{"oid", "ename"}) // You will notice that some tests expect user.Id instead of user.id. // This is because we now pre-create vindex columns in the symbol @@ -238,6 +241,13 @@ func addPKs(t *testing.T, vschema *vindexes.VSchema, ks string, tbls []string) { } } +func addPKsProvided(t *testing.T, vschema *vindexes.VSchema, ks string, tbls []string, pks []string) { + for _, tbl := range tbls { + require.NoError(t, + vschema.AddPrimaryKey(ks, tbl, pks)) + } +} + func TestSystemTables57(t *testing.T) { // first we move everything to use 5.7 logic env, err := vtenv.New(vtenv.Options{ @@ -280,6 +290,9 @@ func TestOne(t *testing.T) { setFks(t, lv) addPKs(t, lv, "user", []string{"user", "music"}) addPKs(t, lv, "main", []string{"unsharded"}) + addPKsProvided(t, lv, "user", []string{"user_extra"}, []string{"id", "user_id"}) + addPKsProvided(t, lv, "ordering", []string{"order"}, []string{"oid", "region_id"}) + addPKsProvided(t, lv, "ordering", []string{"order_event"}, []string{"oid", "ename"}) vschema := &vschemawrapper.VSchemaWrapper{ V: lv, TestBuilder: TestBuilder, diff --git a/go/vt/vtgate/planbuilder/testdata/dml_cases.json b/go/vt/vtgate/planbuilder/testdata/dml_cases.json index 6fbc31eb84d..7fb3a577729 100644 --- a/go/vt/vtgate/planbuilder/testdata/dml_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/dml_cases.json @@ -5058,7 +5058,7 @@ "OperatorType": "DMLWithInput", "TargetTabletType": "PRIMARY", "Offset": [ - 0 + "0:[0]" ], "Inputs": [ { @@ -5134,7 +5134,7 @@ "OperatorType": "DMLWithInput", "TargetTabletType": "PRIMARY", "Offset": [ - 0 + "0:[0]" ], "Inputs": [ { @@ -5206,7 +5206,7 @@ "OperatorType": "DMLWithInput", "TargetTabletType": "PRIMARY", "Offset": [ - 0 + "0:[0]" ], "Inputs": [ { @@ -5304,7 +5304,7 @@ "OperatorType": "DMLWithInput", "TargetTabletType": "PRIMARY", "Offset": [ - 0 + "0:[0]" ], "Inputs": [ { @@ -5377,7 +5377,7 @@ "OperatorType": "DMLWithInput", "TargetTabletType": "PRIMARY", "Offset": [ - 0 + "0:[0]" ], "Inputs": [ { @@ -5450,7 +5450,7 @@ "OperatorType": "DMLWithInput", "TargetTabletType": "PRIMARY", "Offset": [ - 0 + "0:[0]" ], "Inputs": [ { @@ -5505,7 +5505,7 @@ "OperatorType": "DMLWithInput", "TargetTabletType": "PRIMARY", "Offset": [ - 0 + "0:[0]" ], "Inputs": [ { @@ -5561,7 +5561,7 @@ "OperatorType": "DMLWithInput", "TargetTabletType": "PRIMARY", "Offset": [ - 0 + "0:[0]" ], "Inputs": [ { @@ -5613,7 +5613,7 @@ "OperatorType": "DMLWithInput", "TargetTabletType": "PRIMARY", "Offset": [ - 0 + "0:[0]" ], "Inputs": [ { @@ -5671,7 +5671,7 @@ "OperatorType": "DMLWithInput", "TargetTabletType": "PRIMARY", "Offset": [ - 0 + "0:[0]" ], "Inputs": [ { @@ -5750,7 +5750,7 @@ "OperatorType": "DMLWithInput", "TargetTabletType": "PRIMARY", "Offset": [ - 0 + "0:[0]" ], "Inputs": [ { @@ -6188,5 +6188,484 @@ "user.user" ] } + }, + { + "comment": "multi target delete on sharded table", + "query": "delete u, m from user u, music m where u.col = m.col and u.foo = m.bar and u.baz = 12 and m.baz = 21", + "plan": { + "QueryType": "DELETE", + "Original": "delete u, m from user u, music m where u.col = m.col and u.foo = m.bar and u.baz = 12 and m.baz = 21", + "Instructions": { + "OperatorType": "DMLWithInput", + "TargetTabletType": "PRIMARY", + "Offset": [ + "0:[0]", + "1:[1]" + ], + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,R:0", + "JoinVars": { + "u_col": 1, + "u_foo": 2 + }, + "TableName": "`user`_music", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select u.id, u.col, u.foo from `user` as u where 1 != 1", + "Query": "select u.id, u.col, u.foo from `user` as u where u.baz = 12 for update", + "Table": "`user`" + }, + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select m.id from music as m where 1 != 1", + "Query": "select m.id from music as m where m.baz = 21 and m.bar = :u_foo and m.col = :u_col for update", + "Table": "music" + } + ] + }, + { + "OperatorType": "Delete", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "TargetTabletType": "PRIMARY", + "KsidLength": 1, + "KsidVindex": "user_index", + "OwnedVindexQuery": "select Id, `Name`, Costly from `user` as u where u.id in ::dml_vals for update", + "Query": "delete from `user` as u where u.id in ::dml_vals", + "Table": "user", + "Values": [ + "::dml_vals" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Delete", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "TargetTabletType": "PRIMARY", + "KsidLength": 1, + "KsidVindex": "user_index", + "OwnedVindexQuery": "select user_id, id from music as m where m.id in ::dml_vals for update", + "Query": "delete from music as m where m.id in ::dml_vals", + "Table": "music", + "Values": [ + "::dml_vals" + ], + "Vindex": "music_user_map" + } + ] + }, + "TablesUsed": [ + "user.music", + "user.user" + ] + } + }, + { + "comment": "delete with multi-table targets", + "query": "delete music,user from music inner join user where music.id = user.id", + "plan": { + "QueryType": "DELETE", + "Original": "delete music,user from music inner join user where music.id = user.id", + "Instructions": { + "OperatorType": "DMLWithInput", + "TargetTabletType": "PRIMARY", + "Offset": [ + "0:[0]", + "1:[1]" + ], + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,R:0", + "JoinVars": { + "music_id": 0 + }, + "TableName": "music_`user`", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select music.id from music where 1 != 1", + "Query": "select music.id from music for update", + "Table": "music" + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select `user`.id from `user` where 1 != 1", + "Query": "select `user`.id from `user` where `user`.id = :music_id for update", + "Table": "`user`", + "Values": [ + ":music_id" + ], + "Vindex": "user_index" + } + ] + }, + { + "OperatorType": "Delete", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "TargetTabletType": "PRIMARY", + "KsidLength": 1, + "KsidVindex": "user_index", + "OwnedVindexQuery": "select user_id, id from music where music.id in ::dml_vals for update", + "Query": "delete from music where music.id in ::dml_vals", + "Table": "music", + "Values": [ + "::dml_vals" + ], + "Vindex": "music_user_map" + }, + { + "OperatorType": "Delete", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "TargetTabletType": "PRIMARY", + "KsidLength": 1, + "KsidVindex": "user_index", + "OwnedVindexQuery": "select Id, `Name`, Costly from `user` where `user`.id in ::dml_vals for update", + "Query": "delete from `user` where `user`.id in ::dml_vals", + "Table": "user", + "Values": [ + "::dml_vals" + ], + "Vindex": "user_index" + } + ] + }, + "TablesUsed": [ + "user.music", + "user.user" + ] + } + }, + { + "comment": "multi table delete with 2 sharded tables join on vindex column", + "query": "delete u, m from user u join music m on u.id = m.user_id", + "plan": { + "QueryType": "DELETE", + "Original": "delete u, m from user u join music m on u.id = m.user_id", + "Instructions": { + "OperatorType": "DMLWithInput", + "TargetTabletType": "PRIMARY", + "Offset": [ + "0:[0]", + "1:[1]" + ], + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select u.id, m.id from `user` as u, music as m where 1 != 1", + "Query": "select u.id, m.id from `user` as u, music as m where u.id = m.user_id for update", + "Table": "`user`, music" + }, + { + "OperatorType": "Delete", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "TargetTabletType": "PRIMARY", + "KsidLength": 1, + "KsidVindex": "user_index", + "OwnedVindexQuery": "select Id, `Name`, Costly from `user` as u where u.id in ::dml_vals for update", + "Query": "delete from `user` as u where u.id in ::dml_vals", + "Table": "user", + "Values": [ + "::dml_vals" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Delete", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "TargetTabletType": "PRIMARY", + "KsidLength": 1, + "KsidVindex": "user_index", + "OwnedVindexQuery": "select user_id, id from music as m where m.id in ::dml_vals for update", + "Query": "delete from music as m where m.id in ::dml_vals", + "Table": "music", + "Values": [ + "::dml_vals" + ], + "Vindex": "music_user_map" + } + ] + }, + "TablesUsed": [ + "user.music", + "user.user" + ] + } + }, + { + "comment": "multi table delete with 2 sharded tables join on non-vindex column", + "query": "delete u, m from user u join music m on u.col = m.col", + "plan": { + "QueryType": "DELETE", + "Original": "delete u, m from user u join music m on u.col = m.col", + "Instructions": { + "OperatorType": "DMLWithInput", + "TargetTabletType": "PRIMARY", + "Offset": [ + "0:[0]", + "1:[1]" + ], + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "L:0,R:0", + "JoinVars": { + "u_col": 1 + }, + "TableName": "`user`_music", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select u.id, u.col from `user` as u where 1 != 1", + "Query": "select u.id, u.col from `user` as u for update", + "Table": "`user`" + }, + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select m.id from music as m where 1 != 1", + "Query": "select m.id from music as m where m.col = :u_col for update", + "Table": "music" + } + ] + }, + { + "OperatorType": "Delete", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "TargetTabletType": "PRIMARY", + "KsidLength": 1, + "KsidVindex": "user_index", + "OwnedVindexQuery": "select Id, `Name`, Costly from `user` as u where u.id in ::dml_vals for update", + "Query": "delete from `user` as u where u.id in ::dml_vals", + "Table": "user", + "Values": [ + "::dml_vals" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Delete", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "TargetTabletType": "PRIMARY", + "KsidLength": 1, + "KsidVindex": "user_index", + "OwnedVindexQuery": "select user_id, id from music as m where m.id in ::dml_vals for update", + "Query": "delete from music as m where m.id in ::dml_vals", + "Table": "music", + "Values": [ + "::dml_vals" + ], + "Vindex": "music_user_map" + } + ] + }, + "TablesUsed": [ + "user.music", + "user.user" + ] + } + }, + { + "comment": "multi target delete with composite primary key having single column vindex", + "query": "delete u, ue from user u join user_extra ue on u.id = ue.user_id", + "plan": { + "QueryType": "DELETE", + "Original": "delete u, ue from user u join user_extra ue on u.id = ue.user_id", + "Instructions": { + "OperatorType": "DMLWithInput", + "TargetTabletType": "PRIMARY", + "Offset": [ + "0:[0]", + "1:[1 2]" + ], + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select u.id, ue.id, ue.user_id from `user` as u, user_extra as ue where 1 != 1", + "Query": "select u.id, ue.id, ue.user_id from `user` as u, user_extra as ue where u.id = ue.user_id for update", + "Table": "`user`, user_extra" + }, + { + "OperatorType": "Delete", + "Variant": "IN", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "TargetTabletType": "PRIMARY", + "KsidLength": 1, + "KsidVindex": "user_index", + "OwnedVindexQuery": "select Id, `Name`, Costly from `user` as u where u.id in ::dml_vals for update", + "Query": "delete from `user` as u where u.id in ::dml_vals", + "Table": "user", + "Values": [ + "::dml_vals" + ], + "Vindex": "user_index" + }, + { + "OperatorType": "Delete", + "Variant": "MultiEqual", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from user_extra as ue where (ue.id, ue.user_id) in ::dml_vals", + "Table": "user_extra", + "Values": [ + "dml_vals:1" + ], + "Vindex": "user_index" + } + ] + }, + "TablesUsed": [ + "user.user", + "user.user_extra" + ] + } + }, + { + "comment": "multi target delete with composite primary key with lookup vindex as sharding column", + "query": "delete o, ev from `order` o join order_event ev where o.oid = ev.oid and ev.ename = 'a'", + "plan": { + "QueryType": "DELETE", + "Original": "delete o, ev from `order` o join order_event ev where o.oid = ev.oid and ev.ename = 'a'", + "Instructions": { + "OperatorType": "DMLWithInput", + "TargetTabletType": "PRIMARY", + "Offset": [ + "0:[0 1]", + "1:[2 3]" + ], + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "ordering", + "Sharded": true + }, + "FieldQuery": "select ev.oid, ev.ename, o.oid, o.region_id from `order` as o, order_event as ev where 1 != 1", + "Query": "select ev.oid, ev.ename, o.oid, o.region_id from `order` as o, order_event as ev where ev.ename = 'a' and o.oid = ev.oid for update", + "Table": "`order`, order_event" + }, + { + "OperatorType": "Delete", + "Variant": "MultiEqual", + "Keyspace": { + "Name": "ordering", + "Sharded": true + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from order_event as ev where (ev.oid, ev.ename) in ::dml_vals", + "Table": "order_event", + "Values": [ + "dml_vals:0" + ], + "Vindex": "oid_vdx" + }, + { + "OperatorType": "Delete", + "Variant": "MultiEqual", + "Keyspace": { + "Name": "ordering", + "Sharded": true + }, + "TargetTabletType": "PRIMARY", + "KsidLength": 1, + "KsidVindex": "xxhash", + "OwnedVindexQuery": "select region_id, oid from `order` as o where (o.oid, o.region_id) in ::dml_vals for update", + "Query": "delete from `order` as o where (o.oid, o.region_id) in ::dml_vals", + "Table": "order", + "Values": [ + "dml_vals:1" + ], + "Vindex": "xxhash" + } + ] + }, + "TablesUsed": [ + "ordering.order", + "ordering.order_event" + ] + } } ] diff --git a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json index c8f059b7b88..b28955e368f 100644 --- a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json @@ -1255,7 +1255,7 @@ "OperatorType": "DMLWithInput", "TargetTabletType": "PRIMARY", "Offset": [ - 0 + "0:[0]" ], "Inputs": [ { @@ -3131,7 +3131,7 @@ "OperatorType": "DMLWithInput", "TargetTabletType": "PRIMARY", "Offset": [ - 0 + "0:[0]" ], "Inputs": [ { @@ -3141,8 +3141,8 @@ "Name": "unsharded_fk_allow", "Sharded": false }, - "FieldQuery": "select u_tbl6.id from u_tbl6 as u, u_tbl5 as m where 1 != 1", - "Query": "select u_tbl6.id from u_tbl6 as u, u_tbl5 as m where u.col2 = 4 and m.col3 = 6 and u.col = m.col for update", + "FieldQuery": "select u.id from u_tbl6 as u, u_tbl5 as m where 1 != 1", + "Query": "select u.id from u_tbl6 as u, u_tbl5 as m where u.col2 = 4 and m.col3 = 6 and u.col = m.col for update", "Table": "u_tbl5, u_tbl6" }, { @@ -3157,7 +3157,7 @@ "Sharded": false }, "FieldQuery": "select u_tbl6.col6 from u_tbl6 as u where 1 != 1", - "Query": "select u_tbl6.col6 from u_tbl6 as u where u_tbl6.id in ::dml_vals for update", + "Query": "select u_tbl6.col6 from u_tbl6 as u where u.id in ::dml_vals for update", "Table": "u_tbl6" }, { @@ -3185,7 +3185,7 @@ "Sharded": false }, "TargetTabletType": "PRIMARY", - "Query": "delete from u_tbl6 as u where u_tbl6.id in ::dml_vals", + "Query": "delete from u_tbl6 as u where u.id in ::dml_vals", "Table": "u_tbl6" } ] @@ -3209,7 +3209,7 @@ "OperatorType": "DMLWithInput", "TargetTabletType": "PRIMARY", "Offset": [ - 0 + "0:[0]" ], "Inputs": [ { @@ -3286,7 +3286,7 @@ "OperatorType": "DMLWithInput", "TargetTabletType": "PRIMARY", "Offset": [ - 0 + "0:[0]" ], "Inputs": [ { @@ -3399,7 +3399,7 @@ "OperatorType": "DMLWithInput", "TargetTabletType": "PRIMARY", "Offset": [ - 0 + "0:[0]" ], "Inputs": [ { @@ -3740,7 +3740,7 @@ "OperatorType": "DMLWithInput", "TargetTabletType": "PRIMARY", "Offset": [ - 0 + "0:[0]" ], "Inputs": [ { @@ -3806,5 +3806,10 @@ "unsharded_fk_allow.u_tbl8" ] } + }, + { + "comment": "multi table delete on foreign key enabled tables", + "query": "delete u, m from u_tbl6 u join u_tbl5 m on u.col = m.col where u.col2 = 4 and m.col3 = 6", + "plan": "VT12001: unsupported: multi table delete with foreign keys" } ] diff --git a/go/vt/vtgate/planbuilder/testdata/foreignkey_checks_on_cases.json b/go/vt/vtgate/planbuilder/testdata/foreignkey_checks_on_cases.json index 311ff6adbff..6962ce50621 100644 --- a/go/vt/vtgate/planbuilder/testdata/foreignkey_checks_on_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/foreignkey_checks_on_cases.json @@ -1255,7 +1255,7 @@ "OperatorType": "DMLWithInput", "TargetTabletType": "PRIMARY", "Offset": [ - 0 + "0:[0]" ], "Inputs": [ { @@ -1266,7 +1266,7 @@ "Sharded": false }, "FieldQuery": "select u_tbl2.id from u_tbl2 where 1 != 1", - "Query": "select u_tbl2.id from u_tbl2 limit 2 for update", + "Query": "select /*+ SET_VAR(foreign_key_checks=On) */ u_tbl2.id from u_tbl2 limit 2", "Table": "u_tbl2" }, { @@ -1281,7 +1281,7 @@ "Sharded": false }, "FieldQuery": "select u_tbl2.col2 from u_tbl2 where 1 != 1", - "Query": "select u_tbl2.col2 from u_tbl2 where u_tbl2.id in ::dml_vals for update", + "Query": "select /*+ SET_VAR(foreign_key_checks=On) */ u_tbl2.col2 from u_tbl2 where u_tbl2.id in ::dml_vals", "Table": "u_tbl2" }, { @@ -1297,7 +1297,7 @@ "Cols": [ 0 ], - "Query": "update /*+ SET_VAR(foreign_key_checks=ON) */ u_tbl3 set col3 = null where (col3) in ::fkc_vals", + "Query": "update /*+ SET_VAR(foreign_key_checks=On) */ u_tbl3 set col3 = null where (col3) in ::fkc_vals", "Table": "u_tbl3" }, { diff --git a/go/vt/vtgate/planbuilder/testdata/show_cases.json b/go/vt/vtgate/planbuilder/testdata/show_cases.json index 896f762819e..e7a810bb42f 100644 --- a/go/vt/vtgate/planbuilder/testdata/show_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/show_cases.json @@ -734,7 +734,7 @@ "Keyspace": "VARCHAR", "Sharded": "VARCHAR" }, - "RowCount": 7 + "RowCount": 8 } } }, diff --git a/go/vt/vtgate/planbuilder/testdata/unknown_schema_cases.json b/go/vt/vtgate/planbuilder/testdata/unknown_schema_cases.json index 888127f11c4..df4459d9e0f 100644 --- a/go/vt/vtgate/planbuilder/testdata/unknown_schema_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/unknown_schema_cases.json @@ -68,5 +68,15 @@ "comment": "unsharded insert, no col list with auto-inc", "query": "insert into unsharded_auto values(1,1)", "plan": "VT09004: INSERT should contain column list or the table should have authoritative columns in vschema" + }, + { + "comment": "We need schema tracking to allow unexpanded columns inside UNION", + "query": "select x from (select t.*, 0 as x from user t union select t.*, 1 as x from user_extra t) AS t", + "plan": "VT09015: schema tracking required" + }, + { + "comment": "multi table delete with 1 sharded and 1 reference table", + "query": "delete u, r from user u join ref_with_source r on u.col = r.col", + "plan": "VT09015: schema tracking required" } ] diff --git a/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json b/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json index 8007a0e1a0a..1fb4e61b37a 100644 --- a/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/unsupported_cases.json @@ -104,11 +104,6 @@ "query": "replace into user(id) values (1), (2)", "plan": "VT12001: unsupported: REPLACE INTO with sharded keyspace" }, - { - "comment": "delete with multi-table targets", - "query": "delete music,user from music inner join user where music.id = user.id", - "plan": "VT12001: unsupported: multi-table DELETE statement with multi-target" - }, { "comment": "select get_lock with non-dual table", "query": "select get_lock('xyz', 10) from user", @@ -329,26 +324,6 @@ "query": "SELECT (SELECT sum(user.name) FROM music LIMIT 1) FROM user", "plan": "VT12001: unsupported: correlated subquery is only supported for EXISTS" }, - { - "comment": "We need schema tracking to allow unexpanded columns inside UNION", - "query": "select x from (select t.*, 0 as x from user t union select t.*, 1 as x from user_extra t) AS t", - "plan": "VT09015: schema tracking required" - }, - { - "comment": "multi table delete with 2 sharded tables join on vindex column", - "query": "delete u, m from user u join music m on u.id = m.user_id", - "plan": "VT12001: unsupported: multi-table DELETE statement with multi-target" - }, - { - "comment": "multi table delete with 2 sharded tables join on non-vindex column", - "query": "delete u, m from user u join music m on u.col = m.col", - "plan": "VT12001: unsupported: multi-table DELETE statement with multi-target" - }, - { - "comment": "multi table delete with 1 sharded and 1 reference table", - "query": "delete u, r from user u join ref_with_source r on u.col = r.col", - "plan": "VT12001: unsupported: multi-table DELETE statement with multi-target" - }, { "comment": "reference table delete with join", "query": "delete r from user u join ref_with_source r on u.col = r.col", diff --git a/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json b/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json index 7aaa2648388..a8fe91e5d49 100644 --- a/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json +++ b/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json @@ -895,6 +895,53 @@ "u_multicol_tbl2": {}, "u_multicol_tbl3": {} } + }, + "ordering": { + "sharded": true, + "vindexes": { + "xxhash": { + "type": "xxhash" + }, + "oid_vdx": { + "type": "consistent_lookup_unique", + "params": { + "table": "oid_idx", + "from": "oid", + "to": "keyspace_id" + }, + "owner": "order" + } + }, + "tables": { + "order": { + "column_vindexes": [ + { + "column": "region_id", + "name": "xxhash" + }, + { + "column": "oid", + "name": "oid_vdx" + } + ] + }, + "oid_idx": { + "column_vindexes": [ + { + "column": "oid", + "name": "xxhash" + } + ] + }, + "order_event": { + "column_vindexes": [ + { + "column": "oid", + "name": "oid_vdx" + } + ] + } + } } } }