Feature: Adding support for Vindex Hints to allow for greater control…

… over shard routing (#15172)

Signed-off-by: Manan Gupta <[email protected]>
Signed-off-by: Andres Taylor <[email protected]>
Signed-off-by: Harshit Gangal <[email protected]>
Co-authored-by: Andres Taylor <[email protected]>
Co-authored-by: Harshit Gangal <[email protected]>

changelog/20.0/20.0.0/summary.md

@@ -0,0 +1,29 @@
+## Summary
+
+### Table of Contents
+
+- **[Major Changes](#major-changes)**
+  - **[Query Serving](#query-serving)**
+    - [Vindex Hints](#vindex-hints)
+- **[Minor Changes](#minor-changes)**
+
+
+## <a id="major-changes"/>Major Changes
+
+
+### <a id="query-serving"/>Query Serving
+
+#### <a id="vindex-hints"/> Vindex Hints
+
+Vitess now supports Vindex hints that provide a way for users to influence the shard routing of queries in Vitess by specifying, which vindexes should be considered or ignored by the query planner. This feature enhances the control over query execution, allowing for potentially more efficient data access patterns in sharded databases.
+
+Example:
+ ```sql
+    SELECT * FROM user USE VINDEX (hash_user_id, secondary_vindex) WHERE user_id = 123;
+    SELECT * FROM order IGNORE VINDEX (range_order_id) WHERE order_date = '2021-01-01';
+ ```
+
+For more information about Vindex hints and its usage, please consult the documentation.
+
+## <a id="minor-changes"/>Minor Changes
+

changelog/20.0/README.md

@@ -0,0 +1,2 @@
+## v20.0
+* **[20.0.0](20.0.0)**

changelog/README.md

@@ -1,4 +1,5 @@
 ## Releases
+* [20.0](20.0)
 * [19.0](19.0)
 * [18.0](18.0)
 * [17.0](17.0)

go/mysql/sqlerror/sql_error.go

@@ -217,6 +217,7 @@ var stateToMysqlCode = map[vterrors.State]mysqlCode{
 	vterrors.WrongValueCountOnRow:         {num: ERWrongValueCountOnRow, state: SSWrongValueCountOnRow},
 	vterrors.WrongArguments:               {num: ERWrongArguments, state: SSUnknownSQLState},
 	vterrors.UnknownStmtHandler:           {num: ERUnknownStmtHandler, state: SSUnknownSQLState},
+	vterrors.KeyDoesNotExist:              {num: ERKeyDoesNotExist, state: SSClientError},
 	vterrors.UnknownTimeZone:              {num: ERUnknownTimeZone, state: SSUnknownSQLState},
 	vterrors.RegexpStringNotTerminated:    {num: ERRegexpStringNotTerminated, state: SSUnknownSQLState},
 	vterrors.RegexpBufferOverflow:         {num: ERRegexpBufferOverflow, state: SSUnknownSQLState},

go/test/endtoend/vtgate/queries/misc/misc_test.go

@@ -36,7 +36,7 @@ func start(t *testing.T) (utils.MySQLCompare, func()) {
 	require.NoError(t, err)
 
 	deleteAll := func() {
-		tables := []string{"t1", "uks.unsharded"}
+		tables := []string{"t1", "tbl", "unq_idx", "nonunq_idx", "uks.unsharded"}
 		for _, table := range tables {
 			_, _ = mcmp.ExecAndIgnore("delete from " + table)
 		}
@@ -137,6 +137,41 @@ func TestCast(t *testing.T) {
 	mcmp.AssertMatches("select cast('3.2' as unsigned)", `[[UINT64(3)]]`)
 }
 
+// TestVindexHints tests that vindex hints work as intended.
+func TestVindexHints(t *testing.T) {
+	utils.SkipIfBinaryIsBelowVersion(t, 20, "vtgate")
+	mcmp, closer := start(t)
+	defer closer()
+
+	mcmp.Exec("insert into tbl(id, unq_col, nonunq_col) values (1,0,10), (2,10,10), (3,4,20), (4,30,20), (5,40,10)")
+	mcmp.AssertMatches("select id, unq_col, nonunq_col from tbl where unq_col = 10 and id = 2 and nonunq_col in (10, 20)", "[[INT64(2) INT64(10) INT64(10)]]")
+
+	// Verify that without any vindex hints, the query plan uses a hash vindex.
+	res, err := mcmp.VtConn.ExecuteFetch("vexplain plan select id, unq_col, nonunq_col from tbl where unq_col = 10 and id = 2 and nonunq_col in (10, 20)", 100, false)
+	require.NoError(t, err)
+	require.Contains(t, fmt.Sprintf("%v", res.Rows), "hash")
+
+	// Now we make the query explicitly use the unique lookup vindex.
+	// We make sure the query still works.
+	res, err = mcmp.VtConn.ExecuteFetch("select id, unq_col, nonunq_col from tbl USE VINDEX (unq_vdx) where unq_col = 10 and id = 2 and nonunq_col in (10, 20)", 100, false)
+	require.NoError(t, err)
+	require.EqualValues(t, fmt.Sprintf("%v", res.Rows), "[[INT64(2) INT64(10) INT64(10)]]")
+	// Verify that we are using the unq_vdx, that we requested explicitly.
+	res, err = mcmp.VtConn.ExecuteFetch("vexplain plan select id, unq_col, nonunq_col from tbl USE VINDEX (unq_vdx) where unq_col = 10 and id = 2 and nonunq_col in (10, 20)", 100, false)
+	require.NoError(t, err)
+	require.Contains(t, fmt.Sprintf("%v", res.Rows), "unq_vdx")
+
+	// Now we make the query explicitly refuse two of the three vindexes.
+	// We make sure the query still works.
+	res, err = mcmp.VtConn.ExecuteFetch("select id, unq_col, nonunq_col from tbl IGNORE VINDEX (hash, unq_vdx) where unq_col = 10 and id = 2 and nonunq_col in (10, 20)", 100, false)
+	require.NoError(t, err)
+	require.EqualValues(t, fmt.Sprintf("%v", res.Rows), "[[INT64(2) INT64(10) INT64(10)]]")
+	// Verify that we are using the nonunq_vdx, which is the only one left to be used.
+	res, err = mcmp.VtConn.ExecuteFetch("vexplain plan select id, unq_col, nonunq_col from tbl IGNORE VINDEX (hash, unq_vdx) where unq_col = 10 and id = 2 and nonunq_col in (10, 20)", 100, false)
+	require.NoError(t, err)
+	require.Contains(t, fmt.Sprintf("%v", res.Rows), "nonunq_vdx")
+}
+
 func TestOuterJoinWithPredicate(t *testing.T) {
 	mcmp, closer := start(t)
 	defer closer()

go/test/endtoend/vtgate/queries/misc/schema.sql

@@ -2,4 +2,28 @@ create table if not exists t1(
                    id1 bigint,
                    id2 bigint,
                    primary key(id1)
-) Engine=InnoDB;
+) Engine=InnoDB;
+
+create table unq_idx
+(
+    unq_col     bigint,
+    keyspace_id varbinary(20),
+    primary key (unq_col)
+) Engine = InnoDB;
+
+create table nonunq_idx
+(
+    nonunq_col  bigint,
+    id          bigint,
+    keyspace_id varbinary(20),
+    primary key (nonunq_col, id)
+) Engine = InnoDB;
+
+create table tbl
+(
+    id         bigint,
+    unq_col    bigint,
+    nonunq_col bigint,
+    primary key (id),
+    unique (unq_col)
+) Engine = InnoDB;

go/test/endtoend/vtgate/queries/misc/vschema.json

@@ -3,6 +3,26 @@
   "vindexes": {
     "hash": {
       "type": "hash"
+    },
+    "unq_vdx": {
+      "type": "consistent_lookup_unique",
+      "params": {
+        "table": "unq_idx",
+        "from": "unq_col",
+        "to": "keyspace_id",
+        "ignore_nulls": "true"
+      },
+      "owner": "tbl"
+    },
+    "nonunq_vdx": {
+      "type": "consistent_lookup",
+      "params": {
+        "table": "nonunq_idx",
+        "from": "nonunq_col,id",
+        "to": "keyspace_id",
+        "ignore_nulls": "true"
+      },
+      "owner": "tbl"
     }
   },
   "tables": {
@@ -13,6 +33,41 @@
           "name": "hash"
         }
       ]
+    },
+    "tbl": {
+      "column_vindexes": [
+        {
+          "column": "id",
+          "name": "hash"
+        },
+        {
+          "column": "unq_col",
+          "name": "unq_vdx"
+        },
+        {
+          "columns": [
+            "nonunq_col",
+            "id"
+          ],
+          "name": "nonunq_vdx"
+        }
+      ]
+    },
+    "unq_idx": {
+      "column_vindexes": [
+        {
+          "column": "unq_col",
+          "name": "hash"
+        }
+      ]
+    },
+    "nonunq_idx": {
+      "column_vindexes": [
+        {
+          "column": "nonunq_col",
+          "name": "hash"
+        }
+      ]
     }
   }
 }

go/vt/sqlparser/ast_format.go

@@ -1241,7 +1241,7 @@ func (node IndexHints) Format(buf *TrackedBuffer) {
 
 // Format formats the node.
 func (node *IndexHint) Format(buf *TrackedBuffer) {
-	buf.astPrintf(node, " %sindex ", node.Type.ToString())
+	buf.astPrintf(node, " %s ", node.Type.ToString())
 	if node.ForType != NoForType {
 		buf.astPrintf(node, "for %s ", node.ForType.ToString())
 	}

go/vt/sqlparser/ast_funcs.go

@@ -1559,14 +1559,28 @@ func (ty IndexHintType) ToString() string {
 	case UseOp:
 		return UseStr
 	case IgnoreOp:
-		return IgnoreStr
+		return IgnoreIndexStr
 	case ForceOp:
 		return ForceStr
+	case UseVindexOp:
+		return UseVindexStr
+	case IgnoreVindexOp:
+		return IgnoreVindexStr
 	default:
 		return "Unknown IndexHintType"
 	}
 }
 
+// IsVindexHint returns if the given hint is a Vindex hint or not.
+func (ty IndexHintType) IsVindexHint() bool {
+	switch ty {
+	case UseVindexOp, IgnoreVindexOp:
+		return true
+	default:
+		return false
+	}
+}
+
 // ToString returns the type as a string
 func (ty IndexHintForType) ToString() string {
 	switch ty {

go/vt/sqlparser/constants.go

@@ -122,10 +122,15 @@ const (
 	NaturalLeftJoinStr  = "natural left join"
 	NaturalRightJoinStr = "natural right join"
 
-	// Index hints.
-	UseStr    = "use "
+	// IgnoreStr string.
 	IgnoreStr = "ignore "
-	ForceStr  = "force "
+
+	// Index hints.
+	UseStr          = "use index"
+	IgnoreIndexStr  = "ignore index"
+	ForceStr        = "force index"
+	UseVindexStr    = "use vindex"
+	IgnoreVindexStr = "ignore vindex"
 
 	// Index hints For types.
 	JoinForStr    = "join"
@@ -760,6 +765,8 @@ const (
 	UseOp IndexHintType = iota
 	IgnoreOp
 	ForceOp
+	UseVindexOp
+	IgnoreVindexOp
 )
 
 // Constant for Enum Type - IndexHintForType

go/vt/sqlparser/parse_test.go

@@ -2659,6 +2659,20 @@ var (
 	}, {
 		input:  "SELECT id FROM blog_posts USE INDEX (PRIMARY) WHERE id = 10",
 		output: "select id from blog_posts use index (`PRIMARY`) where id = 10",
+	}, {
+		input: "select * from payment_pulls ignore vindex (lookup_vindex_name) where customer_id in (1, 10) and payment_id = 5",
+	}, {
+		input:  "select * from payment_pulls ignore vindex (lookup_vindex_name, x, t) order by id",
+		output: "select * from payment_pulls ignore vindex (lookup_vindex_name, x, t) order by id asc",
+	}, {
+		input: "select * from payment_pulls use vindex (lookup_vindex_name) where customer_id in (1, 10) and payment_id = 5",
+	}, {
+		input:  "select * from payment_pulls use vindex (lookup_vindex_name, x, t) order by id",
+		output: "select * from payment_pulls use vindex (lookup_vindex_name, x, t) order by id asc",
+	}, {
+		input: "select * from payment_pulls use vindex (lookup_vindex_name, x, t) ignore vindex (x, t)",
+	}, {
+		input: "select * from payment_pulls use vindex (lookup_vindex_name, x, t) ignore vindex (x, t) join tab ignore vindex (y)",
 	}, {
 		input:  "select name, group_concat(score) from t group by name",
 		output: "select `name`, group_concat(score) from t group by `name`",