[release-18.0] Column alias expanding on ORDER BY (#15302)

go/mysql/sqlerror/sql_error.go

@@ -243,6 +243,7 @@ var stateToMysqlCode = map[vterrors.State]mysqlCode{
 	vterrors.CharacterSetMismatch:         {num: ERCharacterSetMismatch, state: SSUnknownSQLState},
 	vterrors.WrongParametersToNativeFct:   {num: ERWrongParametersToNativeFct, state: SSUnknownSQLState},
 	vterrors.KillDeniedError:              {num: ERKillDenied, state: SSUnknownSQLState},
+	vterrors.InvalidGroupFuncUse:          {num: ERInvalidGroupFuncUse, state: SSUnknownSQLState},
 }
 
 func getStateToMySQLState(state vterrors.State) mysqlCode {

go/test/endtoend/utils/cmp.go

@@ -70,6 +70,12 @@ func (mcmp *MySQLCompare) AssertMatches(query, expected string) {
 	}
 }
 
+// SkipIfBinaryIsBelowVersion should be used instead of using utils.SkipIfBinaryIsBelowVersion(t,
+// This is because we might be inside a Run block that has a different `t` variable
+func (mcmp *MySQLCompare) SkipIfBinaryIsBelowVersion(majorVersion int, binary string) {
+	SkipIfBinaryIsBelowVersion(mcmp.t, majorVersion, binary)
+}
+
 // AssertMatchesAny ensures the given query produces any one of the expected results.
 func (mcmp *MySQLCompare) AssertMatchesAny(query string, expected ...string) {
 	mcmp.t.Helper()

go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go

@@ -94,7 +94,7 @@ func TestEqualFilterOnScatter(t *testing.T) {
 
 	workloads := []string{"oltp", "olap"}
 	for _, workload := range workloads {
-		t.Run(workload, func(t *testing.T) {
+		mcmp.Run(workload, func(mcmp *utils.MySQLCompare) {
 			utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = '%s'", workload))
 
 			mcmp.AssertMatches("select count(*) as a from aggr_test having 1 = 1", `[[INT64(5)]]`)
@@ -179,7 +179,7 @@ func TestNotEqualFilterOnScatter(t *testing.T) {
 
 	workloads := []string{"oltp", "olap"}
 	for _, workload := range workloads {
-		t.Run(workload, func(t *testing.T) {
+		mcmp.Run(workload, func(mcmp *utils.MySQLCompare) {
 			utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = '%s'", workload))
 
 			mcmp.AssertMatches("select count(*) as a from aggr_test having a != 5", `[]`)
@@ -203,7 +203,7 @@ func TestLessFilterOnScatter(t *testing.T) {
 
 	workloads := []string{"oltp", "olap"}
 	for _, workload := range workloads {
-		t.Run(workload, func(t *testing.T) {
+		mcmp.Run(workload, func(mcmp *utils.MySQLCompare) {
 			utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = '%s'", workload))
 			mcmp.AssertMatches("select count(*) as a from aggr_test having a < 10", `[[INT64(5)]]`)
 			mcmp.AssertMatches("select count(*) as a from aggr_test having 1 < a", `[[INT64(5)]]`)
@@ -226,7 +226,7 @@ func TestLessEqualFilterOnScatter(t *testing.T) {
 
 	workloads := []string{"oltp", "olap"}
 	for _, workload := range workloads {
-		t.Run(workload, func(t *testing.T) {
+		mcmp.Run(workload, func(mcmp *utils.MySQLCompare) {
 			utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = '%s'", workload))
 
 			mcmp.AssertMatches("select count(*) as a from aggr_test having a <= 10", `[[INT64(5)]]`)
@@ -250,7 +250,7 @@ func TestGreaterFilterOnScatter(t *testing.T) {
 
 	workloads := []string{"oltp", "olap"}
 	for _, workload := range workloads {
-		t.Run(workload, func(t *testing.T) {
+		mcmp.Run(workload, func(mcmp *utils.MySQLCompare) {
 			utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = '%s'", workload))
 
 			mcmp.AssertMatches("select count(*) as a from aggr_test having a > 1", `[[INT64(5)]]`)
@@ -274,7 +274,7 @@ func TestGreaterEqualFilterOnScatter(t *testing.T) {
 
 	workloads := []string{"oltp", "olap"}
 	for _, workload := range workloads {
-		t.Run(workload, func(t *testing.T) {
+		mcmp.Run(workload, func(mcmp *utils.MySQLCompare) {
 			utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = '%s'", workload))
 
 			mcmp.AssertMatches("select count(*) as a from aggr_test having a >= 1", `[[INT64(5)]]`)
@@ -309,7 +309,7 @@ func TestAggOnTopOfLimit(t *testing.T) {
 	mcmp.Exec("insert into aggr_test(id, val1, val2) values(1,'a',6), (2,'a',1), (3,'b',1), (4,'c',3), (5,'c',4), (6,'b',null), (7,null,2), (8,null,null)")
 
 	for _, workload := range []string{"oltp", "olap"} {
-		t.Run(workload, func(t *testing.T) {
+		mcmp.Run(workload, func(mcmp *utils.MySQLCompare) {
 			utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = '%s'", workload))
 			mcmp.AssertMatches(" select count(*) from (select id, val1 from aggr_test where val2 < 4 limit 2) as x", "[[INT64(2)]]")
 			mcmp.AssertMatches(" select count(val1) from (select id, val1 from aggr_test where val2 < 4 order by val1 desc limit 2) as x", "[[INT64(2)]]")
@@ -342,7 +342,7 @@ func TestEmptyTableAggr(t *testing.T) {
 	defer closer()
 
 	for _, workload := range []string{"oltp", "olap"} {
-		t.Run(workload, func(t *testing.T) {
+		mcmp.Run(workload, func(mcmp *utils.MySQLCompare) {
 			utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = %s", workload))
 			mcmp.AssertMatches(" select count(*) from t1 inner join t2 on (t1.t1_id = t2.id) where t1.value = 'foo'", "[[INT64(0)]]")
 			mcmp.AssertMatches(" select count(*) from t2 inner join t1 on (t1.t1_id = t2.id) where t1.value = 'foo'", "[[INT64(0)]]")
@@ -354,7 +354,7 @@ func TestEmptyTableAggr(t *testing.T) {
 	mcmp.Exec("insert into t1(t1_id, `name`, `value`, shardkey) values(1,'a1','foo',100), (2,'b1','foo',200), (3,'c1','foo',300), (4,'a1','foo',100), (5,'b1','bar',200)")
 
 	for _, workload := range []string{"oltp", "olap"} {
-		t.Run(workload, func(t *testing.T) {
+		mcmp.Run(workload, func(mcmp *utils.MySQLCompare) {
 			utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = %s", workload))
 			mcmp.AssertMatches(" select count(*) from t1 inner join t2 on (t1.t1_id = t2.id) where t1.value = 'foo'", "[[INT64(0)]]")
 			mcmp.AssertMatches(" select count(*) from t2 inner join t1 on (t1.t1_id = t2.id) where t1.value = 'foo'", "[[INT64(0)]]")

go/test/endtoend/vtgate/queries/dml/insert_test.go

@@ -38,7 +38,7 @@ func TestSimpleInsertSelect(t *testing.T) {
 	mcmp.Exec("insert into u_tbl(id, num) values (1,2),(3,4)")
 
 	for i, mode := range []string{"oltp", "olap"} {
-		t.Run(mode, func(t *testing.T) {
+		mcmp.Run(mode, func(mcmp *utils.MySQLCompare) {
 			utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = %s", mode))
 
 			qr := mcmp.Exec(fmt.Sprintf("insert into s_tbl(id, num) select id*%d, num*%d from s_tbl where id < 10", 10+i, 20+i))
@@ -65,7 +65,7 @@ func TestFailureInsertSelect(t *testing.T) {
 	mcmp.Exec("insert into u_tbl(id, num) values (1,2),(3,4)")
 
 	for _, mode := range []string{"oltp", "olap"} {
-		t.Run(mode, func(t *testing.T) {
+		mcmp.Run(mode, func(mcmp *utils.MySQLCompare) {
 			utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = %s", mode))
 
 			// primary key same
@@ -386,7 +386,7 @@ func TestInsertSelectUnshardedUsingSharded(t *testing.T) {
 	mcmp.Exec("insert into s_tbl(id, num) values (1,2),(3,4)")
 
 	for _, mode := range []string{"oltp", "olap"} {
-		t.Run(mode, func(t *testing.T) {
+		mcmp.Run(mode, func(mcmp *utils.MySQLCompare) {
 			utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = %s", mode))
 			qr := mcmp.Exec("insert into u_tbl(id, num) select id, num from s_tbl where s_tbl.id in (1,3)")
 			assert.EqualValues(t, 2, qr.RowsAffected)

go/test/endtoend/vtgate/queries/lookup_queries/main_test.go

@@ -134,7 +134,7 @@ func TestLookupQueries(t *testing.T) {
 	(3, 'monkey', 'monkey')`)
 
 	for _, workload := range []string{"olap", "oltp"} {
-		t.Run(workload, func(t *testing.T) {
+		mcmp.Run(workload, func(mcmp *utils.MySQLCompare) {
 			utils.Exec(t, mcmp.VtConn, "set workload = "+workload)
 
 			mcmp.AssertMatches("select id from user where lookup = 'apa'", "[[INT64(1)] [INT64(2)]]")

go/test/endtoend/vtgate/queries/misc/misc_test.go

@@ -268,7 +268,7 @@ func TestAnalyze(t *testing.T) {
 	defer closer()
 
 	for _, workload := range []string{"olap", "oltp"} {
-		t.Run(workload, func(t *testing.T) {
+		mcmp.Run(workload, func(mcmp *utils.MySQLCompare) {
 			utils.Exec(t, mcmp.VtConn, fmt.Sprintf("set workload = %s", workload))
 			utils.Exec(t, mcmp.VtConn, "analyze table t1")
 			utils.Exec(t, mcmp.VtConn, "analyze table uks.unsharded")

go/test/endtoend/vtgate/queries/orderby/orderby_test.go

@@ -83,3 +83,73 @@ func TestOrderBy(t *testing.T) {
 		mcmp.AssertMatches("select /*vt+ PLANNER=Gen4 */ id1, id2 from t4 order by reverse(id2) desc", `[[INT64(5) VARCHAR("test")] [INT64(8) VARCHAR("F")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(2) VARCHAR("Abc")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(1) VARCHAR("a")]]`)
 	}
 }
+
+func TestOrderByComplex(t *testing.T) {
+	// tests written to try to trick the ORDER BY engine and planner
+	utils.SkipIfBinaryIsBelowVersion(t, 20, "vtgate")
+
+	mcmp, closer := start(t)
+	defer closer()
+
+	mcmp.Exec("insert into user(id, col, email) values(1,1,'a'), (2,2,'Abc'), (3,3,'b'), (4,4,'c'), (5,2,'test'), (6,1,'test'), (7,2,'a'), (8,3,'b'), (9,4,'c3'), (10,2,'d')")
+
+	queries := []string{
+		"select email, max(col) from user group by email order by col",
+		"select email, max(col) from user group by email order by col + 1",
+		"select email, max(col) from user group by email order by max(col)",
+		"select email, max(col) from user group by email order by max(col) + 1",
+		"select email, max(col) from user group by email order by min(col)",
+		"select email, max(col) as col from user group by email order by col",
+		"select email, max(col) as col from user group by email order by max(col)",
+		"select email, max(col) as col from user group by email order by col + 1",
+		"select email, max(col) as col from user group by email order by email + col",
+		"select email, max(col) as col from user group by email order by email + max(col)",
+		"select email, max(col) as col from user group by email order by email, col",
+		"select email, max(col) as xyz from user group by email order by email, xyz",
+		"select email, max(col) as xyz from user group by email order by email, max(xyz)",
+		"select email, max(col) as xyz from user group by email order by email, abs(xyz)",
+		"select email, max(col) as xyz from user group by email order by email, max(col)",
+		"select email, max(col) as xyz from user group by email order by email, abs(col)",
+		"select email, max(col) as xyz from user group by email order by xyz + email",
+		"select email, max(col) as xyz from user group by email order by abs(xyz) + email",
+		"select email, max(col) as xyz from user group by email order by abs(xyz)",
+		"select email, max(col) as xyz from user group by email order by abs(col)",
+		"select email, max(col) as max_col from user group by email order by max_col desc, length(email)",
+		"select email, max(col) as max_col, min(col) as min_col from user group by email order by max_col - min_col",
+		"select email, max(col) as col1, count(*) as col2 from user group by email order by col2 * col1",
+		"select email, sum(col) as sum_col from user group by email having sum_col > 10 order by sum_col / count(email)",
+		"select email, max(col) as max_col, char_length(email) as len_email from user group by email order by len_email, max_col desc",
+		"select email, max(col) as col_alias from user group by email order by case when col_alias > 100 then 0 else 1 end, col_alias",
+		"select email, count(*) as cnt, max(col) as max_col from user group by email order by cnt desc, max_col + cnt",
+		"select email, max(col) as max_col from user group by email order by if(max_col > 50, max_col, -max_col) desc",
+		"select email, max(col) as col, sum(col) as sum_col from user group by email order by col * sum_col desc",
+		"select email, max(col) as col, (select min(col) from user as u2 where u2.email = user.email) as min_col from user group by email order by col - min_col",
+		"select email, max(col) as max_col, (max(col) % 10) as mod_col from user group by email order by mod_col, max_col",
+		"select email, max(col) as 'value', count(email) as 'number' from user group by email order by 'number', 'value'",
+		"select email, max(col) as col, concat('email: ', email, ' col: ', max(col)) as complex_alias from user group by email order by complex_alias desc",
+		"select email, max(col) as max_col from user group by email union select email, min(col) as min_col from user group by email order by email",
+		"select email, max(col) as col from user where col > 50 group by email order by col desc",
+		"select email, max(col) as col from user group by email order by length(email), col",
+		"select email, max(col) as max_col, substring(email, 1, 3) as sub_email from user group by email order by sub_email, max_col desc",
+		"select email, max(col) as max_col from user group by email order by reverse(email), max_col",
+		"select email, max(col) as max_col from user group by email having max_col > avg(max_col) order by max_col desc",
+		"select email, count(*) as count, max(col) as max_col from user group by email order by count * max_col desc",
+		"select email, max(col) as col_alias from user group by email order by col_alias limit 10",
+		"select email, max(col) as col from user group by email order by col desc, email",
+		"select concat(email, ' ', max(col)) as combined from user group by email order by combined desc",
+		"select email, max(col) as max_col from user group by email order by ascii(email), max_col",
+		"select email, char_length(email) as email_length, max(col) as max_col from user group by email order by email_length desc, max_col",
+		"select email, max(col) as col from user group by email having col between 10 and 100 order by col",
+		"select email, max(col) as max_col, min(col) as min_col from user group by email order by max_col + min_col desc",
+		"select email, max(col) as 'max', count(*) as 'count' from user group by email order by 'max' desc, 'count'",
+		"select email, max(col) as max_col from (select email, col from user where col > 20) as filtered group by email order by max_col",
+		"select a.email, a.max_col from (select email, max(col) as max_col from user group by email) as a order by a.max_col desc",
+		"select email, max(col) as max_col from user where email like 'a%' group by email order by max_col, email",
+	}
+
+	for _, query := range queries {
+		mcmp.Run(query, func(mcmp *utils.MySQLCompare) {
+			_, _ = mcmp.ExecAllowAndCompareError(query)
+		})
+	}
+}

go/test/endtoend/vtgate/queries/orderby/schema.sql

@@ -27,3 +27,12 @@ create table t4_id2_idx
 ) Engine = InnoDB
   DEFAULT charset = utf8mb4
   COLLATE = utf8mb4_general_ci;
+
+create table user
+(
+    id    bigint primary key,
+    col   bigint,
+    email varchar(20)
+) Engine = InnoDB
+  DEFAULT charset = utf8mb4
+  COLLATE = utf8mb4_general_ci;

go/test/endtoend/vtgate/queries/orderby/vschema.json

@@ -4,7 +4,7 @@
     "hash": {
       "type": "hash"
     },
-    "unicode_loose_md5" : {
+    "unicode_loose_md5": {
       "type": "unicode_loose_md5"
     },
     "t1_id2_vdx": {
@@ -54,7 +54,10 @@
           "name": "hash"
         },
         {
-          "columns": ["id2", "id1"],
+          "columns": [
+            "id2",
+            "id1"
+          ],
           "name": "t4_id2_vdx"
         }
       ]
@@ -66,6 +69,14 @@
           "name": "unicode_loose_md5"
         }
       ]
+    },
+    "user": {
+      "column_vindexes": [
+        {
+          "column": "id",
+          "name": "hash"
+        }
+      ]
     }
   }
 }

go/test/endtoend/vtgate/queries/union/union_test.go

@@ -57,7 +57,7 @@ func TestUnionDistinct(t *testing.T) {
 	mcmp.Exec("insert into t2(id3, id4) values (2, 3), (3, 4), (4,4), (5,5)")
 
 	for _, workload := range []string{"oltp", "olap"} {
-		t.Run(workload, func(t *testing.T) {
+		mcmp.Run(workload, func(mcmp *utils.MySQLCompare) {
 			utils.Exec(t, mcmp.VtConn, "set workload = "+workload)
 			mcmp.AssertMatches("select 1 union select null", "[[INT64(1)] [NULL]]")
 			mcmp.AssertMatches("select null union select null", "[[NULL]]")
@@ -69,10 +69,7 @@ func TestUnionDistinct(t *testing.T) {
 			mcmp.AssertMatchesNoOrder("select id1 from t1 where id1 = 1 union select 452 union select id1 from t1 where id1 = 4", "[[INT64(1)] [INT64(452)] [INT64(4)]]")
 			mcmp.AssertMatchesNoOrder("select id1, id2 from t1 union select 827, 452 union select id3,id4 from t2",
 				"[[INT64(4) INT64(4)] [INT64(1) INT64(1)] [INT64(2) INT64(2)] [INT64(3) INT64(3)] [INT64(827) INT64(452)] [INT64(2) INT64(3)] [INT64(3) INT64(4)] [INT64(5) INT64(5)]]")
-			t.Run("skipped for now", func(t *testing.T) {
-				t.Skip()
-				mcmp.AssertMatches("select 1 from dual where 1 IN (select 1 as col union select 2)", "[[INT64(1)]]")
-			})
+			mcmp.AssertMatches("select 1 from dual where 1 IN (select 1 as col union select 2)", "[[INT64(1)]]")
 			mcmp.AssertMatches(`SELECT 1 from t1 UNION SELECT 2 from t1`, `[[INT64(1)] [INT64(2)]]`)
 			mcmp.AssertMatches(`SELECT 5 from t1 UNION SELECT 6 from t1`, `[[INT64(5)] [INT64(6)]]`)
 			mcmp.AssertMatchesNoOrder(`SELECT id1 from t1 UNION SELECT id2 from t1`, `[[INT64(1)] [INT64(2)] [INT64(3)] [INT64(4)]]`)
@@ -95,7 +92,7 @@ func TestUnionAll(t *testing.T) {
 	mcmp.Exec("insert into t2(id3, id4) values(3, 3), (4, 4)")
 
 	for _, workload := range []string{"oltp", "olap"} {
-		t.Run(workload, func(t *testing.T) {
+		mcmp.Run(workload, func(mcmp *utils.MySQLCompare) {
 			utils.Exec(t, mcmp.VtConn, "set workload = "+workload)
 			// union all between two selectuniqueequal
 			mcmp.AssertMatches("select id1 from t1 where id1 = 1 union all select id1 from t1 where id1 = 4", "[[INT64(1)]]")

go/vt/vterrors/state.go

@@ -47,6 +47,7 @@ const (
 	WrongValueCountOnRow
 	WrongValue
 	WrongArguments
+	InvalidGroupFuncUse
 
 	// failed precondition
 	NoDB