feat: support invisible columns (#14366)

go/vt/sqlparser/ast_funcs.go

@@ -2496,3 +2496,7 @@ func IsLiteral(expr Expr) bool {
 		return false
 	}
 }
+
+func (ct *ColumnType) Invisible() bool {
+	return ct.Options.Invisible != nil && *ct.Options.Invisible
+}

go/vt/vtgate/planbuilder/testdata/select_cases.json

@@ -4765,5 +4765,27 @@
         "user.customer"
       ]
     }
+  },
+  {
+    "comment": "invisible column is not expanded, but valid in predicate",
+    "query": "select * from samecolvin where secret = 12",
+    "plan": {
+      "QueryType": "SELECT",
+      "Original": "select * from samecolvin where secret = 12",
+      "Instructions": {
+        "OperatorType": "Route",
+        "Variant": "Scatter",
+        "Keyspace": {
+          "Name": "user",
+          "Sharded": true
+        },
+        "FieldQuery": "select col from samecolvin where 1 != 1",
+        "Query": "select col from samecolvin where secret = 12",
+        "Table": "samecolvin"
+      },
+      "TablesUsed": [
+        "user.samecolvin"
+      ]
+    }
   }
 ]

go/vt/vtgate/planbuilder/testdata/vschemas/schema.json

@@ -302,6 +302,10 @@
           "columns": [
             {
               "name": "col"
+            },
+            {
+              "name": "secret",
+              "invisible": true
             }
           ],
           "column_list_authoritative": true

go/vt/vtgate/schema/tracker.go

@@ -307,6 +307,7 @@ func getColumns(tblSpec *sqlparser.TableSpec) []vindexes.Column {
 				Name:          column.Name,
 				Type:          column.Type.SQLType(),
 				CollationName: colCollation,
+				Invisible:     column.Type.Invisible(),
 			})
 	}
 	return cols

go/vt/vtgate/semantics/early_rewriter.go

@@ -574,6 +574,9 @@ func (e *expanderState) processColumnsFor(tbl TableInfo) error {
 outer:
 	// in this first loop we just find columns used in any JOIN USING used on this table
 	for _, col := range tbl.getColumns() {
+		if col.Invisible {
+			continue
+		}
 		ts, found := usingCols[col.Name]
 		if found {
 			for i, ts := range ts.Constituents() {
@@ -590,6 +593,10 @@ outer:
 
 	// and this time around we are printing any columns not involved in any JOIN USING
 	for _, col := range tbl.getColumns() {
+		if col.Invisible {
+			continue
+		}
+
 		if ts, found := usingCols[col.Name]; found && currTable.IsSolvedBy(ts) {
 			continue
 		}

go/vt/vtgate/semantics/early_rewriter_test.go

@@ -48,6 +48,10 @@ func TestExpandStar(t *testing.T) {
 				}, {
 					Name: sqlparser.NewIdentifierCI("c"),
 					Type: sqltypes.VarChar,
+				}, {
+					Name:      sqlparser.NewIdentifierCI("secret"),
+					Type:      sqltypes.Decimal,
+					Invisible: true,
 				}},
 				ColumnListAuthoritative: true,
 			},

go/vt/vtgate/semantics/real_table.go

@@ -119,6 +119,7 @@ func vindexTableToColumnInfo(tbl *vindexes.Table) []ColumnInfo {
 				Type: col.Type,
 				Coll: collation,
 			},
+			Invisible: col.Invisible,
 		})
 		nameMap[col.Name.String()] = nil
 	}

go/vt/vtgate/semantics/semantic_state.go

@@ -62,8 +62,9 @@ type (
 
 	// ColumnInfo contains information about columns
 	ColumnInfo struct {
-		Name string
-		Type evalengine.Type
+		Name      string
+		Type      evalengine.Type
+		Invisible bool
 	}
 
 	// ExprDependencies stores the tables that an expression depends on as a map

go/vt/vtgate/vindexes/vschema.go

@@ -178,6 +178,9 @@ type Column struct {
 	Name          sqlparser.IdentifierCI `json:"name"`
 	Type          querypb.Type           `json:"type"`
 	CollationName string                 `json:"collation_name"`
+
+	// Invisible marks this as a column that will not be automatically included in `*` projections
+	Invisible bool `json:"invisible"`
 }
 
 // MarshalJSON returns a JSON representation of Column.
@@ -610,7 +613,7 @@ func buildTables(ks *vschemapb.Keyspace, vschema *VSchema, ksvschema *KeyspaceSc
 				)
 			}
 			colNames[name.Lowered()] = true
-			t.Columns = append(t.Columns, Column{Name: name, Type: col.Type})
+			t.Columns = append(t.Columns, Column{Name: name, Type: col.Type, Invisible: col.Invisible})
 		}
 
 		// Initialize ColumnVindexes.

proto/vschema.proto

@@ -126,6 +126,7 @@ message AutoIncrement {
 message Column {
   string name = 1;
   query.Type type = 2;
+  bool invisible = 3;
 }
 
 // SrvVSchema is the roll-up of all the Keyspace schema for a cell.