ZDM-606: Support colon operator in fuzzy SAI queries

antlr/README.md

@@ -2,10 +2,17 @@
 
 ## How to generate the grammar files
 
-First, install antlr 4.
+1. Install ANTLR 4.
+   - On a Mac: `brew install antlr`.
 
-On a Mac: `brew install antlr`.
+2. Generate the Go files for the `SimplifiedCql` grammar:
+    ```
+    antlr -Dlanguage=Go antlr/SimplifiedCql.g4
+    ```
 
-Second, generate the Go files for the `SimplifiedCql` grammar:
-
-    antlr -Dlanguage=Go antlr/SimplifiedCql.g4
+Current ZDM code works with ANTLR 4.9.3, so if you have issues downloading it from system package managers you can:
+1. Download JAR file from https://repo1.maven.org/maven2/org/antlr/antlr4/4.9.3
+2. Generate Go files for simplified CQL grammar:
+   ```
+   java -Xmx500M -cp ".:/path/to/antlr4-4.9.3-complete.jar" org.antlr.v4.Tool -Dlanguage=Go antlr/SimplifiedCql.g4
+   ```

antlr/SimplifiedCql.g4

@@ -211,6 +211,7 @@ operator
     | '>'
     | '>='
     | '!='
+    | ':'
     ;
 
 // CQL literals

antlr/SimplifiedCql.tokens

@@ -122,8 +122,8 @@ OTHER=107
 '>'=15
 '>='=16
 '!='=17
-'{'=18
-'}'=19
-':'=20
+':'=18
+'{'=19
+'}'=20
 '?'=96
 ';'=106

antlr/SimplifiedCqlLexer.tokens

@@ -122,8 +122,8 @@ OTHER=107
 '>'=15
 '>='=16
 '!='=17
-'{'=18
-'}'=19
-':'=20
+':'=18
+'{'=19
+'}'=20
 '?'=96
 ';'=106

integration-tests/basicselect_test.go

@@ -0,0 +1,57 @@
+package integration_tests
+
+import (
+	"fmt"
+	"github.com/datastax/zdm-proxy/integration-tests/env"
+	"github.com/datastax/zdm-proxy/integration-tests/setup"
+	"github.com/datastax/zdm-proxy/integration-tests/utils"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestSaiSelect(t *testing.T) {
+	if !env.RunCcmTests {
+		t.Skip("Test requires CCM, set RUN_CCMTESTS env variable to TRUE")
+	}
+	if !(env.IsDse && env.CompareServerVersion("6.9") >= 0) {
+		t.Skip("Test requires DSE 6.9 cluster")
+	}
+
+	proxyInstance, err := NewProxyInstanceForGlobalCcmClusters()
+	require.Nil(t, err)
+	defer proxyInstance.Shutdown()
+
+	originCluster, targetCluster, err := SetupOrGetGlobalCcmClusters()
+	require.Nil(t, err)
+
+	// Initialize test data
+	dataIds1 := []string{
+		"43bfd159-dede-47bf-a4da-6d446065e618",
+		"17a424d1-e611-47e2-8bcd-2cec885edf28"}
+	dataTasks1 := []string{
+		"Mile",
+		"Mission"}
+
+	// Seed originCluster and targetCluster w/ schema and data
+	setup.SeedData(originCluster.GetSession(), targetCluster.GetSession(), setup.TestTable3, dataIds1, dataTasks1)
+
+	// Connect to proxy as a "client"
+	proxy, err := utils.ConnectToCluster("127.0.0.1", "", "", 14002)
+
+	if err != nil {
+		t.Log("Unable to connect to proxy session.")
+		t.Fatal(err)
+	}
+	defer proxy.Close()
+
+	// run query on proxied connection
+	// use lowercased version of the actual value
+	itr := proxy.Query(fmt.Sprintf("SELECT * FROM %s.%s where task : 'mile';", setup.TestKeyspace, setup.TestTable3)).Iter()
+	row := make(map[string]interface{})
+
+	// Assertions!
+	require.True(t, itr.MapScan(row))
+	task := setup.MapToTask(row)
+
+	setup.AssertEqual(t, "Mile", task.Task)
+}

integration-tests/ccm/ccm.go

@@ -57,9 +57,12 @@ func Create(name string, version string, isDse bool) (string, error) {
 
 }
 
-func Add(seed bool, address string, remoteDebugPort int, jmxPort int, name string) (string, error) {
+func Add(seed bool, address string, remoteDebugPort int, jmxPort int, name string, isDse bool) (string, error) {
 	var addArgs = []string{
 		"-i", address, "-r", fmt.Sprintf("%d", remoteDebugPort), "-j", fmt.Sprintf("%d", jmxPort), name}
+	if isDse {
+		addArgs = append(addArgs, "--dse")
+	}
 	if seed {
 		return execCcm(append([]string{"add", "-s"}, addArgs...)...)
 	} else {

integration-tests/ccm/cluster.go

@@ -74,7 +74,8 @@ func (ccmCluster *Cluster) Create(numberOfNodes int, start bool) error {
 			fmt.Sprintf("127.0.0.%d", nodeIndex),
 			2000+nodeIndex*100,
 			7000+nodeIndex*100,
-			fmt.Sprintf("node%d", nodeIndex))
+			fmt.Sprintf("node%d", nodeIndex),
+			ccmCluster.isDse)
 
 		if err != nil {
 			Remove(ccmCluster.name)
@@ -152,7 +153,8 @@ func (ccmCluster *Cluster) AddNode(index int) error {
 		fmt.Sprintf("127.0.0.%d", nodeIndex),
 		2000+nodeIndex*100,
 		7000+nodeIndex*100,
-		fmt.Sprintf("node%d", nodeIndex))
+		fmt.Sprintf("node%d", nodeIndex),
+		ccmCluster.isDse)
 	return err
 }
 

integration-tests/env/vars.go

@@ -87,6 +87,42 @@ func InitGlobalVars() {
 	}
 }
 
+func CompareServerVersion(version string) int {
+	v1 := parseVersion(ServerVersion)
+	v2 := parseVersion(version)
+	return compareVersion(v1, v2)
+}
+
+func parseVersion(version string) []int {
+	// remove optional suffix (e.g. 5.0-beta1 becomes 5.0) and split version segments
+	segmentsStr := strings.Split(strings.Split(version, "-")[0], ".")
+	segments := make([]int, len(segmentsStr))
+	for i, str := range segmentsStr {
+		val, err := strconv.Atoi(str)
+		if err != nil {
+			return []int{0, 0, 0}
+		}
+		segments[i] = val
+	}
+	// if we have less than 3 segments, pad with zeros
+	for i := len(segments); i < 3; i++ {
+		segments = append(segments, 0)
+	}
+	return segments
+}
+
+func compareVersion(v1 []int, v2 []int) int {
+	for i := 0; i < len(v1); i++ {
+		if v1[i] == v2[i] {
+			continue
+		} else if v1[i] < v2[i] {
+			return -1
+		}
+		return 1
+	}
+	return 0
+}
+
 func getEnvironmentVariableOrDefault(key string, defaultValue string) string {
 	if value, ok := os.LookupEnv(key); ok {
 		return value

integration-tests/setup/constants.go

@@ -9,5 +9,23 @@ const TestTable = "tasks"
 // TestTable2 is another dedicated table for testing
 const TestTable2 = "people"
 
+// TestTable3 is dedicated table for custom data types testing
+const TestTable3 = "cyclist"
+
 // TestTables is an array of dedicated tables for testing
-var TestTables = [...]string{TestTable, TestTable2}
+var TestTables = [...]string{TestTable, TestTable2, TestTable3}
+
+var TestTableDefinitions = map[string][]string{
+	TestTable:  {"CREATE TABLE %s.%s(id UUID, task text, PRIMARY KEY(id));"},
+	TestTable2: {"CREATE TABLE %s.%s(id UUID, task text, PRIMARY KEY(id));"},
+	TestTable3: {
+		"CREATE TABLE %s.%s(id UUID, task text, PRIMARY KEY(id));",
+		"CREATE CUSTOM INDEX example_sai_idx ON %s.%s(task) USING 'StorageAttachedIndex' WITH OPTIONS = {'case_sensitive': 'false'};",
+	},
+}
+
+var TestTableInserts = map[string]string{
+	TestTable:  "INSERT INTO %s.%s(id, task) VALUES (%s, '%s');",
+	TestTable2: "INSERT INTO %s.%s(id, task) VALUES (%s, '%s');",
+	TestTable3: "INSERT INTO %s.%s(id, task) VALUES (%s, '%s');",
+}

integration-tests/setup/data.go

@@ -43,20 +43,24 @@ func SeedData(source *gocql.Session, dest *gocql.Session, table string, dataIds
 
 	log.Info("Seeding tables...")
 	// Create the table in source
-	err = source.Query(fmt.Sprintf("CREATE TABLE %s.%s(id UUID, task text, PRIMARY KEY(id));", TestKeyspace, table)).Exec()
-	if err != nil {
-		log.WithError(err).Error("Error creating table in source cluster.")
+	for _, statement := range TestTableDefinitions[table] {
+		err = source.Query(fmt.Sprintf(statement, TestKeyspace, table)).Exec()
+		if err != nil {
+			log.WithError(err).Error("Error creating table in source cluster.")
+		}
 	}
 
-	err = dest.Query(fmt.Sprintf("CREATE TABLE %s.%s(id UUID, task text, PRIMARY KEY(id));", TestKeyspace, table)).Exec()
-	if err != nil {
-		log.WithError(err).Error("Error creating table in dest cluster.")
+	for _, statement := range TestTableDefinitions[table] {
+		err = dest.Query(fmt.Sprintf(statement, TestKeyspace, table)).Exec()
+		if err != nil {
+			log.WithError(err).Error("Error creating table in dest cluster.")
+		}
 	}
 
 	// Seed the rows
 	for i := 0; i < len(dataIds); i++ {
 		id, task := dataIds[i], dataEntries[i]
-		err = source.Query(fmt.Sprintf("INSERT INTO %s.%s(id, task) VALUES (%s, '%s');", TestKeyspace, table, id, task)).Exec()
+		err = source.Query(fmt.Sprintf(TestTableInserts[table], TestKeyspace, table, id, task)).Exec()
 		if err != nil {
 			log.WithError(err).Error("Error inserting into table for source cluster.")
 		}

proxy/pkg/zdmproxy/cqlparser_search_test.go

@@ -39,6 +39,10 @@ func TestParseAndInspect_SearchRequests(t *testing.T) {
 			"select * from person where solr_query='{\"q\" : \"*:*\", \"distrib.singlePass\" : true}';",
 			NewGenericRequestInfo(forwardToOrigin, true, true),
 		},
+		{"Query using SAI fuzzy search operator",
+			"select * from person where nick_name:'fuzzy';",
+			NewGenericRequestInfo(forwardToOrigin, true, true),
+		},
 	}
 
 	for _, tt := range tests {