Add a null token

TomWright · Oct 1, 2024 · b792aaf · b792aaf
1 parent 5058ae6
commit b792aaf
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 31 deletions.
diff --git a/selector/lexer/token.go b/selector/lexer/token.go
@@ -46,6 +46,7 @@ const (
 	LessThan
 	LessThanOrEqual
 	Exclamation
+	Null
 )
 
 type Tokens []Token

diff --git a/selector/lexer/tokenize.go b/selector/lexer/tokenize.go
@@ -172,6 +172,9 @@ func (p *Tokenizer) parseCurRune() (Token, error) {
 	default:
 		pos := p.i
 
+		if pos+3 < p.srcLen && strings.EqualFold(p.src[pos:pos+4], "null") {
+			return NewToken(Null, p.src[pos:pos+4], p.i, 4), nil
+		}
 		if pos+3 < p.srcLen && strings.EqualFold(p.src[pos:pos+4], "true") {
 			return NewToken(Bool, p.src[pos:pos+4], p.i, 4), nil
 		}

diff --git a/selector/lexer/tokenize_test.go b/selector/lexer/tokenize_test.go
@@ -38,44 +38,17 @@ func TestTokenizer_Parse(t *testing.T) {
 	}))
 
 	t.Run("everything", runTest(testCase{
-		in: "foo.bar.baz[1] != 42.123 || foo.bar.baz['hello'] == 42 && x == 'a\\'b' + false . .... asd... $name",
+		in: "foo.bar.baz[1] != 42.123 || foo.bar.baz['hello'] == 42 && x == 'a\\'b' + false true . .... asd... $name null",
 		out: []TokenKind{
 			Symbol, Dot, Symbol, Dot, Symbol, OpenBracket, Number, CloseBracket, NotEqual, Number,
 			Or,
 			Symbol, Dot, Symbol, Dot, Symbol, OpenBracket, String, CloseBracket, Equal, Number,
 			And,
 			Symbol, Equal, String,
-			Plus, Bool,
+			Plus, Bool, Bool,
 			Dot, Spread, Dot,
 			Symbol, Spread,
-			Variable,
+			Variable, Null,
 		},
 	}))
-
-	tok := NewTokenizer("foo.bar.baz[1] != 42.123 || foo.bar.baz['hello'] == 42 && x == 'a\\'b' + false . .... asd... $name")
-	tokens, err := tok.Tokenize()
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-	exp := []TokenKind{
-		Symbol, Dot, Symbol, Dot, Symbol, OpenBracket, Number, CloseBracket, NotEqual, Number,
-		Or,
-		Symbol, Dot, Symbol, Dot, Symbol, OpenBracket, String, CloseBracket, Equal, Number,
-		And,
-		Symbol, Equal, String,
-		Plus, Bool,
-		Dot, Spread, Dot,
-		Symbol, Spread,
-		Variable,
-	}
-	if len(tokens) != len(exp) {
-		t.Fatalf("unexpected number of tokens: %d", len(tokens))
-	}
-
-	for i := range tokens {
-		if tokens[i].Kind != exp[i] {
-			t.Errorf("unexpected token kind at position %d: exp %v, got %v", i, exp[i], tokens[i].Kind)
-			return
-		}
-	}
 }
diff --git a/selector/parser/denotations.go b/selector/parser/denotations.go
@@ -44,7 +44,7 @@ var tokenBindingPowers = map[lexer.TokenKind]bindingPower{
 	lexer.String: bpLiteral,
 	lexer.Number: bpLiteral,
 	lexer.Bool:   bpLiteral,
-	//lexer.Null:             bpLiteral,
+	lexer.Null:   bpLiteral,
 
 	lexer.Variable:    bpProperty,
 	lexer.Dot:         bpProperty,