From b792aafcb296aa2adfca95546536d955fe5dccb7 Mon Sep 17 00:00:00 2001
From: Tom Wright <contact@tomwright.me>
Date: Tue, 1 Oct 2024 01:46:08 +0100
Subject: [PATCH] Add a null token

---
 selector/lexer/token.go         |  1 +
 selector/lexer/tokenize.go      |  3 +++
 selector/lexer/tokenize_test.go | 33 +++------------------------------
 selector/parser/denotations.go  |  2 +-
 4 files changed, 8 insertions(+), 31 deletions(-)

diff --git a/selector/lexer/token.go b/selector/lexer/token.go
index 2c9f360..f86167c 100644
--- a/selector/lexer/token.go
+++ b/selector/lexer/token.go
@@ -46,6 +46,7 @@ const (
 	LessThan
 	LessThanOrEqual
 	Exclamation
+	Null
 )
 
 type Tokens []Token
diff --git a/selector/lexer/tokenize.go b/selector/lexer/tokenize.go
index 23d200a..82c5eb7 100644
--- a/selector/lexer/tokenize.go
+++ b/selector/lexer/tokenize.go
@@ -172,6 +172,9 @@ func (p *Tokenizer) parseCurRune() (Token, error) {
 	default:
 		pos := p.i
 
+		if pos+3 < p.srcLen && strings.EqualFold(p.src[pos:pos+4], "null") {
+			return NewToken(Null, p.src[pos:pos+4], p.i, 4), nil
+		}
 		if pos+3 < p.srcLen && strings.EqualFold(p.src[pos:pos+4], "true") {
 			return NewToken(Bool, p.src[pos:pos+4], p.i, 4), nil
 		}
diff --git a/selector/lexer/tokenize_test.go b/selector/lexer/tokenize_test.go
index 3d0401d..add571a 100644
--- a/selector/lexer/tokenize_test.go
+++ b/selector/lexer/tokenize_test.go
@@ -38,44 +38,17 @@ func TestTokenizer_Parse(t *testing.T) {
 	}))
 
 	t.Run("everything", runTest(testCase{
-		in: "foo.bar.baz[1] != 42.123 || foo.bar.baz['hello'] == 42 && x == 'a\\'b' + false . .... asd... $name",
+		in: "foo.bar.baz[1] != 42.123 || foo.bar.baz['hello'] == 42 && x == 'a\\'b' + false true . .... asd... $name null",
 		out: []TokenKind{
 			Symbol, Dot, Symbol, Dot, Symbol, OpenBracket, Number, CloseBracket, NotEqual, Number,
 			Or,
 			Symbol, Dot, Symbol, Dot, Symbol, OpenBracket, String, CloseBracket, Equal, Number,
 			And,
 			Symbol, Equal, String,
-			Plus, Bool,
+			Plus, Bool, Bool,
 			Dot, Spread, Dot,
 			Symbol, Spread,
-			Variable,
+			Variable, Null,
 		},
 	}))
-
-	tok := NewTokenizer("foo.bar.baz[1] != 42.123 || foo.bar.baz['hello'] == 42 && x == 'a\\'b' + false . .... asd... $name")
-	tokens, err := tok.Tokenize()
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-	exp := []TokenKind{
-		Symbol, Dot, Symbol, Dot, Symbol, OpenBracket, Number, CloseBracket, NotEqual, Number,
-		Or,
-		Symbol, Dot, Symbol, Dot, Symbol, OpenBracket, String, CloseBracket, Equal, Number,
-		And,
-		Symbol, Equal, String,
-		Plus, Bool,
-		Dot, Spread, Dot,
-		Symbol, Spread,
-		Variable,
-	}
-	if len(tokens) != len(exp) {
-		t.Fatalf("unexpected number of tokens: %d", len(tokens))
-	}
-
-	for i := range tokens {
-		if tokens[i].Kind != exp[i] {
-			t.Errorf("unexpected token kind at position %d: exp %v, got %v", i, exp[i], tokens[i].Kind)
-			return
-		}
-	}
 }
diff --git a/selector/parser/denotations.go b/selector/parser/denotations.go
index c976cf6..49eed60 100644
--- a/selector/parser/denotations.go
+++ b/selector/parser/denotations.go
@@ -44,7 +44,7 @@ var tokenBindingPowers = map[lexer.TokenKind]bindingPower{
 	lexer.String: bpLiteral,
 	lexer.Number: bpLiteral,
 	lexer.Bool:   bpLiteral,
-	//lexer.Null:             bpLiteral,
+	lexer.Null:   bpLiteral,
 
 	lexer.Variable:    bpProperty,
 	lexer.Dot:         bpProperty,