Merge commit '7dc4fb60390218b09bc351062eeede7dcdbb4d9f'

CHANGELOG.md

@@ -2,7 +2,7 @@
 
 ### Parsing
 
-Updated Elixir, Erlang and Go parsers.
+Updated Elixir, Erlang, Go and Racket parsers.
 
 ### Display
 

build.rs

@@ -298,7 +298,7 @@ fn main() {
         TreeSitterParser {
             name: "tree-sitter-racket",
             src_dir: "vendored_parsers/tree-sitter-racket-src",
-            extra_files: vec!["scanner.cc"],
+            extra_files: vec!["scanner.c"],
         },
         TreeSitterParser {
             name: "tree-sitter-ruby",

vendored_parsers/tree-sitter-racket/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "tree-sitter-racket"
 description = "racket grammar for the tree-sitter parsing library"
-version = "0.0.1"
+version = "0.3.0"
 keywords = ["incremental", "parsing", "racket"]
 categories = ["parsing", "text-editors"]
 repository = "https://github.com/tree-sitter/tree-sitter-racket"

vendored_parsers/tree-sitter-racket/README.md

@@ -8,15 +8,20 @@ This grammar only implements the Racket language with the default readtable.
 
 ## Status
 
-It should recognize most grammar with the default readtable.
+It should be complete and compatible with Racket 8.9.
+
+There are no plans to add support for new language currently.
+
+## News
+
+Starting from June 24, 2023, ([commit](https://github.com/6cdh/tree-sitter-racket/commit/989c3e631a7f2d87bb6a66a5394870aaeb6c56e7)) or release 0.3.0, the external scanner was written in C.
 
 ## Build and Try
 
 You need
 
 * nodejs
 * a C compiler
-* a C++11 compiler
 
 then run
 

vendored_parsers/tree-sitter-racket/binding.gyp

@@ -9,8 +9,7 @@
       "sources": [
         "bindings/node/binding.cc",
         "src/parser.c",
-        "src/scanner.cc",
-        # If your language uses an external scanner, add it here.
+        "src/scanner.c",
       ],
       "cflags_c": [
         "-std=c99",

vendored_parsers/tree-sitter-racket/bindings/rust/build.rs

@@ -2,24 +2,18 @@ fn main() {
     let src_dir = std::path::Path::new("src");
 
     let mut c_config = cc::Build::new();
-    c_config.include(&src_dir);
+    c_config.include(src_dir);
     c_config
         .flag_if_supported("-Wno-unused-parameter")
         .flag_if_supported("-Wno-unused-but-set-variable")
         .flag_if_supported("-Wno-trigraphs");
     let parser_path = src_dir.join("parser.c");
     c_config.file(&parser_path);
-    c_config.compile("parser");
-    println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());
 
-    let mut cpp_config = cc::Build::new();
-    cpp_config.cpp(true);
-    cpp_config.include(&src_dir);
-    cpp_config
-        .flag_if_supported("-Wno-unused-parameter")
-        .flag_if_supported("-Wno-unused-but-set-variable");
-    let scanner_path = src_dir.join("scanner.cc");
-    cpp_config.file(&scanner_path);
-    cpp_config.compile("scanner");
+    let scanner_path = src_dir.join("scanner.c");
+    c_config.file(&scanner_path);
     println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
+
+    println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());
+    c_config.compile("parser");
 }

vendored_parsers/tree-sitter-racket/corpus/here_string/.gitattributes

@@ -1,2 +1,2 @@
-win_* text eol=crlf
+win_* eol=crlf
 * eol=lf

vendored_parsers/tree-sitter-racket/corpus/simple.rkt

@@ -31,6 +31,8 @@ number
 #b101
 +inf.t
 -nan.t
++i
+0##+i
 
 ---
 
@@ -47,6 +49,8 @@ number
   (number)
   (number)
   (number)
+  (number)
+  (number)
   (number))
 
 ===
@@ -79,6 +83,8 @@ Ap\ ple
 app123app123
 123app123
 中文
+a
+fec
 
 ---
 (program
@@ -92,6 +98,8 @@ app123app123
   (symbol)
   (symbol)
   (symbol)
+  (symbol)
+  (symbol)
   (symbol))
 
 ===

vendored_parsers/tree-sitter-racket/fuzztest/.gitignore

@@ -0,0 +1 @@
+*.txt

vendored_parsers/tree-sitter-racket/fuzztest/README.md

@@ -0,0 +1,34 @@
+# fuzz test
+
+The directory contains the scripts that test the implementation to avoid the problem that a symbol is parsed as a number or vice versa.
+
+## Resource
+
+* ~6 minutes to run
+* ~300M generated files
+
+## Run
+
+```shell
+$ cd fuzztest
+# 30s
+$ racket gen_cases.rkt
+cpu time: 26531 real time: 26835 gc time: 1069
+5114173 cases generated
+$ cd ..
+$ tree-sitter generate
+# 3 minutes
+$ tree-sitter parse fuzztest/case.txt > fuzztest/res1.txt
+# 2 minutes
+$ cd fuzztest && racket postprocess.rkt
+# should show nothing
+$ sdiff -s <(cat -n expect.txt) <(cat -n res.txt)
+
+# If there is some error, run
+$ sdiff -s <(cat -n expect.txt) <(cat -n res.txt) | less
+# then get the first error case at `N`-th line
+$ cat case.txt | sed -n 'Np'
+```
+
+You can edit `gen_cases.rkt` to generate less cases during development.
+

vendored_parsers/tree-sitter-racket/fuzztest/gen_cases.rkt

@@ -0,0 +1,57 @@
+#lang racket
+
+(require racket/extflonum)
+
+;; all characters that can appear in a valid number/exflonum
+;; remove some insignificant parts to improve performance
+;; (define alphabet-char "abdefilnost")
+(define alphabet-char "abdefilnostx")
+(define special-char "#./@+-")
+;; (define numeric-char "0123456789")
+(define numeric-char "0179")
+(define all-char
+  (string-append alphabet-char
+                 special-char
+                 numeric-char))
+
+(define cnt 0)
+(define max-len 5)
+
+(define case-port (open-output-file "case.txt" #:exists 'replace))
+(define expect-port (open-output-file "expect.txt" #:exists 'replace))
+
+(define (gen i case)
+  (with-handlers ([exn:fail? (lambda _ (void))])
+    (when (> i 0)
+      (define case-str (list->string case))
+      ;; "[email protected]" should be a number according the document,
+      ;; but it's actually a symbol.
+      ;; It's a bug of Racket reader, and will fix in new Racket release.
+      ;; we skip these cases.
+      (when (not (string-contains? case-str "@."))
+        (with-handlers ([exn:fail? void])
+          (with-input-from-string case-str
+            (lambda ()
+              (define fst (read))
+              (define snd (read))
+              (when (eof-object? snd)
+                (cond [(symbol? fst)
+                       (set! cnt (add1 cnt))
+                       (displayln case-str case-port)
+                       (displayln "symbol" expect-port)]
+                      [(number? fst)
+                       (set! cnt (add1 cnt))
+                       (displayln case-str case-port)
+                       (displayln "number" expect-port)]
+                      ;; it's here for possible future change that
+                      ;; split extflonum from number
+                      [(extflonum? fst)
+                       (set! cnt (add1 cnt))
+                       (displayln case-str case-port)
+                       (displayln "number" expect-port)]))))))))
+  (when (< i max-len)
+    (for ([c all-char])
+      (gen (add1 i) (cons c case)))))
+
+(time (gen 0 '()))
+(displayln (format "~a cases generated" cnt))

vendored_parsers/tree-sitter-racket/fuzztest/postprocess.rkt

@@ -0,0 +1,26 @@
+#lang racket
+
+(define port (open-input-file "res1.txt"))
+(define all-result (drop (read port) 4))
+(define all-result-line
+  (for/list ([r all-result])
+    (cons (car r) (caadr r))))
+(with-output-to-file "res.txt"
+  #:exists 'replace
+  (lambda ()
+    (let loop ([line 0]
+               [firstline? #t]
+               [lst all-result-line])
+      (match lst
+        ['() (void)]
+        [(cons fst rem)
+         #:when (= (cdr fst) line)
+         (when (not firstline?)
+           (display " "))
+         (display (car fst))
+         (loop line (if firstline? #t #f) rem)]
+        [(cons fst rem)
+         (newline)
+         (display (car fst))
+         (loop (add1 line) #f rem)]))
+    (newline)))