Address #15 join_names_with_dot and function_def_decorators. Idea for…

… Union and Array types in the grammar patch

.gitignore

@@ -2,4 +2,4 @@
 dist/*
 *__pycache__
 .idea/
-*tmp.txt
+*tmp.*

src/parser/generated_parser.ts

@@ -47,12 +47,10 @@ function CHECK_VERSION(i: number, msg: string, ret: any) {
 }
 
 /** @todo */
-function CHECK_NULL_ALLOWED(p: Parser, result: any) {
+function CHECK_NULL_ALLOWED(result: any) {
     return result;
 }
 
-type KeywordOrStarredArray = KeywordOrStarred[];
-
 export class GeneratedParser extends Parser {
     start_rule: StartRule;
     flags: number;
@@ -1090,7 +1088,7 @@ export class GeneratedParser extends Parser {
     }
 
     @memoize
-    function_def(): stmt | null {
+    function_def(): FunctionDef | AsyncFunctionDef | null {
         // function_def: decorators function_def_raw | function_def_raw
         let d, f, function_def_raw;
         const mark = this.mark();
@@ -1107,7 +1105,7 @@ export class GeneratedParser extends Parser {
     }
 
     @memoize
-    function_def_raw(): stmt | null {
+    function_def_raw(): FunctionDef | AsyncFunctionDef | null {
         // function_def_raw: 'def' NAME '(' params? ')' ['->' expression] ':' func_type_comment? block | ASYNC 'def' NAME '(' params? ')' ['->' expression] ':' func_type_comment? block
         let a, async, b, keyword, literal, literal_1, literal_2, n, params, tc;
         const mark = this.mark();
@@ -2879,7 +2877,7 @@ export class GeneratedParser extends Parser {
     }
 
     @memoize
-    kwargs(): KeywordOrStarredArray | null {
+    kwargs(): KeywordOrStarred[] | null {
         // kwargs: ','.kwarg_or_starred+ ',' ','.kwarg_or_double_starred+ | ','.kwarg_or_starred+ | ','.kwarg_or_double_starred+
         let _gather_112, _gather_114, a, b, literal;
         const mark = this.mark();
@@ -2942,12 +2940,12 @@ export class GeneratedParser extends Parser {
         const mark = this.mark();
         if ((a = this.name()) && (literal = this.expect("=")) && (b = this.expression())) {
             const EXTRA = this.extra(mark);
-            return pegen.keyword_or_starred(this, CHECK(new astnodes.keyword(a.id, b, ...EXTRA)), 1);
+            return pegen.keyword_or_starred(this, CHECK(new astnodes.keyword(a.id, b, ...EXTRA)), true);
         }
         this.reset(mark);
         if ((literal = this.expect("**")) && (a = this.expression())) {
             const EXTRA = this.extra(mark);
-            return pegen.keyword_or_starred(this, CHECK(new astnodes.keyword(null, a, ...EXTRA)), 1);
+            return pegen.keyword_or_starred(this, CHECK(new astnodes.keyword(null, a, ...EXTRA)), true);
         }
         this.reset(mark);
         if ((invalid_kwarg = this.invalid_kwarg())) {

src/parser/pegen.ts

@@ -16,6 +16,8 @@ import {
     arguments_,
     arg,
     keyword,
+    FunctionDef,
+    AsyncFunctionDef,
 } from "../ast/astnodes.ts";
 import { DOT, ELLIPSIS, NAME } from "../tokenize/token.ts";
 import type { TokenInfo } from "../tokenize/tokenize.ts";
@@ -1467,7 +1469,20 @@ export function seq_flatten(p: Parser, seqs: AST[][]): AST[] {
 //     return flattened_seq;
 // }
 
-// /* Creates a new name of the form <first_name>.<second_name> */
+/* Creates a new name of the form <first_name>.<second_name> */
+export function join_names_with_dot(p: Parser, first_name: Name, second_name: Name): Name {
+    const first_identifier = first_name.id;
+    const second_identifier = second_name.id;
+    /** @todo if we make these pyStrings we'll have to change this */
+    return new Name(
+        first_identifier + "." + second_identifier,
+        Load,
+        first_name.lineno,
+        first_name.col_offset,
+        second_name.end_lineno,
+        second_name.end_col_offset
+    );
+}
 // expr_ty
 // _PyPegen_join_names_with_dot(Parser *p, expr_ty first_name, expr_ty second_name)
 // {
@@ -2217,7 +2232,26 @@ export function augoperator(p: Parser, kind: operator) {
 //     return a;
 // }
 
-// /* Construct a FunctionDef equivalent to function_def, but with decorators */
+/* Construct a FunctionDef equivalent to function_def, but with decorators */
+export function function_def_decorators<T extends FunctionDef | AsyncFunctionDef>(
+    p: Parser,
+    decorators: expr[],
+    fdef: T
+): T extends FunctionDef ? FunctionDef : AsyncFunctionDef {
+    assert(fdef !== null);
+    return new (fdef.constructor as typeof FunctionDef | typeof AsyncFunctionDef)(
+        fdef.name,
+        fdef.args,
+        fdef.body,
+        decorators,
+        fdef.returns,
+        fdef.type_comment,
+        fdef.lineno,
+        fdef.col_offset,
+        fdef.end_lineno,
+        fdef.end_col_offset
+    );
+}
 // stmt_ty
 // _PyPegen_function_def_decorators(Parser *p, asdl_seq *decorators, stmt_ty function_def)
 // {
@@ -2305,12 +2339,8 @@ export function keyword_or_starred(p: Parser, element: keyword, is_keyword: bool
 //     return n;
 // }
 
-function isKeyword(kw: expr | keyword): kw is keyword {
-    return kw instanceof keyword;
-}
-
 export function seq_extract_starred_exprs(p: Parser, kwargs: KeywordOrStarred[]): expr[] {
-    return kwargs.map((kw) => kw.element).filter((kw) => !isKeyword(kw));
+    return kwargs.filter((kw) => !kw.is_keyword).map((kw) => kw.element);
 }
 
 // /* Extract the starred expressions of an asdl_seq* of KeywordOrStarred*s */
@@ -2337,7 +2367,7 @@ export function seq_extract_starred_exprs(p: Parser, kwargs: KeywordOrStarred[])
 // }
 
 export function seq_delete_starred_exprs(p: Parser, kwargs: KeywordOrStarred[]): keyword[] {
-    return kwargs.map((kw) => kw.element).filter(isKeyword);
+    return kwargs.filter((kw) => kw.is_keyword).map((kw) => kw.element as keyword);
 }
 
 // /* Return a new asdl_seq* with only the keywords in kwargs */

tools/gen_parser/skulpt_parser_genarator.py

@@ -65,12 +65,10 @@
 }}
 
 /** @todo */
-function CHECK_NULL_ALLOWED(p: Parser, result: any) {{
+function CHECK_NULL_ALLOWED(result: any) {{
     return result;
 }}
 
-type KeywordOrStarredArray = KeywordOrStarred[]
-
 """
 
 MODULE_SUFFIX = """
@@ -134,6 +132,11 @@ def fix_reserved(name):
     return name + "_" if name in reserved else name
 
 
+def clean_type(type):
+    # because grammar doesn't allow certain characters
+    return type.replace("_UNION_", "|").replace("_ARRAY", "[]")
+
+
 non_exact_tok = (
     "AWAIT",
     "OP",
@@ -356,7 +359,7 @@ def visit_Rule(self, node: Rule) -> None:
                 self.print("@logger")
         else:
             self.print("@memoize")
-        node_type = node.type or "any"
+        node_type = clean_type(node.type or "any")
         self.print(f"{node.name}(): {node_type} | null {{")  # -> Optional[{node_type}] {{")
         with self.indent():
             self.print(f"// {node.name}: {rhs}")

tools/patch/grammar.patch

@@ -1,5 +1,5 @@
 diff --git a/Grammar/python.gram b/Grammar/python.gram
-index 64e205e7fd..ddc442a97b 100644
+index 64e205e7fd..c9bea63124 100644
 --- a/Grammar/python.gram
 +++ b/Grammar/python.gram
 @@ -2,7 +2,7 @@
@@ -296,12 +296,12 @@ index 64e205e7fd..ddc442a97b 100644
 
 -function_def[stmt_ty]:
 -    | d=decorators f=function_def_raw { _PyPegen_function_def_decorators(p, d, f) }
-+function_def[stmt]:
++function_def[FunctionDef_UNION_AsyncFunctionDef]:
 +    | d=decorators f=function_def_raw { pegen.function_def_decorators(this, d, f) }
      | function_def_raw
 
 -function_def_raw[stmt_ty]:
-+function_def_raw[stmt]:
++function_def_raw[FunctionDef_UNION_AsyncFunctionDef]:
      | 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block {
 -        _Py_FunctionDef(n->v.Name.id,
 -                        (params) ? params : CHECK(_PyPegen_empty_arguments(p)),
@@ -828,7 +828,7 @@ index 64e205e7fd..ddc442a97b 100644
 +                          CHECK_NULL_ALLOWED(pegen.seq_extract_starred_exprs(this, a)),
 +                          CHECK_NULL_ALLOWED(pegen.seq_delete_starred_exprs(this, a)),
 +                          ...EXTRA) }
-+kwargs[KeywordOrStarredArray]:
++kwargs[KeywordOrStarred_ARRAY]:
 +    | a=','.kwarg_or_starred+ ',' b=','.kwarg_or_double_starred+ { pegen.join_sequences(this, a, b) }
      | ','.kwarg_or_starred+
      | ','.kwarg_or_double_starred+
@@ -849,8 +849,8 @@ index 64e205e7fd..ddc442a97b 100644
      | a=NAME '=' b=expression {
 -        _PyPegen_keyword_or_starred(p, CHECK(_Py_keyword(a->v.Name.id, b, EXTRA)), 1) }
 -    | '**' a=expression { _PyPegen_keyword_or_starred(p, CHECK(_Py_keyword(NULL, a, EXTRA)), 1) }
-+        pegen.keyword_or_starred(this, CHECK(new astnodes.keyword(a.id, b, ...EXTRA)), 1) }
-+    | '**' a=expression { pegen.keyword_or_starred(this, CHECK(new astnodes.keyword(null, a, ...EXTRA)), 1) }
++        pegen.keyword_or_starred(this, CHECK(new astnodes.keyword(a.id, b, ...EXTRA)), true) }
++    | '**' a=expression { pegen.keyword_or_starred(this, CHECK(new astnodes.keyword(null, a, ...EXTRA)), true) }
      | invalid_kwarg
 
  # NOTE: star_targets may contain *bitwise_or, targets may not.