diff --git a/.gitignore b/.gitignore index 23f3d3e..2116452 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,4 @@ dist/* *__pycache__ .idea/ -*tmp.txt +*tmp.* diff --git a/src/parser/generated_parser.ts b/src/parser/generated_parser.ts index 6552261..8c458de 100644 --- a/src/parser/generated_parser.ts +++ b/src/parser/generated_parser.ts @@ -47,12 +47,10 @@ function CHECK_VERSION(i: number, msg: string, ret: any) { } /** @todo */ -function CHECK_NULL_ALLOWED(p: Parser, result: any) { +function CHECK_NULL_ALLOWED(result: any) { return result; } -type KeywordOrStarredArray = KeywordOrStarred[]; - export class GeneratedParser extends Parser { start_rule: StartRule; flags: number; @@ -1090,7 +1088,7 @@ export class GeneratedParser extends Parser { } @memoize - function_def(): stmt | null { + function_def(): FunctionDef | AsyncFunctionDef | null { // function_def: decorators function_def_raw | function_def_raw let d, f, function_def_raw; const mark = this.mark(); @@ -1107,7 +1105,7 @@ export class GeneratedParser extends Parser { } @memoize - function_def_raw(): stmt | null { + function_def_raw(): FunctionDef | AsyncFunctionDef | null { // function_def_raw: 'def' NAME '(' params? ')' ['->' expression] ':' func_type_comment? block | ASYNC 'def' NAME '(' params? ')' ['->' expression] ':' func_type_comment? block let a, async, b, keyword, literal, literal_1, literal_2, n, params, tc; const mark = this.mark(); @@ -2879,7 +2877,7 @@ export class GeneratedParser extends Parser { } @memoize - kwargs(): KeywordOrStarredArray | null { + kwargs(): KeywordOrStarred[] | null { // kwargs: ','.kwarg_or_starred+ ',' ','.kwarg_or_double_starred+ | ','.kwarg_or_starred+ | ','.kwarg_or_double_starred+ let _gather_112, _gather_114, a, b, literal; const mark = this.mark(); @@ -2942,12 +2940,12 @@ export class GeneratedParser extends Parser { const mark = this.mark(); if ((a = this.name()) && (literal = this.expect("=")) && (b = this.expression())) { const EXTRA = this.extra(mark); - return pegen.keyword_or_starred(this, CHECK(new astnodes.keyword(a.id, b, ...EXTRA)), 1); + return pegen.keyword_or_starred(this, CHECK(new astnodes.keyword(a.id, b, ...EXTRA)), true); } this.reset(mark); if ((literal = this.expect("**")) && (a = this.expression())) { const EXTRA = this.extra(mark); - return pegen.keyword_or_starred(this, CHECK(new astnodes.keyword(null, a, ...EXTRA)), 1); + return pegen.keyword_or_starred(this, CHECK(new astnodes.keyword(null, a, ...EXTRA)), true); } this.reset(mark); if ((invalid_kwarg = this.invalid_kwarg())) { diff --git a/src/parser/pegen.ts b/src/parser/pegen.ts index d13a88c..fa098fa 100644 --- a/src/parser/pegen.ts +++ b/src/parser/pegen.ts @@ -16,6 +16,8 @@ import { arguments_, arg, keyword, + FunctionDef, + AsyncFunctionDef, } from "../ast/astnodes.ts"; import { DOT, ELLIPSIS, NAME } from "../tokenize/token.ts"; import type { TokenInfo } from "../tokenize/tokenize.ts"; @@ -1467,7 +1469,20 @@ export function seq_flatten(p: Parser, seqs: AST[][]): AST[] { // return flattened_seq; // } -// /* Creates a new name of the form . */ +/* Creates a new name of the form . */ +export function join_names_with_dot(p: Parser, first_name: Name, second_name: Name): Name { + const first_identifier = first_name.id; + const second_identifier = second_name.id; + /** @todo if we make these pyStrings we'll have to change this */ + return new Name( + first_identifier + "." + second_identifier, + Load, + first_name.lineno, + first_name.col_offset, + second_name.end_lineno, + second_name.end_col_offset + ); +} // expr_ty // _PyPegen_join_names_with_dot(Parser *p, expr_ty first_name, expr_ty second_name) // { @@ -2217,7 +2232,26 @@ export function augoperator(p: Parser, kind: operator) { // return a; // } -// /* Construct a FunctionDef equivalent to function_def, but with decorators */ +/* Construct a FunctionDef equivalent to function_def, but with decorators */ +export function function_def_decorators( + p: Parser, + decorators: expr[], + fdef: T +): T extends FunctionDef ? FunctionDef : AsyncFunctionDef { + assert(fdef !== null); + return new (fdef.constructor as typeof FunctionDef | typeof AsyncFunctionDef)( + fdef.name, + fdef.args, + fdef.body, + decorators, + fdef.returns, + fdef.type_comment, + fdef.lineno, + fdef.col_offset, + fdef.end_lineno, + fdef.end_col_offset + ); +} // stmt_ty // _PyPegen_function_def_decorators(Parser *p, asdl_seq *decorators, stmt_ty function_def) // { @@ -2305,12 +2339,8 @@ export function keyword_or_starred(p: Parser, element: keyword, is_keyword: bool // return n; // } -function isKeyword(kw: expr | keyword): kw is keyword { - return kw instanceof keyword; -} - export function seq_extract_starred_exprs(p: Parser, kwargs: KeywordOrStarred[]): expr[] { - return kwargs.map((kw) => kw.element).filter((kw) => !isKeyword(kw)); + return kwargs.filter((kw) => !kw.is_keyword).map((kw) => kw.element); } // /* Extract the starred expressions of an asdl_seq* of KeywordOrStarred*s */ @@ -2337,7 +2367,7 @@ export function seq_extract_starred_exprs(p: Parser, kwargs: KeywordOrStarred[]) // } export function seq_delete_starred_exprs(p: Parser, kwargs: KeywordOrStarred[]): keyword[] { - return kwargs.map((kw) => kw.element).filter(isKeyword); + return kwargs.filter((kw) => kw.is_keyword).map((kw) => kw.element as keyword); } // /* Return a new asdl_seq* with only the keywords in kwargs */ diff --git a/tools/gen_parser/skulpt_parser_genarator.py b/tools/gen_parser/skulpt_parser_genarator.py index 513de64..332e12c 100644 --- a/tools/gen_parser/skulpt_parser_genarator.py +++ b/tools/gen_parser/skulpt_parser_genarator.py @@ -65,12 +65,10 @@ }} /** @todo */ -function CHECK_NULL_ALLOWED(p: Parser, result: any) {{ +function CHECK_NULL_ALLOWED(result: any) {{ return result; }} -type KeywordOrStarredArray = KeywordOrStarred[] - """ MODULE_SUFFIX = """ @@ -134,6 +132,11 @@ def fix_reserved(name): return name + "_" if name in reserved else name +def clean_type(type): + # because grammar doesn't allow certain characters + return type.replace("_UNION_", "|").replace("_ARRAY", "[]") + + non_exact_tok = ( "AWAIT", "OP", @@ -356,7 +359,7 @@ def visit_Rule(self, node: Rule) -> None: self.print("@logger") else: self.print("@memoize") - node_type = node.type or "any" + node_type = clean_type(node.type or "any") self.print(f"{node.name}(): {node_type} | null {{") # -> Optional[{node_type}] {{") with self.indent(): self.print(f"// {node.name}: {rhs}") diff --git a/tools/patch/grammar.patch b/tools/patch/grammar.patch index 4907dcf..e2e40de 100644 --- a/tools/patch/grammar.patch +++ b/tools/patch/grammar.patch @@ -1,5 +1,5 @@ diff --git a/Grammar/python.gram b/Grammar/python.gram -index 64e205e7fd..ddc442a97b 100644 +index 64e205e7fd..c9bea63124 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -2,7 +2,7 @@ @@ -296,12 +296,12 @@ index 64e205e7fd..ddc442a97b 100644 -function_def[stmt_ty]: - | d=decorators f=function_def_raw { _PyPegen_function_def_decorators(p, d, f) } -+function_def[stmt]: ++function_def[FunctionDef_UNION_AsyncFunctionDef]: + | d=decorators f=function_def_raw { pegen.function_def_decorators(this, d, f) } | function_def_raw -function_def_raw[stmt_ty]: -+function_def_raw[stmt]: ++function_def_raw[FunctionDef_UNION_AsyncFunctionDef]: | 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block { - _Py_FunctionDef(n->v.Name.id, - (params) ? params : CHECK(_PyPegen_empty_arguments(p)), @@ -828,7 +828,7 @@ index 64e205e7fd..ddc442a97b 100644 + CHECK_NULL_ALLOWED(pegen.seq_extract_starred_exprs(this, a)), + CHECK_NULL_ALLOWED(pegen.seq_delete_starred_exprs(this, a)), + ...EXTRA) } -+kwargs[KeywordOrStarredArray]: ++kwargs[KeywordOrStarred_ARRAY]: + | a=','.kwarg_or_starred+ ',' b=','.kwarg_or_double_starred+ { pegen.join_sequences(this, a, b) } | ','.kwarg_or_starred+ | ','.kwarg_or_double_starred+ @@ -849,8 +849,8 @@ index 64e205e7fd..ddc442a97b 100644 | a=NAME '=' b=expression { - _PyPegen_keyword_or_starred(p, CHECK(_Py_keyword(a->v.Name.id, b, EXTRA)), 1) } - | '**' a=expression { _PyPegen_keyword_or_starred(p, CHECK(_Py_keyword(NULL, a, EXTRA)), 1) } -+ pegen.keyword_or_starred(this, CHECK(new astnodes.keyword(a.id, b, ...EXTRA)), 1) } -+ | '**' a=expression { pegen.keyword_or_starred(this, CHECK(new astnodes.keyword(null, a, ...EXTRA)), 1) } ++ pegen.keyword_or_starred(this, CHECK(new astnodes.keyword(a.id, b, ...EXTRA)), true) } ++ | '**' a=expression { pegen.keyword_or_starred(this, CHECK(new astnodes.keyword(null, a, ...EXTRA)), true) } | invalid_kwarg # NOTE: star_targets may contain *bitwise_or, targets may not.