diff --git a/cdd/__init__.py b/cdd/__init__.py index f25e7fad..80edee74 100644 --- a/cdd/__init__.py +++ b/cdd/__init__.py @@ -9,7 +9,7 @@ from logging import getLogger as get_logger __author__ = "Samuel Marks" # type: str -__version__ = "0.0.99rc46" # type: str +__version__ = "0.0.99rc47" # type: str __description__ = ( "Open API to/fro routes, models, and tests. " "Convert between docstrings, classes, methods, argparse, pydantic, and SQLalchemy." diff --git a/cdd/docstring/utils/parse_utils.py b/cdd/docstring/utils/parse_utils.py index 0d075c0e..a059747e 100644 --- a/cdd/docstring/utils/parse_utils.py +++ b/cdd/docstring/utils/parse_utils.py @@ -13,6 +13,7 @@ from cdd.shared.ast_utils import deduplicate from cdd.shared.pure_utils import ( count_iter_items, + pp, simple_types, sliding_window, type_to_name, @@ -56,6 +57,7 @@ ("List", " ", "of"): "List", ("Tuple", " ", "of"): "Tuple", ("Dictionary", " ", "of"): "Mapping", + ("One", " ", "of"): "Union", } @@ -369,8 +371,19 @@ def _parse_adhoc_doc_for_typ_phase0(doc, words): word_chars: str = "{0}{1}`'\"/|".format(string.digits, string.ascii_letters) sentence_ends: int = -1 break_the_union: bool = False # lincoln - for i, ch in enumerate(doc): - if ( + counter = Counter(doc) # Imperfect because won't catch escaped quote marks + balanced_single: bool = counter["'"] > 0 and counter["'"] & 1 == 0 + balanced_double: bool = counter['"'] > 0 and counter['"'] & 1 == 0 + + i: int = 0 + n: int = len(doc) + while i < n: + ch: str = doc[i] + if (ch == "'" and balanced_single or ch == '"' and balanced_double) and ( + i == 0 or doc[i - 1] != "\\" + ): + i = eat_quoted(ch, doc, i, words, n) + elif ( ch in word_chars or ch == "." and len(doc) > (i + 1) @@ -380,14 +393,20 @@ def _parse_adhoc_doc_for_typ_phase0(doc, words): ): words[-1].append(ch) elif ch in frozenset((".", ";", ",")) or ch.isspace(): - words[-1] = "".join(words[-1]) - words.append(ch) + if words[-1]: + words[-1] = "".join(words[-1]) + words.append(ch) + else: + words[-1] = ch if ch == "." and sentence_ends == -1: sentence_ends: int = len(words) elif ch == ";": break_the_union = True words.append([]) + i += 1 words[-1] = "".join(words[-1]) + if not words[-1]: + del words[-1] candidate_type: Optional[str] = next( map( adhoc_type_to_type.__getitem__, @@ -414,4 +433,47 @@ def _parse_adhoc_doc_for_typ_phase0(doc, words): return candidate_type, fst_sentence, sentence +def eat_quoted(ch, doc, chomp_start_idx, words, n): + """ + Chomp from quoted character `ch` to quoted character `ch` + + :param ch: Character of `'` or `"` + :type ch: ```Literal["'", '"']``` + + :param doc: Possibly ambiguous docstring for argument, that *might* hint as to the type + :type doc: ```str``` + + :param chomp_start_idx: chomp_start_idx + :type chomp_start_idx: ```int``` + + :param words: Words + :type words: ```List[Union[List[str], str]]``` + + :param n: Length of `doc` + :type n: ```int``` + + :return: chomp_end_idx + :rtype: ```int``` + """ + chomp_end_idx: int = next( + filter( + lambda _chomp_end_idx: doc[_chomp_end_idx + 1] != ch + or doc[_chomp_end_idx] == "\\", + range(chomp_start_idx, n), + ), + chomp_start_idx, + ) + quoted_str: str = doc[chomp_start_idx:chomp_end_idx] + from operator import iadd + + pp({"b4::words": words, '"".join(words[-1])': "".join(words[-1])}) + ( + iadd(words, (quoted_str, [])) + if len(words[-1]) == 1 and words[-1][-1] == "`" + else iadd(words, ("".join(words[-1]), quoted_str, [])) + ) + pp({"words": words}) + return chomp_end_idx + + __all__ = ["parse_adhoc_doc_for_typ"] # type: list[str] diff --git a/cdd/shared/ast_utils.py b/cdd/shared/ast_utils.py index 39999517..10df9e65 100644 --- a/cdd/shared/ast_utils.py +++ b/cdd/shared/ast_utils.py @@ -427,8 +427,10 @@ def get_default_val(val): ) elif _param.get("typ") == "Str": _param["typ"] = "str" - elif _param.get("typ") in frozenset(("Constant", "NameConstant", "Num")): + elif _param.get("typ") in frozenset(("Constant", "NameConstant")): _param["typ"] = "object" + elif _param.get("typ") == "Num": + _param["typ"] = "float" if "typ" in _param and needs_quoting(_param["typ"]): default = ( _param.get("default") diff --git a/cdd/shared/defaults_utils.py b/cdd/shared/defaults_utils.py index bf9da56a..51ccf389 100644 --- a/cdd/shared/defaults_utils.py +++ b/cdd/shared/defaults_utils.py @@ -228,27 +228,38 @@ def _parse_out_default_and_doc( :rtype: Tuple[str, Optional[str]] """ if typ is not None and typ in simple_types and default not in none_types: - lit = ( - ast.AST() - if typ != "str" - and any( - map( - partial(contains, frozenset(("*", "^", "&", "|", "$", "@", "!"))), - default, + keep_default: bool = False + try: + lit = ( + ast.AST() + if typ != "str" + and any( + map( + partial( + contains, frozenset(("*", "^", "&", "|", "$", "@", "!")) + ), + default, + ) ) + else literal_eval("({default})".format(default=default)) ) - else literal_eval("({default})".format(default=default)) - ) + except ValueError as e: + assert e.args[0].startswith("malformed node or string"), e + keep_default = True default = ( - "```{default}```".format(default=default) - if isinstance(lit, ast.AST) - else { - "bool": bool, - "int": int, - "float": float, - "complex": complex, - "str": str, - }[typ](lit) + default + if keep_default + else ( + "```{default}```".format(default=default) + if isinstance(lit, ast.AST) + else { + "bool": bool, + "int": int, + "float": float, + "complex": complex, + "str": str, + }[typ](lit) + ) ) elif default.isdecimal(): default = int(default) diff --git a/cdd/sqlalchemy/utils/emit_utils.py b/cdd/sqlalchemy/utils/emit_utils.py index 63180b47..d34ad2ba 100644 --- a/cdd/sqlalchemy/utils/emit_utils.py +++ b/cdd/sqlalchemy/utils/emit_utils.py @@ -38,7 +38,6 @@ from json import dumps from operator import attrgetter, eq, methodcaller from os import path -from platform import system from typing import Any, Dict, List, Optional import cdd.shared.ast_utils @@ -79,8 +78,7 @@ def param_to_sqlalchemy_column_calls(name_param, include_name): :return: Iterable of elements in form of: `Column(…)` :rtype: ```Iterable[Call]``` """ - if system() == "Darwin": - print("param_to_sqlalchemy_column_calls::include_name:", include_name, ";") + # if system() == "Darwin": print("param_to_sqlalchemy_column_calls::include_name:", include_name, ";") name, _param = name_param del name_param diff --git a/cdd/sqlalchemy/utils/shared_utils.py b/cdd/sqlalchemy/utils/shared_utils.py index beb062dc..47052495 100644 --- a/cdd/sqlalchemy/utils/shared_utils.py +++ b/cdd/sqlalchemy/utils/shared_utils.py @@ -87,9 +87,12 @@ def update_args_infer_typ_sqlalchemy(_param, args, name, nullable, x_typ_sql): parsed_typ: Call = cast( Call, cdd.shared.ast_utils.get_value(ast.parse(_param["typ"]).body[0]) ) - assert parsed_typ.value.id == "Literal", "Expected `Literal` got: {!r}".format( - parsed_typ.value.id - ) + try: + assert ( + parsed_typ.value.id == "Literal" + ), "Expected `Literal` got: {!r}".format(parsed_typ.value.id) + except AssertionError: + raise val = cdd.shared.ast_utils.get_value(parsed_typ.slice) ( args.append( diff --git a/cdd/tests/mocks/docstrings.py b/cdd/tests/mocks/docstrings.py index 9a2440d5..550e5391 100644 --- a/cdd/tests/mocks/docstrings.py +++ b/cdd/tests/mocks/docstrings.py @@ -500,20 +500,22 @@ ) docstring_google_pytorch_lbfgs_str: str = "\n".join(docstring_google_pytorch_lbfgs) -docstring_google_str: str = ( - """{docstring_header_str} -Args: +docstring_google_args_str: str = """Args: dataset_name (str): name of dataset. Defaults to "mnist" tfds_dir (str): directory to look for models in. Defaults to "~/tensorflow_datasets" K (Literal['np', 'tf']): backend engine, e.g., `np` or `tf`. Defaults to "np" as_numpy (Optional[bool]): Convert to numpy ndarrays data_loader_kwargs (Optional[dict]): pass this as arguments to data_loader function - -Returns: +""" +docstring_google_footer_return_str: str = """Returns: Union[Tuple[tf.data.Dataset, tf.data.Dataset], Tuple[np.ndarray, np.ndarray]]: Train and tests dataset splits. Defaults to (np.empty(0), np.empty(0)) -""".format( - docstring_header_str=docstring_header_str +""" +docstring_google_str: str = "\n".join( + ( + docstring_header_str, + docstring_google_args_str, + docstring_google_footer_return_str, ) ) diff --git a/cdd/tests/test_compound/test_exmod_utils.py b/cdd/tests/test_compound/test_exmod_utils.py index 56cbb0da..168fa45b 100644 --- a/cdd/tests/test_compound/test_exmod_utils.py +++ b/cdd/tests/test_compound/test_exmod_utils.py @@ -57,6 +57,7 @@ def test_emit_file_on_hierarchy(self) -> None: with patch( "cdd.compound.exmod_utils.EXMOD_OUT_STREAM", new_callable=StringIO ), TemporaryDirectory() as tempdir: + output_directory: str = path.join(tempdir, "haz") open(path.join(tempdir, INIT_FILENAME), "a").close() emit_file_on_hierarchy( ("foo.bar", "foo_dir", ir), @@ -65,13 +66,13 @@ def test_emit_file_on_hierarchy(self) -> None: "", True, filesystem_layout="as_input", - output_directory=tempdir, + output_directory=output_directory, first_output_directory=tempdir, no_word_wrap=None, dry_run=False, extra_modules_to_all=None, ) - self.assertTrue(path.isdir(tempdir)) + self.assertTrue(path.isdir(output_directory)) def test__emit_symbols_isfile_emit_filename_true(self) -> None: """Test `_emit_symbol` when `isfile_emit_filename is True`""" diff --git a/cdd/tests/test_marshall_docstring.py b/cdd/tests/test_marshall_docstring.py index 7bbf6e31..b74b53eb 100644 --- a/cdd/tests/test_marshall_docstring.py +++ b/cdd/tests/test_marshall_docstring.py @@ -248,6 +248,25 @@ def test_from_docstring_google_str(self) -> None: _intermediate_repr_no_default_doc["doc"] = docstring_header_str self.assertDictEqual(ir, _intermediate_repr_no_default_doc) + def test_from_docstring_google_str_extra_after(self) -> None: + """ + Tests whether `parse_docstring` produces `intermediate_repr_no_default_doc` + "\n\n\ntrailer" + from `docstring_google_str` + "\n\n\ntrailer" + """ + ir: IntermediateRepr = parse_docstring( + "{}\n\n\ntrailer".format(docstring_google_str) + ) + _intermediate_repr_no_default_doc = deepcopy( + intermediate_repr_no_default_with_nones_doc + ) + _intermediate_repr_no_default_doc["doc"] = "{0}\n\n\n\ntrailer".format( + docstring_header_str + ) + + self.assertDictEqual(ir, _intermediate_repr_no_default_doc) + + maxDiff = None + def test_from_docstring_google_keras_squared_hinge(self) -> None: """ Tests whether `parse_docstring` produces the right IR diff --git a/cdd/tests/test_shared/test_ast_utils.py b/cdd/tests/test_shared/test_ast_utils.py index fb515788..482baffc 100644 --- a/cdd/tests/test_shared/test_ast_utils.py +++ b/cdd/tests/test_shared/test_ast_utils.py @@ -1011,6 +1011,39 @@ def test_param2ast_with_wrapped_default(self) -> None: ), ) + def test_param2ast_with_simple_types(self) -> None: + """Check that `param2ast` behaves correctly with simple types""" + deque( + map( + lambda typ_res: run_ast_test( + self, + param2ast( + ("zion", {"typ": typ_res[0], "default": NoneStr}), + ), + gold=AnnAssign( + annotation=Name( + typ_res[1], Load(), lineno=None, col_offset=None + ), + simple=1, + target=Name("zion", Store(), lineno=None, col_offset=None), + value=set_value(None), + expr=None, + expr_target=None, + expr_annotation=None, + col_offset=None, + lineno=None, + ), + ), + ( + ("Str", "str"), + ("Constant", "object"), + ("NameConstant", "object"), + ("Num", "float"), + ), + ), + maxlen=0, + ) + def test_param2argparse_param_none_default(self) -> None: """ Tests that param2argparse_param works to reparse the default diff --git a/cdd/tests/test_sqlalchemy/test_emit_sqlalchemy_utils.py b/cdd/tests/test_sqlalchemy/test_emit_sqlalchemy_utils.py index 6b845916..0073cc36 100644 --- a/cdd/tests/test_sqlalchemy/test_emit_sqlalchemy_utils.py +++ b/cdd/tests/test_sqlalchemy/test_emit_sqlalchemy_utils.py @@ -337,6 +337,34 @@ def test_update_args_infer_typ_sqlalchemy_early_exit(self) -> None: (True, None), ) + def test_update_args_infer_typ_sqlalchemy_list_struct(self) -> None: + """Tests that `update_args_infer_typ_sqlalchemy` behaves correctly on List[struct]""" + args = [] + self.assertTupleEqual( + update_args_infer_typ_sqlalchemy( + {"typ": "List[struct]"}, + args=args, + name="", + nullable=True, + x_typ_sql={}, + ), + (True, None), + ) + self.assertEqual(len(args), 1) + run_ast_test( + self, + args[0], + Call( + func=Name("ARRAY", Load(), lineno=None, col_offset=None), + args=[Name("JSON", Load(), lineno=None, col_offset=None)], + keywords=[], + expr=None, + expr_func=None, + lineno=None, + col_offset=None, + ), + ) + def test_update_with_imports_from_columns(self) -> None: """ Tests basic `cdd.sqlalchemy.utils.emit_utils.update_with_imports_from_columns` usage