Skip to content

Commit

Permalink
100% Test Success (#5)
Browse files Browse the repository at this point in the history
  • Loading branch information
SamuelMarks committed Sep 4, 2024
1 parent 7ba481f commit ff504dd
Show file tree
Hide file tree
Showing 11 changed files with 198 additions and 39 deletions.
2 changes: 1 addition & 1 deletion cdd/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from logging import getLogger as get_logger

__author__ = "Samuel Marks" # type: str
__version__ = "0.0.99rc46" # type: str
__version__ = "0.0.99rc47" # type: str
__description__ = (
"Open API to/fro routes, models, and tests. "
"Convert between docstrings, classes, methods, argparse, pydantic, and SQLalchemy."
Expand Down
70 changes: 66 additions & 4 deletions cdd/docstring/utils/parse_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from cdd.shared.ast_utils import deduplicate
from cdd.shared.pure_utils import (
count_iter_items,
pp,
simple_types,
sliding_window,
type_to_name,
Expand Down Expand Up @@ -56,6 +57,7 @@
("List", " ", "of"): "List",
("Tuple", " ", "of"): "Tuple",
("Dictionary", " ", "of"): "Mapping",
("One", " ", "of"): "Union",
}


Expand Down Expand Up @@ -369,8 +371,19 @@ def _parse_adhoc_doc_for_typ_phase0(doc, words):
word_chars: str = "{0}{1}`'\"/|".format(string.digits, string.ascii_letters)
sentence_ends: int = -1
break_the_union: bool = False # lincoln
for i, ch in enumerate(doc):
if (
counter = Counter(doc) # Imperfect because won't catch escaped quote marks
balanced_single: bool = counter["'"] > 0 and counter["'"] & 1 == 0
balanced_double: bool = counter['"'] > 0 and counter['"'] & 1 == 0

i: int = 0
n: int = len(doc)
while i < n:
ch: str = doc[i]
if (ch == "'" and balanced_single or ch == '"' and balanced_double) and (
i == 0 or doc[i - 1] != "\\"
):
i = eat_quoted(ch, doc, i, words, n)
elif (
ch in word_chars
or ch == "."
and len(doc) > (i + 1)
Expand All @@ -380,14 +393,20 @@ def _parse_adhoc_doc_for_typ_phase0(doc, words):
):
words[-1].append(ch)
elif ch in frozenset((".", ";", ",")) or ch.isspace():
words[-1] = "".join(words[-1])
words.append(ch)
if words[-1]:
words[-1] = "".join(words[-1])
words.append(ch)
else:
words[-1] = ch
if ch == "." and sentence_ends == -1:
sentence_ends: int = len(words)
elif ch == ";":
break_the_union = True
words.append([])
i += 1
words[-1] = "".join(words[-1])
if not words[-1]:
del words[-1]
candidate_type: Optional[str] = next(
map(
adhoc_type_to_type.__getitem__,
Expand All @@ -414,4 +433,47 @@ def _parse_adhoc_doc_for_typ_phase0(doc, words):
return candidate_type, fst_sentence, sentence


def eat_quoted(ch, doc, chomp_start_idx, words, n):
"""
Chomp from quoted character `ch` to quoted character `ch`
:param ch: Character of `'` or `"`
:type ch: ```Literal["'", '"']```
:param doc: Possibly ambiguous docstring for argument, that *might* hint as to the type
:type doc: ```str```
:param chomp_start_idx: chomp_start_idx
:type chomp_start_idx: ```int```
:param words: Words
:type words: ```List[Union[List[str], str]]```
:param n: Length of `doc`
:type n: ```int```
:return: chomp_end_idx
:rtype: ```int```
"""
chomp_end_idx: int = next(
filter(
lambda _chomp_end_idx: doc[_chomp_end_idx + 1] != ch
or doc[_chomp_end_idx] == "\\",
range(chomp_start_idx, n),
),
chomp_start_idx,
)
quoted_str: str = doc[chomp_start_idx:chomp_end_idx]
from operator import iadd

pp({"b4::words": words, '"".join(words[-1])': "".join(words[-1])})
(
iadd(words, (quoted_str, []))
if len(words[-1]) == 1 and words[-1][-1] == "`"
else iadd(words, ("".join(words[-1]), quoted_str, []))
)
pp({"words": words})
return chomp_end_idx


__all__ = ["parse_adhoc_doc_for_typ"] # type: list[str]
4 changes: 3 additions & 1 deletion cdd/shared/ast_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -427,8 +427,10 @@ def get_default_val(val):
)
elif _param.get("typ") == "Str":
_param["typ"] = "str"
elif _param.get("typ") in frozenset(("Constant", "NameConstant", "Num")):
elif _param.get("typ") in frozenset(("Constant", "NameConstant")):
_param["typ"] = "object"
elif _param.get("typ") == "Num":
_param["typ"] = "float"
if "typ" in _param and needs_quoting(_param["typ"]):
default = (
_param.get("default")
Expand Down
47 changes: 29 additions & 18 deletions cdd/shared/defaults_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -228,27 +228,38 @@ def _parse_out_default_and_doc(
:rtype: Tuple[str, Optional[str]]
"""
if typ is not None and typ in simple_types and default not in none_types:
lit = (
ast.AST()
if typ != "str"
and any(
map(
partial(contains, frozenset(("*", "^", "&", "|", "$", "@", "!"))),
default,
keep_default: bool = False
try:
lit = (
ast.AST()
if typ != "str"
and any(
map(
partial(
contains, frozenset(("*", "^", "&", "|", "$", "@", "!"))
),
default,
)
)
else literal_eval("({default})".format(default=default))
)
else literal_eval("({default})".format(default=default))
)
except ValueError as e:
assert e.args[0].startswith("malformed node or string"), e
keep_default = True
default = (
"```{default}```".format(default=default)
if isinstance(lit, ast.AST)
else {
"bool": bool,
"int": int,
"float": float,
"complex": complex,
"str": str,
}[typ](lit)
default
if keep_default
else (
"```{default}```".format(default=default)
if isinstance(lit, ast.AST)
else {
"bool": bool,
"int": int,
"float": float,
"complex": complex,
"str": str,
}[typ](lit)
)
)
elif default.isdecimal():
default = int(default)
Expand Down
4 changes: 1 addition & 3 deletions cdd/sqlalchemy/utils/emit_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@
from json import dumps
from operator import attrgetter, eq, methodcaller
from os import path
from platform import system
from typing import Any, Dict, List, Optional

import cdd.shared.ast_utils
Expand Down Expand Up @@ -79,8 +78,7 @@ def param_to_sqlalchemy_column_calls(name_param, include_name):
:return: Iterable of elements in form of: `Column(…)`
:rtype: ```Iterable[Call]```
"""
if system() == "Darwin":
print("param_to_sqlalchemy_column_calls::include_name:", include_name, ";")
# if system() == "Darwin": print("param_to_sqlalchemy_column_calls::include_name:", include_name, ";")
name, _param = name_param
del name_param

Expand Down
9 changes: 6 additions & 3 deletions cdd/sqlalchemy/utils/shared_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,9 +87,12 @@ def update_args_infer_typ_sqlalchemy(_param, args, name, nullable, x_typ_sql):
parsed_typ: Call = cast(
Call, cdd.shared.ast_utils.get_value(ast.parse(_param["typ"]).body[0])
)
assert parsed_typ.value.id == "Literal", "Expected `Literal` got: {!r}".format(
parsed_typ.value.id
)
try:
assert (
parsed_typ.value.id == "Literal"
), "Expected `Literal` got: {!r}".format(parsed_typ.value.id)
except AssertionError:
raise
val = cdd.shared.ast_utils.get_value(parsed_typ.slice)
(
args.append(
Expand Down
16 changes: 9 additions & 7 deletions cdd/tests/mocks/docstrings.py
Original file line number Diff line number Diff line change
Expand Up @@ -500,20 +500,22 @@
)
docstring_google_pytorch_lbfgs_str: str = "\n".join(docstring_google_pytorch_lbfgs)

docstring_google_str: str = (
"""{docstring_header_str}
Args:
docstring_google_args_str: str = """Args:
dataset_name (str): name of dataset. Defaults to "mnist"
tfds_dir (str): directory to look for models in. Defaults to "~/tensorflow_datasets"
K (Literal['np', 'tf']): backend engine, e.g., `np` or `tf`. Defaults to "np"
as_numpy (Optional[bool]): Convert to numpy ndarrays
data_loader_kwargs (Optional[dict]): pass this as arguments to data_loader function
Returns:
"""
docstring_google_footer_return_str: str = """Returns:
Union[Tuple[tf.data.Dataset, tf.data.Dataset], Tuple[np.ndarray, np.ndarray]]:
Train and tests dataset splits. Defaults to (np.empty(0), np.empty(0))
""".format(
docstring_header_str=docstring_header_str
"""
docstring_google_str: str = "\n".join(
(
docstring_header_str,
docstring_google_args_str,
docstring_google_footer_return_str,
)
)

Expand Down
5 changes: 3 additions & 2 deletions cdd/tests/test_compound/test_exmod_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ def test_emit_file_on_hierarchy(self) -> None:
with patch(
"cdd.compound.exmod_utils.EXMOD_OUT_STREAM", new_callable=StringIO
), TemporaryDirectory() as tempdir:
output_directory: str = path.join(tempdir, "haz")
open(path.join(tempdir, INIT_FILENAME), "a").close()
emit_file_on_hierarchy(
("foo.bar", "foo_dir", ir),
Expand All @@ -65,13 +66,13 @@ def test_emit_file_on_hierarchy(self) -> None:
"",
True,
filesystem_layout="as_input",
output_directory=tempdir,
output_directory=output_directory,
first_output_directory=tempdir,
no_word_wrap=None,
dry_run=False,
extra_modules_to_all=None,
)
self.assertTrue(path.isdir(tempdir))
self.assertTrue(path.isdir(output_directory))

def test__emit_symbols_isfile_emit_filename_true(self) -> None:
"""Test `_emit_symbol` when `isfile_emit_filename is True`"""
Expand Down
19 changes: 19 additions & 0 deletions cdd/tests/test_marshall_docstring.py
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,25 @@ def test_from_docstring_google_str(self) -> None:
_intermediate_repr_no_default_doc["doc"] = docstring_header_str
self.assertDictEqual(ir, _intermediate_repr_no_default_doc)

def test_from_docstring_google_str_extra_after(self) -> None:
"""
Tests whether `parse_docstring` produces `intermediate_repr_no_default_doc` + "\n\n\ntrailer"
from `docstring_google_str` + "\n\n\ntrailer"
"""
ir: IntermediateRepr = parse_docstring(
"{}\n\n\ntrailer".format(docstring_google_str)
)
_intermediate_repr_no_default_doc = deepcopy(
intermediate_repr_no_default_with_nones_doc
)
_intermediate_repr_no_default_doc["doc"] = "{0}\n\n\n\ntrailer".format(
docstring_header_str
)

self.assertDictEqual(ir, _intermediate_repr_no_default_doc)

maxDiff = None

def test_from_docstring_google_keras_squared_hinge(self) -> None:
"""
Tests whether `parse_docstring` produces the right IR
Expand Down
33 changes: 33 additions & 0 deletions cdd/tests/test_shared/test_ast_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1011,6 +1011,39 @@ def test_param2ast_with_wrapped_default(self) -> None:
),
)

def test_param2ast_with_simple_types(self) -> None:
"""Check that `param2ast` behaves correctly with simple types"""
deque(
map(
lambda typ_res: run_ast_test(
self,
param2ast(
("zion", {"typ": typ_res[0], "default": NoneStr}),
),
gold=AnnAssign(
annotation=Name(
typ_res[1], Load(), lineno=None, col_offset=None
),
simple=1,
target=Name("zion", Store(), lineno=None, col_offset=None),
value=set_value(None),
expr=None,
expr_target=None,
expr_annotation=None,
col_offset=None,
lineno=None,
),
),
(
("Str", "str"),
("Constant", "object"),
("NameConstant", "object"),
("Num", "float"),
),
),
maxlen=0,
)

def test_param2argparse_param_none_default(self) -> None:
"""
Tests that param2argparse_param works to reparse the default
Expand Down
28 changes: 28 additions & 0 deletions cdd/tests/test_sqlalchemy/test_emit_sqlalchemy_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,34 @@ def test_update_args_infer_typ_sqlalchemy_early_exit(self) -> None:
(True, None),
)

def test_update_args_infer_typ_sqlalchemy_list_struct(self) -> None:
"""Tests that `update_args_infer_typ_sqlalchemy` behaves correctly on List[struct]"""
args = []
self.assertTupleEqual(
update_args_infer_typ_sqlalchemy(
{"typ": "List[struct]"},
args=args,
name="",
nullable=True,
x_typ_sql={},
),
(True, None),
)
self.assertEqual(len(args), 1)
run_ast_test(
self,
args[0],
Call(
func=Name("ARRAY", Load(), lineno=None, col_offset=None),
args=[Name("JSON", Load(), lineno=None, col_offset=None)],
keywords=[],
expr=None,
expr_func=None,
lineno=None,
col_offset=None,
),
)

def test_update_with_imports_from_columns(self) -> None:
"""
Tests basic `cdd.sqlalchemy.utils.emit_utils.update_with_imports_from_columns` usage
Expand Down

0 comments on commit ff504dd

Please sign in to comment.