diff --git a/README.md b/README.md index e4f3d62..5171fe6 100644 --- a/README.md +++ b/README.md @@ -10,10 +10,10 @@ just one possible use case. It can take any dictionary-equivalent model rudimentary support for cross-references. Generators then transform this model in the desired output. -A `Model` is simply a Python dictionary (which may be manually iniatialized or -sourced from a YAML or JSON file) representing a hierarchy, typically with deep -nesting. The `select` operation is run on `Models`s to select a new sub-`Model` -based on a path selection mechanism. +A `Model` is simply a Python dictionary or enumerable (which may be manually +iniatialized or sourced from a YAML or JSON file) representing a hierarchy, +typically with deep nesting. The `select` operation is run on `Models`s to +select a new sub-`Model` based on a path selection mechanism. Generators are functions annotated with the `@gen()` decorator, which gives some extra powers to special f-strings expressions in them: automagic @@ -40,6 +40,7 @@ file in this project): ```py from fstringen import gen, Model +PREAMBLE = "// File generated by fstringen. DO NOT EDIT.\n\n" model = Model.fromDict({ "structs": { "mystruct": { @@ -66,7 +67,7 @@ def gen_struct(struct): *""" -@gen(model=model, fname="myfile.go", comment="//") +@gen(model=model, fname="myfile.go", preamble=PREAMBLE) def gen_myfile(model): return f"""* package main @@ -77,6 +78,7 @@ def gen_myfile(model): ``` All generator functions using fstringstars must be decorated with `@gen()`. + When no parameters are passed to `gen()`, the generator is considerate a subordinate generator (i.e., they need to be called explicitly from other generators). @@ -145,8 +147,8 @@ supported (i.e., `f'''*...*'''` is not a valid fstringstar). ## Caveats fstringen does dangerous things with scope and function re-declaration. This means you should only use it in scripts that are code generators, and code -generators alone (i.e., they don't do anything else). We sacrificed correctness -for neatness and ease-to-use. +generators alone (i.e., they don't do anything else). Correctness and safety +were sacrificed for neatness and ease-to-use. Since fstringen tramples over all common sense, pretty much all exceptions are intercepted and transformed into custom error messages. Otherwise, because of diff --git a/example.py b/example.py index 73490e2..4b61c28 100644 --- a/example.py +++ b/example.py @@ -1,5 +1,6 @@ from fstringen import gen, Model +PREAMBLE = "// File generated by fstringen. DO NOT EDIT.\n\n" model = Model.fromDict({ "structs": { "mystruct": { @@ -26,7 +27,7 @@ def gen_struct(struct): *""" -@gen(model=model, fname="myfile.go", comment="//") +@gen(model=model, fname="myfile.go", preamble=PREAMBLE) def gen_myfile(model): return f"""* package main diff --git a/fstringen/__init__.py b/fstringen/__init__.py index bafbf4d..b5a6b26 100644 --- a/fstringen/__init__.py +++ b/fstringen/__init__.py @@ -1 +1,8 @@ -from fstringen.fstringen import gen, Model, ModelError, FStringenError # noqa +# flake8: noqa +# type: ignore + +from .model import * +from .generator import * + + +__all__ = model.__all__ + generator.__all__ diff --git a/fstringen/fstringen.py b/fstringen/fstringen.py deleted file mode 100644 index 8dc4cbf..0000000 --- a/fstringen/fstringen.py +++ /dev/null @@ -1,445 +0,0 @@ -import atexit -import inspect -import json -import re -import sys -import textwrap -import traceback - -try: - import yaml # type: ignore -except ImportError: - yaml = None - - -def _putify(template): - if "\n" in template: - template = textwrap.dedent(template) - - lines = template.split("\n") - newlines = [] - for line in lines: - # We don't care about indent in single-line strings - if len(lines) == 1: - indent = '' - else: - indent = line[:len(line) - len(line.lstrip())] - # Take out all the isolated '{' and '}' characters - line = line.replace("{{", "#_fstringen_ocbblock#").replace( - "}}", "#_fstringen_ccbblock#") - nl = line.replace("{", "{_put(").replace("}", ", '" + indent + "')}") - # Put back all the isolated '{' and '}' characters - nl = nl.replace("#_fstringen_ocbblock#", - "{{").replace("#_fstringen_ccbblock#", "}}") - newlines.append(nl) - return "\n".join(newlines) - - -def _put(obj, indent): - if isinstance(obj, list) or isinstance(obj, tuple): - newobj = [] - for el in obj: - if el is None: - continue - el = textwrap.dedent(str(el)).replace("\n", "\n" + indent) - newobj.append(el) - return ("\n" + indent).join(newobj) - else: - if obj is None: - return "" - return str(obj).replace("\n", "\n" + indent) - - -def _errmsg(exc_info, fnname, code=None, fstringstar=None): - excl = exc_info[0] - exc = exc_info[1] - tb = exc_info[2] - - lineno = traceback.extract_tb(tb)[-1][1] - 1 - # If we have precise line information, use it - line = traceback.extract_tb(tb)[-1][3] - if fstringstar is None and code is None and line: - subfn = traceback.extract_tb(tb)[-1][2] - return f""" -Error in function '{subfn}' called (directly or indirectly) by -generator '{fnname}': -{"-"*80} -Line {lineno + 1}: {line} <- {excl.__name__}: {str(exc).strip()} -{"-"*80} -""" - if code is not None: - fnlines = code.split("\n") - startline = max(lineno - 3, 0) - endline = min(lineno + 3, len(fnlines) - 1) - try: - fnlines[lineno] = (fnlines[lineno] + - f" <- {type(exc).__name__}: " + - f"{str(exc)}") - msg = "\n".join(fnlines[startline:endline]) - if endline < len(fnlines) - 1: - msg += "\n[...]" - except IndexError: - msg = ("[Could not determine source code location]\n" + - f"{type(exc).__name__}: {str(exc)}") - - return f""" -Error in generator '{fnname}': -{"-"*80} -{textwrap.dedent(msg).strip()} -{"-"*80} -""" - - if fstringstar is not None: - return f""" -Error generating fstringstar in generator '{fnname}': -{"-"*80} -fstringstar: f\"\"\"* -{textwrap.dedent(fstringstar).strip()} -*\"\"\" <- {excl.__name__}: {str(exc).strip()} -{"-"*80} -""" - - -def _normalize_whitespace(fstringstar): - if fstringstar[0] == "\n": - fstringstar = textwrap.dedent(fstringstar) - fstringstar = fstringstar[1:] - - lines = fstringstar.split("\n") - if len(lines) >= 2 and lines[-1].strip() == "": - lines = lines[:-1] - fstringstar = "\n".join(lines) - - return fstringstar - - -def _compile(fstringstar): - gen_frame = inspect.currentframe().f_back - globals_ = gen_frame.f_globals - locals_ = gen_frame.f_locals - - fstringstar = _normalize_whitespace(fstringstar) - fstring = "_fstringstar = f\"\"\"{}\"\"\"".format(_putify(fstringstar)) - try: - exec(fstring, globals_, locals_) - except Exception: - fnname = gen_frame.f_code.co_name - msg = _errmsg(sys.exc_info(), fnname, fstringstar=fstringstar) - raise FStringenError(msg) from None - - output = locals_["_fstringstar"] - return output - - -_original_excepthook = sys.excepthook - - -def exception_handler(exception_type, exception, traceback): - # Traceback is useless, given the distortions we introduce - if isinstance(exception, FStringenError): - sys.stderr.write(f"{exception_type.__name__}: " + - f"{str(exception).strip()}\n") - # For non fstringen-related errors however, traceback may be useful - else: - _original_excepthook(exception_type, exception, traceback) - - -sys.excepthook = exception_handler - - -def gen(model=None, fname=None, comment=None, notice=True): - def realgen(fn): - original_name = fn.__name__ - code = textwrap.dedent(inspect.getsource(fn)) - newcode = "\n".join(code.split("\n")[1:]) # Remove decorator - original_code = newcode - newcode = re.sub(r"(f\"\"\"\*)", "_compile(\"\"\"", newcode) - newcode = re.sub(r"(\*\"\"\")", "\"\"\")", newcode) - - # Re-execute function definition with the new code, in its own - # globals/locals scope - globals_ = inspect.currentframe().f_back.f_globals - globals_["_put"] = _put - globals_["_compile"] = _compile - locals_ = inspect.currentframe().f_back.f_locals - exec(newcode, globals_, locals_) - newgen = locals_[original_name] - - def newfn(*args, **kwargs): - try: - r = newgen(*args, **kwargs) - # If the error is already an FStringenError, we have nothing to add - except FStringenError as e: - raise e from None - except Exception: - msg = _errmsg(sys.exc_info(), original_name, - code=original_code) - raise FStringenError(msg) from None - - if r is None: - return - elif isinstance(r, str): - return textwrap.dedent(r) - else: - return r - - if fname: - if notice and not comment: - raise FStringenError( - "Cannot place a notice without having the comment prefix") - global _output - _output[fname] = { - "notice": notice, - "comment": comment, - "fn": newfn, - "model": model - } - - # Put the new function in globals, so other @gen code can call it. - # This is obviously dangerous, and it's one of the the reasons why - # fstringen must not be used in anything else other than generators. - globals_[original_name] = newfn - - newfn.code = original_code - return newfn - - return realgen - - -_output = {} # type: ignore - - -def _generate_all(): - for fname in _output: - genopts = _output[fname] - f = open(fname, "w") - if genopts["notice"]: - f.write(genopts["comment"] + - " File generated by fstringen. DO NOT EDIT.\n\n") - fn = genopts["fn"] - model = genopts["model"] - f.write(fn(model)) - f.close() - - -atexit.register(_generate_all) - - -class FStringenError(Exception): - pass - - -class ModelError(Exception): - pass - - -def is_enumerable(obj): - try: - enumerate(obj) - except TypeError: - return False - else: - return True - - -class Model: - def __new__(cls, name, value, refindicator="#", root=None): - # Pick Model methods that should be used. - methods = {} - for method in cls.__dict__: - if not method.startswith("__") or method == "__call__": - methods[method] = cls.__dict__[method] - - original_type = type(value) - # Python does not allow subclassing bool, so we use an adapted int. - if isinstance(value, bool): - value = int(value) - - def custom_repr_str(this): - return str(bool(this.value)) - - methods["__repr__"] = custom_repr_str - methods["__str__"] = custom_repr_str - - # None cannot be subclassed either, use an empty string instead. - elif value is None: - value = "" - - def custom_repr_str(this): - return "None" - - def custom_eq(this, other): - return other is None - - methods["__repr__"] = custom_repr_str - methods["__str__"] = custom_repr_str - methods["__eq__"] = custom_eq - - # Create a dynamic class based on the original type of the value, but - # including methods from Model. - # See: https://docs.python.org/3/library/functions.html#type - newcls = type(cls.__name__, (type(value),), methods) - obj = newcls(value) - # Initialize Model attributes. - obj._initModel(name, original_type, refindicator, root) - return obj - - def _initModel(self, name, original_type, refindicator, root): - """ Sets internal Model values """ - self.name = name - self.value = self - self.type = original_type - self.refindicator = refindicator - self.root = root - if self.root is None: - self.root = self.value - - def _new(self, name, model): - """ Instantiates a Model keeping the same root. """ - return Model(name, model, self.refindicator, self.root) - - @staticmethod - def fromYAML(fname, refindicator="#", root=None): - """ Loads a Model from a YAML file. """ - if yaml is None: - raise ModelError("Cannot find 'yaml' module") - return Model(fname, - yaml.load(open(fname, "r").read(), Loader=yaml.Loader), - refindicator, - root) - - @staticmethod - def fromJSON(fname, refindicator="#", root=None): - """ Loads a Model from a JSON file. """ - return Model(fname, - json.loads(open(fname, "r").read()), - refindicator, - root) - - @staticmethod - def fromDict(dict_, name="dict", refindicator="#", root=None): - """ Loads a Model from a Python dictionary. """ - return Model(name, dict_, refindicator, root) - - def has(self, path=None): - """ Returns True if path exists in the Model. If path is None, returns - True. """ - if path is None: - return True # We know we exist because we exist :-) - callerctx = inspect.currentframe().f_back - try: - self._select(path, callerctx) - except ModelError: - return False - - return True - - def is_reference(self, path=None): - """ Returns True if path is a possible reference. If path is None, - returns True if this Model contains a reference. """ - if path is None: - value = self.value - else: - callerctx = inspect.currentframe().f_back - value = self._select(path, callerctx) - return isinstance(value, str) and value.startswith(self.refindicator) - - def is_enabled(self, path=None): - """ Returns True if path exists and its value is True. If path is None, - returns True if this Model is True. """ - if path is None: - value = self.value - else: - try: - callerctx = inspect.currentframe().f_back - value = self._select(path, callerctx) - except ModelError: - return False - - return isinstance(value, int) and value != 0 - - def __call__(self, path, default=None): - """ __call__ enables a shorthand for calling select. """ - callerctx = inspect.currentframe().f_back - return self._select(path, callerctx, default) - - def select(self, path, default=None): - """ Returns a new Model based on path. """ - callerctx = inspect.currentframe().f_back - return self._select(path, callerctx, default) - - def _select(self, path, callerctx=None, default=None): - if callerctx is None: - callerctx = inspect.currentframe().f_back - - obj = self.value - name = None - curpath = [] - - # Ignore ref indicators and browse accordingly. - if path.startswith(self.refindicator): - path = path[1:] - if path.endswith("->"): - path = path[:-2] - return self._select(path, default) - # When an absolute path is used in a query, revert to the root. - if path.startswith("/"): - path = path[1:] - obj = self.root - curpath.append("") - # Empty path trailings are ignored. - if path.endswith("/"): - path = path[:-1] - - parts = path.split("/") - for i in range(len(parts)): - part = parts[i] - curpath.append(part) - if part == "*" and i == len(parts) - 1: - if isinstance(obj, dict): - elements = tuple(self._new(k, v) for k, v in obj.items()) - elif is_enumerable(obj): - elements = type(obj)(self._new(str(i), v) for i, v in - enumerate(obj)) - else: - raise ModelError("Cannot iterate over '{}'" - .format("/".join(curpath[:-1]))) - obj = elements - name = "*" - elif part.endswith("->"): - part = part[:-2] - newpath = obj[part] - newmodel = self._new(part, obj) - obj = newmodel._select(newpath, callerctx, default) - name = obj.name - else: - try: - obj = obj[part] - except TypeError: - if not is_enumerable(obj): - raise ModelError( - "Cannot lookup path '{}' in value '{}'".format( - part, str(obj))) - try: - part = int(part) - except ValueError: - raise ModelError( - "Enumerable navigation requires integers") - try: - obj = obj[part] - except IndexError: - if default is not None: - obj = default - break - raise ModelError( - "Could not find path '{}' in '{}'" - .format("/".join(curpath), obj)) - except KeyError: - if default is not None: - obj = default - break - raise ModelError("Could not find path '{}' in '{}'" - .format("/".join(curpath), obj)) - name = part - - return self._new(name, obj) diff --git a/fstringen/generator.py b/fstringen/generator.py new file mode 100644 index 0000000..a7007d6 --- /dev/null +++ b/fstringen/generator.py @@ -0,0 +1,233 @@ +import atexit +import inspect +import re +import sys +import textwrap +import traceback + + +class FStringenError(Exception): + """ + FStringenError represents an error in generating text using gen. + """ + pass + + +def _putify(template): + if "\n" in template: + template = textwrap.dedent(template) + + lines = template.split("\n") + newlines = [] + for line in lines: + # We don't care about indent in single-line strings + if len(lines) == 1: + indent = '' + else: + indent = line[:len(line) - len(line.lstrip())] + # Take out all the isolated '{' and '}' characters + line = line.replace("{{", "#_fstringen_ocbblock#").replace( + "}}", "#_fstringen_ccbblock#") + nl = line.replace("{", "{_put(").replace("}", ", '" + indent + "')}") + # Put back all the isolated '{' and '}' characters + nl = nl.replace("#_fstringen_ocbblock#", + "{{").replace("#_fstringen_ccbblock#", "}}") + newlines.append(nl) + return "\n".join(newlines) + + +def _put(obj, indent): + if isinstance(obj, list) or isinstance(obj, tuple): + newobj = [] + for el in obj: + if el is None: + continue + el = textwrap.dedent(str(el)).replace("\n", "\n" + indent) + newobj.append(el) + return ("\n" + indent).join(newobj) + else: + if obj is None: + return "" + return str(obj).replace("\n", "\n" + indent) + + +def _errmsg(exc_info, fnname, code=None, fstringstar=None): + excl = exc_info[0] + exc = exc_info[1] + tb = exc_info[2] + + lineno = traceback.extract_tb(tb)[-1][1] - 1 + # If we have precise line information, use it + line = traceback.extract_tb(tb)[-1][3] + if fstringstar is None and code is None and line: + subfn = traceback.extract_tb(tb)[-1][2] + return f""" +Error in function '{subfn}' called (directly or indirectly) by +generator '{fnname}': +{"-"*80} +Line {lineno + 1}: {line} <- {excl.__name__}: {str(exc).strip()} +{"-"*80} +""" + if code is not None: + fnlines = code.split("\n") + startline = max(lineno - 3, 0) + endline = min(lineno + 3, len(fnlines) - 1) + try: + fnlines[lineno] = (fnlines[lineno] + + f" <- {type(exc).__name__}: " + + f"{str(exc)}") + msg = "\n".join(fnlines[startline:endline]) + if endline < len(fnlines) - 1: + msg += "\n[...]" + except IndexError: + msg = ("[Could not determine source code location]\n" + + f"{type(exc).__name__}: {str(exc)}") + + return f""" +Error in generator '{fnname}': +{"-"*80} +{textwrap.dedent(msg).strip()} +{"-"*80} +""" + + if fstringstar is not None: + return f""" +Error generating fstringstar in generator '{fnname}': +{"-"*80} +fstringstar: f\"\"\"* +{textwrap.dedent(fstringstar).strip()} +*\"\"\" <- {excl.__name__}: {str(exc).strip()} +{"-"*80} +""" + + +def _normalize_whitespace(fstringstar): + if fstringstar[0] == "\n": + fstringstar = textwrap.dedent(fstringstar) + fstringstar = fstringstar[1:] + + lines = fstringstar.split("\n") + if len(lines) >= 2 and lines[-1].strip() == "": + lines = lines[:-1] + fstringstar = "\n".join(lines) + + return fstringstar + + +def _compile(fstringstar): + gen_frame = inspect.currentframe().f_back + globals_ = gen_frame.f_globals + locals_ = gen_frame.f_locals + + fstringstar = _normalize_whitespace(fstringstar) + fstring = "_fstringstar = f\"\"\"{}\"\"\"".format(_putify(fstringstar)) + try: + exec(fstring, globals_, locals_) + except Exception: + fnname = gen_frame.f_code.co_name + msg = _errmsg(sys.exc_info(), fnname, fstringstar=fstringstar) + raise FStringenError(msg) from None + + output = locals_["_fstringstar"] + return output + + +_original_excepthook = sys.excepthook + + +def _exception_handler(exception_type, exception, traceback): + # Traceback is useless, given the distortions we introduce + if isinstance(exception, FStringenError): + sys.stderr.write(f"{exception_type.__name__}: " + + f"{str(exception).strip()}\n") + # For non fstringen-related errors however, traceback may be useful + else: + _original_excepthook(exception_type, exception, traceback) + + +sys.excepthook = _exception_handler + + +def gen(model=None, fname=None, preamble=None): + """ + gen is a decorator that turns a function or method into a fstringen-powered + generator. + + If model and fname are passed, the generator will pass model as the first + and only argument to the decorated function, and write the value returned + by that function to the file at fname. If preamble is not None, it will be + included at the beginning of the generated file. + """ + def realgen(fn): + original_name = fn.__name__ + code = textwrap.dedent(inspect.getsource(fn)) + newcode = "\n".join(code.split("\n")[1:]) # Remove decorator + original_code = newcode + newcode = re.sub(r"(f\"\"\"\*)", "_compile(\"\"\"", newcode) + newcode = re.sub(r"(\*\"\"\")", "\"\"\")", newcode) + + # Re-execute function definition with the new code, in its own + # globals/locals scope + globals_ = inspect.currentframe().f_back.f_globals + globals_["_put"] = _put + globals_["_compile"] = _compile + locals_ = inspect.currentframe().f_back.f_locals + exec(newcode, globals_, locals_) + newgen = locals_[original_name] + + def newfn(*args, **kwargs): + try: + r = newgen(*args, **kwargs) + # If the error is already an FStringenError, we have nothing to add + except FStringenError as e: + raise e from None + except Exception: + msg = _errmsg(sys.exc_info(), original_name, + code=original_code) + raise FStringenError(msg) from None + + if r is None: + return + elif isinstance(r, str): + return textwrap.dedent(r) + else: + return r + + if model and fname: + global _output + _output[fname] = { + "fn": newfn, + "model": model, + "preamble": preamble, + } + + # Put the new function in globals, so other @gen code can call it. + # This is obviously dangerous, and it's one of the the reasons why + # fstringen must not be used in anything else other than generators. + globals_[original_name] = newfn + + newfn.code = original_code + return newfn + + return realgen + + +_output = {} # type: ignore + + +def _generate_all(): + for fname in _output: + genopts = _output[fname] + f = open(fname, "w") + if genopts["preamble"]: + f.write(genopts["preamble"]) + fn = genopts["fn"] + model = genopts["model"] + f.write(fn(model)) + f.close() + + +atexit.register(_generate_all) + + +__all__ = "gen", "FStringenError" diff --git a/fstringen/generator_test.py b/fstringen/generator_test.py new file mode 100644 index 0000000..31958f9 --- /dev/null +++ b/fstringen/generator_test.py @@ -0,0 +1,235 @@ +import unittest + +from .generator import gen + + +class TestGen(unittest.TestCase): + def test_simple(self): + @gen() + def fn(): + a = 1 + return f"""* + {a} + *""" + + self.assertEqual(fn(), "1") + + @gen() + def fn2(): + a = 1 + return f"""* + x + {a} + *""" + + self.assertEqual(fn2(), "x\n 1") + + @gen() + def fn2(): + a = 1 + b = "something" + c = "." + + return f"""* + x + {a} + {b} + {c} + *""" + + self.assertEqual(fn2(), "x\n 1\nsomething\n .") + + def test_basic_newline_equivalence(self): + @gen() + def fn1(): + return f"""* + ... + *""" # noqa + + @gen() + def fn2(): + return f"""*...*""" # noqa + + @gen() + def fn3(): + return f"""* + ...*""" # noqa + + @gen() + def fn4(): + return f"""*... + *""" # noqa + + self.assertEqual(fn1(), "...") + self.assertEqual(fn2(), "...") + self.assertEqual(fn3(), "...") + self.assertEqual(fn4(), "...") + + def test_quotes(self): + @gen() + def fn(): + a = 1 + return f"""* + \\"{a}\\" + *""" + self.assertEqual(fn(), "\"1\"") + + @gen() + def fn2(): + a = 1 + return f"""*\\"{a}\\*""" + self.assertEqual(fn(), "\"1\"") + + def test_blank_lines(self): + @gen() + def fn(): + a = 1 + return f"""* + + {a} + *""" + + self.assertEqual(fn(), "\n1") + + @gen() + def fn2(): + a = 1 + return f"""* + + {a} + + *""" + + self.assertEqual(fn2(), "\n1\n") + + @gen() + def fn2(): + a = 1 + return f"""* + + x: + {a} + + + *""" + + self.assertEqual(fn2(), "\nx:\n 1\n\n") + + def test_list(self): + @gen() + def fn(): + elements = [1, 2] + return f"""* + {elements} + *""" + + self.assertEqual(fn(), "1\n2") + + @gen() + def fn(): + elements = [1, 2] + return f"""* + x + {elements} + *""" + + self.assertEqual(fn(), "x\n 1\n 2") + + def test_single_line(self): + @gen() + def fn(): + a = "y\nx" + return f"""* {a} *""" + + self.assertEqual(fn(), " y\nx ") + + @gen() + def fn2(): + a = [1, 2] + return f"""*{a}*""" + + self.assertEqual(fn2(), "1\n2") + + def test_indent(self): + @gen() + def multiline_string(): + return f"""* + multiline + string + *""" # noqa + + # Indentation-level of outside code shouldn't interfere + @gen() + def fn(cond): + a = [1, 2] + if cond: + return f"""* + {a} + *""" + return f"""* + {a} + *""" + self.assertEqual(fn(False), "1\n2") + self.assertEqual(fn(True), "1\n2") + + # Multi-line strings should be dedented + @gen() + def fn2(): + a = [multiline_string(), multiline_string()] + return f"""* + {a} + *""" + self.assertEqual(fn2(), "multiline\n string\nmultiline\n string") + + def test_dict(self): + @gen() + def fn(): + a = {"x": 1} + return f"""* + dict: {a} + *""" + + self.assertEqual(fn(), "dict: {'x': 1}") + + def test_return_none(self): + @gen() + def fn_none(): + return + + @gen() + def fn(): + a = "2" # noqa + l = [1, None, 2] # noqa + return f"""* + a: 2{fn_none()} + {l} + {fn_none()} + b + *""" + + # When None is part of a list, it should be hidden + # When None is a direct value, it shows as "" + self.assertEqual(fn(), "a: 2\n1\n2\n\nb") + + def test_call(self): + @gen() + def abc(): + return "abc" + + @gen() + def fn(): + a = {"x": 1} + return f"""* + dict: {a} + *""" + + @gen() + def fn2(): + return f"""* + {abc()} call: + {fn()} + *""" + + self.assertEqual(fn2(), "abc call:\n dict: {'x': 1}") + + diff --git a/fstringen/integration_test.py b/fstringen/integration_test.py new file mode 100644 index 0000000..417ad14 --- /dev/null +++ b/fstringen/integration_test.py @@ -0,0 +1,62 @@ +import unittest + +from .generator import gen +from .model import Model +from .model_test import test_model + + +class TestIntegration(unittest.TestCase): + # TODO: test file generation + + def test_gen_select(self): + @gen() + def gen_color(component): + return f"""* + color: {component.select("properties/color")} + *""" + + @gen() + def gen_component(component): + parent = None + if component.has("properties/parent"): + parent = component.select("properties/parent->").name + parent = f"parent: {parent}" + + return f"""* + # Component {component.select("properties/name")}: + {parent} + {gen_color(component)} + nicknames: + {["- " + x for x in component.select("properties/nicknames")]} + + *""" + + @gen() + def gen_all(model): + components = [ + gen_component(component) + for component in model.select("/components/*") + ] + + return f"""* + {components} + *""" + + m = Model.fromDict(test_model, refprefix="$") + self.assertEqual( + gen_all(m), """# Component componentA: + + color: blue + nicknames: + - cA + - compA + - A + +# Component componentB: + parent: componentA + color: red + nicknames: + - cB + - compB + - B +""") diff --git a/fstringen/model.py b/fstringen/model.py new file mode 100644 index 0000000..9cfe139 --- /dev/null +++ b/fstringen/model.py @@ -0,0 +1,254 @@ +import json + +try: + import yaml # type: ignore +except ImportError: + yaml = None + + +class ModelError(Exception): + """ + ModelError represents an exception in browsing a model with Model.select. + """ + pass + + +def _is_enumerable(obj): + try: + enumerate(obj) + except TypeError: + return False + else: + return True + + +def _has_items_method(obj): + items = getattr(obj, "items", None) + return items and callable(items) + + +class Model: + """ + Model represents any named (name) Python object (value). It acts as a + subclass of the type of the value object. If that object is a dict-like or + is enumerable, a special path syntax can be used to browse it using + Model.select. If the value contains the special prefix denoted by + refprefix, that value can be used to jump to other parts of the model by + using Model.select. Calling the model directly is equivalent to calling + Model.select. + """ + + def __new__(cls, name, value, refprefix="#", _root=None): + # Pick Model methods that should be used. + methods = {} + for method in cls.__dict__: + if not method.startswith("__") or method == "__call__": + methods[method] = cls.__dict__[method] + + original_type = type(value) + # Python does not allow subclassing bool, so we use an adapted int. + if isinstance(value, bool): + value = int(value) + + def custom_repr_str(this): + return str(bool(this.value)) + + methods["__repr__"] = custom_repr_str + methods["__str__"] = custom_repr_str + + # None cannot be subclassed either, use an empty string instead. + elif value is None: + value = "" + + def custom_repr_str(this): + return "None" + + def custom_eq(this, other): + return other is None + + methods["__repr__"] = custom_repr_str + methods["__str__"] = custom_repr_str + methods["__eq__"] = custom_eq + + # Create a dynamic class based on the original type of the value, but + # including methods from Model. + # See: https://docs.python.org/3/library/functions.html#type + newcls = type(cls.__name__, (type(value),), methods) + obj = newcls(value) + # Initialize Model attributes. + obj._initModel(name, original_type, refprefix, _root) + return obj + + def _initModel(self, name, original_type, refprefix, root): + """ + Sets internal Model values + """ + self.name = name + self.value = self + self.type = original_type + self.refprefix = refprefix + self.root = root + if self.root is None: + self.root = self.value + + def _new(self, name, model): + """ + _new instantiates a Model keeping the same root. + """ + return Model(name, model, self.refprefix, self.root) + + @staticmethod + def fromYAML(fname, refprefix="#"): + """ + Model.fromYAML loads a Model from the YAML file at fname, using the + optional refprefix as the reference prefix for the resulting Model. + """ + if yaml is None: + raise ModelError("Cannot find 'yaml' module") + return Model(fname, + yaml.load(open(fname, "r").read(), Loader=yaml.Loader), + refprefix) + + @staticmethod + def fromJSON(fname, refprefix="#"): + """ + Model.fromJSON loads a Model from the JSON file at fname, using the + optional refprefix as the reference prefix for the resulting Model. + """ + return Model(fname, + json.loads(open(fname, "r").read()), + refprefix) + + @staticmethod + def fromDict(dict_, name="dict", refprefix="#"): + """ + Model.fromDict loads a Model Python dictionary dict_, using the + optional refprefix as the reference prefix for the resulting Model. + """ + return Model(name, dict_, refprefix) + + def has(self, path=None): + """ + has returns True if path exists in the Model. If path is None, returns + True. + """ + if path is None: + return True + try: + self._select(path) + except ModelError: + return False + + return True + + def is_reference(self, path=None): + """ + is_reference returns True if path is a possible reference. If path is + None, returns True if the value of this Model contains a reference. + """ + if path is None: + value = self.value + else: + value = self._select(path) + return isinstance(value, str) and value.startswith(self.refprefix) + + def is_enabled(self, path=None): + """ + if_enabled returns True if path exists and its value is True. If path + is None, returns True if the value of this Model is True. + """ + if path is None: + value = self.value + else: + try: + value = self._select(path) + except ModelError: + return False + + return isinstance(value, int) and value != 0 + + def select(self, path, default=None): + """ + select returns a new Model based on path, with an optional default + value in case the path is valid but cannot be not found. + """ + return self._select(path, default) + + # Make model(...) a shortcut for model.select(...). + __call__ = select + + def _select(self, path, default=None): + obj = self.value + name = None + curpath = [] + + # Ignore ref indicators and browse accordingly. + if path.startswith(self.refprefix): + path = path[1:] + if path.endswith("->"): + path = path[:-2] + return self._select(path, default) + # When an absolute path is used in a query, revert to the root. + if path.startswith("/"): + path = path[1:] + obj = self.root + curpath.append("") + # Empty path trailings are ignored. + if path.endswith("/"): + path = path[:-1] + + parts = path.split("/") + for i in range(len(parts)): + part = parts[i] + curpath.append(part) + if part == "*" and i == len(parts) - 1: + if _has_items_method(obj): + elements = tuple(self._new(k, v) for k, v in obj.items()) + elif _is_enumerable(obj): + elements = type(obj)(self._new(str(i), v) for i, v in + enumerate(obj)) + else: + raise ModelError("Cannot iterate over '{}'" + .format("/".join(curpath[:-1]))) + obj = elements + name = "*" + elif part.endswith("->"): + part = part[:-2] + newpath = obj[part] + newmodel = self._new(part, obj) + obj = newmodel._select(newpath, default) + name = obj.name + else: + try: + obj = obj[part] + except TypeError: + if not _is_enumerable(obj): + raise ModelError( + "Cannot lookup path '{}' in value '{}'".format( + part, str(obj))) + try: + part = int(part) + except ValueError: + raise ModelError( + "Enumerable navigation requires integers") + try: + obj = obj[part] + except IndexError: + if default is not None: + obj = default + break + raise ModelError( + "Could not find path '{}' in '{}'" + .format("/".join(curpath), obj)) + except KeyError: + if default is not None: + obj = default + break + raise ModelError("Could not find path '{}' in '{}'" + .format("/".join(curpath), obj)) + name = part + + return self._new(name, obj) + + +__all__ = "Model", "ModelError" diff --git a/fstringen/fstringen_test.py b/fstringen/model_test.py similarity index 58% rename from fstringen/fstringen_test.py rename to fstringen/model_test.py index 0341b81..5422612 100644 --- a/fstringen/fstringen_test.py +++ b/fstringen/model_test.py @@ -1,5 +1,7 @@ import unittest -from fstringen import gen, Model, ModelError + +from .model import Model, ModelError + test_model = { "components": { @@ -47,8 +49,7 @@ def test_select_absolute(self): self.assertEqual( m.select("/components/componentB"), (Model.fromDict(test_model["components"]["componentB"], - name="componentB", - root=test_model))) + name="componentB"))) # Leaf selection self.assertEqual(m.select("/components/componentB/properties/color"), @@ -98,8 +99,8 @@ def test_select_star(self): [el.name for el in m.select("/components/*")], ["componentA", "componentB"]) self.assertEqual( - [el.refindicator for el in m.select("/components/*")], - [m.refindicator, m.refindicator]) + [el.refprefix for el in m.select("/components/*")], + [m.refprefix, m.refprefix]) self.assertEqual( [el.root for el in m.select("/components/*")], [m, m]) @@ -161,7 +162,7 @@ def test_select_array(self): m.select, "/animals/99") def test_select_ref(self): - m = Model.fromDict(test_model, refindicator="$") + m = Model.fromDict(test_model, refprefix="$") self.assertEqual(m.select("/components/componentB/properties/parent"), "$/components/componentA") @@ -216,290 +217,3 @@ def test_select_call(self): self.assertRaisesRegex(ModelError, "Could not find path .*", m, "attr") self.assertEqual(m("attr", "default value"), "default value") - - -class TestGen(unittest.TestCase): - def test_simple(self): - @gen() - def fn(): - a = 1 - return f"""* - {a} - *""" - - self.assertEqual(fn(), "1") - - @gen() - def fn2(): - a = 1 - return f"""* - x - {a} - *""" - - self.assertEqual(fn2(), "x\n 1") - - @gen() - def fn2(): - a = 1 - b = "something" - c = "." - - return f"""* - x - {a} - {b} - {c} - *""" - - self.assertEqual(fn2(), "x\n 1\nsomething\n .") - - def test_basic_newline_equivalence(self): - @gen() - def fn1(): - return f"""* - ... - *""" # noqa - - @gen() - def fn2(): - return f"""*...*""" # noqa - - @gen() - def fn3(): - return f"""* - ...*""" # noqa - - @gen() - def fn4(): - return f"""*... - *""" # noqa - - self.assertEqual(fn1(), "...") - self.assertEqual(fn2(), "...") - self.assertEqual(fn3(), "...") - self.assertEqual(fn4(), "...") - - def test_quotes(self): - @gen() - def fn(): - a = 1 - return f"""* - \\"{a}\\" - *""" - self.assertEqual(fn(), "\"1\"") - - @gen() - def fn2(): - a = 1 - return f"""*\\"{a}\\*""" - self.assertEqual(fn(), "\"1\"") - - def test_blank_lines(self): - @gen() - def fn(): - a = 1 - return f"""* - - {a} - *""" - - self.assertEqual(fn(), "\n1") - - @gen() - def fn2(): - a = 1 - return f"""* - - {a} - - *""" - - self.assertEqual(fn2(), "\n1\n") - - @gen() - def fn2(): - a = 1 - return f"""* - - x: - {a} - - - *""" - - self.assertEqual(fn2(), "\nx:\n 1\n\n") - - def test_list(self): - @gen() - def fn(): - elements = [1, 2] - return f"""* - {elements} - *""" - - self.assertEqual(fn(), "1\n2") - - @gen() - def fn(): - elements = [1, 2] - return f"""* - x - {elements} - *""" - - self.assertEqual(fn(), "x\n 1\n 2") - - def test_single_line(self): - @gen() - def fn(): - a = "y\nx" - return f"""* {a} *""" - - self.assertEqual(fn(), " y\nx ") - - @gen() - def fn2(): - a = [1, 2] - return f"""*{a}*""" - - self.assertEqual(fn2(), "1\n2") - - def test_indent(self): - @gen() - def multiline_string(): - return f"""* - multiline - string - *""" # noqa - - # Indentation-level of outside code shouldn't interfere - @gen() - def fn(cond): - a = [1, 2] - if cond: - return f"""* - {a} - *""" - return f"""* - {a} - *""" - self.assertEqual(fn(False), "1\n2") - self.assertEqual(fn(True), "1\n2") - - # Multi-line strings should be dedented - @gen() - def fn2(): - a = [multiline_string(), multiline_string()] - return f"""* - {a} - *""" - self.assertEqual(fn2(), "multiline\n string\nmultiline\n string") - - def test_dict(self): - @gen() - def fn(): - a = {"x": 1} - return f"""* - dict: {a} - *""" - - self.assertEqual(fn(), "dict: {'x': 1}") - - def test_return_none(self): - @gen() - def fn_none(): - return - - @gen() - def fn(): - a = "2" # noqa - l = [1, None, 2] # noqa - return f"""* - a: 2{fn_none()} - {l} - {fn_none()} - b - *""" - - # When None is part of a list, it should be hidden - # When None is a direct value, it shows as "" - self.assertEqual(fn(), "a: 2\n1\n2\n\nb") - - def test_call(self): - @gen() - def abc(): - return "abc" - - @gen() - def fn(): - a = {"x": 1} - return f"""* - dict: {a} - *""" - - @gen() - def fn2(): - return f"""* - {abc()} call: - {fn()} - *""" - - self.assertEqual(fn2(), "abc call:\n dict: {'x': 1}") - - -class TestIntegration(unittest.TestCase): - # TODO: test file generation - - def test_gen_select(self): - @gen() - def gen_color(component): - return f"""* - color: {component.select("properties/color")} - *""" - - @gen() - def gen_component(component): - parent = None - if component.has("properties/parent"): - parent = component.select("properties/parent->").name - parent = f"parent: {parent}" - - return f"""* - # Component {component.select("properties/name")}: - {parent} - {gen_color(component)} - nicknames: - {["- " + x for x in component.select("properties/nicknames")]} - - *""" - - @gen() - def gen_all(model): - components = [ - gen_component(component) - for component in model.select("/components/*") - ] - - return f"""* - {components} - *""" - - m = Model.fromDict(test_model, refindicator="$") - self.assertEqual( - gen_all(m), """# Component componentA: - - color: blue - nicknames: - - cA - - compA - - A - -# Component componentB: - parent: componentA - color: red - nicknames: - - cB - - compB - - B -""") diff --git a/setup.py b/setup.py index 6f01fd8..25010f8 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="fstringen", - version="0.0.11", + version="0.0.12", author="Allan Vidal", description="A text generator based on f-strings", long_description=long_description,