From 6296f0a4720443553dd0e4677462ed40c508c6fd Mon Sep 17 00:00:00 2001 From: Sveinung Gundersen Date: Fri, 15 Dec 2023 11:37:10 +0100 Subject: [PATCH] Tmp commit 2 --- pyproject.toml | 3 +++ src/omnipy/__init__.py | 18 ++++++++++++++++++ src/omnipy/data/dataset.py | 11 ++++++++++- src/omnipy/data/model.py | 14 ++++++++++++-- src/omnipy/util/helpers.py | 24 +++++++++++++++++++++++- src/omnipy/util/mako_helpers.py | 25 ++----------------------- 6 files changed, 68 insertions(+), 27 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9b66d8e7..fd79ce78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,9 @@ typing-inspect = "^0.8.0" #orjson = "^3.8.0" #python-slugify = "^7.0.0" isort = "^5.12.0" +pathspec = "0.12.1" +tabulate = "^0.9.0" +devtools = "^0.12.2" [tool.poetry.group.dev.dependencies] deepdiff = "^6.2.1" diff --git a/src/omnipy/__init__.py b/src/omnipy/__init__.py index 5d9aad77..9e7a627d 100644 --- a/src/omnipy/__init__.py +++ b/src/omnipy/__init__.py @@ -1,5 +1,6 @@ __version__ = '0.12.2' +import importlib import os import sys from typing import Optional @@ -7,6 +8,9 @@ from omnipy.data.dataset import Dataset from omnipy.data.model import Model from omnipy.hub.runtime import Runtime +from omnipy.util.helpers import recursive_module_import + +# from omnipy.util.helpers import recursive_module_import ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -25,3 +29,17 @@ def _get_runtime() -> Optional['Runtime']: runtime: Optional['Runtime'] = _get_runtime() __all__ = [Model, Dataset] + + +def __getattr__(attr_name: str) -> object: + omnipy = importlib.import_module(__name__) + all_modules = [] + recursive_module_import(omnipy, all_modules) + print(all_modules) + + +# +# +# print(__file__) +# print(__name__) +# print(list(globals().keys())) diff --git a/src/omnipy/data/dataset.py b/src/omnipy/data/dataset.py index f317b890..d98a1699 100644 --- a/src/omnipy/data/dataset.py +++ b/src/omnipy/data/dataset.py @@ -9,6 +9,7 @@ from pydantic.fields import Undefined, UndefinedType from pydantic.generics import GenericModel from pydantic.utils import lenient_issubclass +from tabulate import tabulate from omnipy.data.model import _cleanup_name_qualname_and_module, Model from omnipy.util.helpers import is_optional, is_strict_subclass, remove_forward_ref_notation @@ -358,7 +359,7 @@ def save(self, directory: str): def load(self, directory: str): serializer_registry = self._get_serializer_registry() - return serializer_registry.load_from_tar_file_path(self, directory, self) + self.include(serializer_registry.load_from_tar_file_path(self, directory, self)) @staticmethod def _get_serializer_registry(): @@ -388,6 +389,14 @@ def __repr__(self) -> str: def __repr_args__(self): return [(k, v.contents) for k, v in self.data.items()] + def __repr__(self): + return tabulate( + ((k, type(v).__name__, len(v) if hasattr(v, '__len__') else 'N/A') \ + for k, v in self.items()), + ('Data file name', 'Type', 'Length'), + tablefmt="rounded_outline", + ) + # TODO: Use json serializer package from the pydantic config instead of 'json' diff --git a/src/omnipy/data/model.py b/src/omnipy/data/model.py index 7eca6478..fffc57bf 100644 --- a/src/omnipy/data/model.py +++ b/src/omnipy/data/model.py @@ -15,7 +15,7 @@ TypeVar, Union) -# from orjson import orjson +from devtools import debug from pydantic import NoneIsNotAllowedError from pydantic import Protocol as PydanticProtocol from pydantic import root_validator, ValidationError @@ -24,6 +24,7 @@ from pydantic.main import ModelMetaclass, validate_model from pydantic.typing import display_as_type, is_none_type from pydantic.utils import lenient_isinstance, lenient_issubclass +from tabulate import tabulate from omnipy.data.methodinfo import MethodInfo, SPECIAL_METHODS_INFO from omnipy.util.contexts import AttribHolder, LastErrorHolder, nothing @@ -508,7 +509,8 @@ def _check_for_root_key(self) -> None: '\t"class MyNumberList(Model[list[int]]): ..."') def __setattr__(self, attr: str, value: Any) -> None: - if attr in ['__module__'] + list(self.__dict__.keys()) and attr not in [ROOT_KEY]: + if attr in ['__module__'] + list( + self.__dict__.keys()) and attr not in [ROOT_KEY] or attr in ['__repr__']: super().__setattr__(attr, value) else: if attr in ['contents']: @@ -617,3 +619,11 @@ def __repr__(self) -> str: def __repr_args__(self): return [(None, self.contents)] + + def __str__(self): + return tabulate( + ((self.__class__.__name__, ''), ('Raw file preview', 'Data structure'), + (str(self.to_data()), debug.format(self))), + maxcolwidths=[(40, 40)], + tablefmt="rounded_outline", + ) diff --git a/src/omnipy/util/helpers.py b/src/omnipy/util/helpers.py index d780b7d6..020c254a 100644 --- a/src/omnipy/util/helpers.py +++ b/src/omnipy/util/helpers.py @@ -1,8 +1,9 @@ from collections.abc import Hashable, Iterable from copy import copy, deepcopy import inspect +from inspect import getmodule, isclass import locale as pkg_locale -from types import GenericAlias, UnionType +from types import GenericAlias, ModuleType, UnionType from typing import (Annotated, Any, cast, @@ -192,3 +193,24 @@ def last_snapshot_taken_of_same_obj(self, obj: object) -> bool: def differs_from_last_snapshot(self, obj: object) -> bool: self._assert_not_empty() return not all_equals(self._last_snapshot.obj_copy, obj) + + +def _is_internal_module(module: ModuleType, imported_modules: list[ModuleType]): + return module not in imported_modules and module.__name__.startswith('omnipy') + + +def recursive_module_import(module: ModuleType, imported_modules: list[ModuleType] = []): + module_vars = vars(module) + imported_modules.append(module) + + for val in module_vars.values(): + if isclass(val): + for base_cls in val.__bases__: + base_cls_module = getmodule(base_cls) + if base_cls_module and _is_internal_module(base_cls_module, imported_modules): + module_vars = create_merged_dict( + recursive_module_import(base_cls_module, imported_modules), + module_vars, + ) + + return module_vars diff --git a/src/omnipy/util/mako_helpers.py b/src/omnipy/util/mako_helpers.py index 0729b5df..9a19403d 100644 --- a/src/omnipy/util/mako_helpers.py +++ b/src/omnipy/util/mako_helpers.py @@ -1,5 +1,5 @@ import ast -from inspect import formatannotation, getmodule, isclass, isgeneratorfunction, Signature +from inspect import formatannotation, isgeneratorfunction, Signature import os from types import ModuleType from typing import Any, get_type_hints @@ -7,7 +7,7 @@ from docstring_parser import DocstringParam, DocstringReturns from pdocs.doc import Doc, External, Function, Module -from omnipy.util.helpers import create_merged_dict +from omnipy.util.helpers import recursive_module_import IGNORED = None IGNORE_PARAMS = ['cls', 'self'] @@ -109,27 +109,6 @@ def get_type_name_from_annotation(module: ModuleType, annotation, empty_obj): return type_name -def _is_internal_module(module: ModuleType, imported_modules: list[ModuleType]): - return module not in imported_modules and module.__name__.startswith('omnipy') - - -def recursive_module_import(module: ModuleType, imported_modules: list[ModuleType] = []): - module_vars = vars(module) - imported_modules.append(module) - - for val in module_vars.values(): - if isclass(val): - for base_cls in val.__bases__: - base_cls_module = getmodule(base_cls) - if base_cls_module and _is_internal_module(base_cls_module, imported_modules): - module_vars = create_merged_dict( - recursive_module_import(base_cls_module, imported_modules), - module_vars, - ) - - return module_vars - - def convert_to_qual_name_type_hint_str(module: ModuleType, type_hint: Any) -> str: def fixed_get_type_hints(obj: Any) -> str: """