From 6a0b7976dc6e9b928cc637c62c8308b79c9fca7f Mon Sep 17 00:00:00 2001 From: dbrembilla Date: Thu, 30 Jun 2022 18:54:04 +0200 Subject: [PATCH] better_documentation --- README.md | 6 ++++-- docs/source/config.rst | 34 ++++++++++++++++++++++++++++++++-- poetry.lock | 34 +++++++++++++++++----------------- pyproject.toml | 2 +- requirements.txt | 2 +- test/ramose.py | 9 ++++++++- 6 files changed, 63 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 4247bd7..4de85ac 100644 --- a/README.md +++ b/README.md @@ -225,6 +225,8 @@ These functions can take parameters as input, while the first unspecified parame ``` In addition, it is possible to run multiple functions sequentially by concatenating them with `"-->"` in the API specification document. In this case the output of the function `f_i` will becomes the input result table of the function `f_i+1`. + +The postprocess function should output a tuple containing the result and whether the function needs to return the type of values in the result. ## Run RAMOSE @@ -319,10 +321,10 @@ call_1 = "%s/%s/%s" % (api_base_1, operation_url_1, request) call_2 = "%s/%s/%s" % (api_base_2, operation_url_2, request) op1 = api_manager.get_op(call_1) -status1, result1 = op1.exec() +status1, result1, result_format1 = op1.exec() op2 = api_manager.get_op(call_2) -status2, result2 = op2.exec() +status2, result2, result_format2 = op2.exec() ``` ## Other functionalities and examples diff --git a/docs/source/config.rst b/docs/source/config.rst index f5802ad..f98d535 100644 --- a/docs/source/config.rst +++ b/docs/source/config.rst @@ -257,9 +257,39 @@ Example: .. code-block:: none - #postprocess postprocess_oci(oci) --> another_process(oci) + #postprocess split_dois(dois) or - #postprocess postprocess_metadata(doi) + #postprocess distinct() + +The preprocess functions should return a tuple explain a tuple of values defining how the particular value passed in the dictionary must be changed. +The postprocess functions should return a tuple having as second element a boolean for whether the function should return the type of value specified. +Example of a preprocess and postprocess functions: + +.. code-block:: python + + #Preprocess + def split_dois(s): + return "\"%s\"" % "\" \"".join(s.split("__")), + + #Postprocess + def distinct(res): + header = res[0] + doi_field = header.index("doi") + result = [header] + + dois = set() + for row in res[1:]: + cur_doi = row[doi_field] + if cur_doi not in dois: + dois.add(cur_doi) + result.append(row) + + return result, True + + + + + #method diff --git a/poetry.lock b/poetry.lock index 3f1ba52..8f32ca4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -16,11 +16,11 @@ python-versions = "*" [[package]] name = "charset-normalizer" -version = "2.0.12" +version = "2.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "main" optional = false -python-versions = ">=3.5.0" +python-versions = ">=3.6.0" [package.extras] unicode_backport = ["unicodedata2"] @@ -62,7 +62,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "importlib-metadata" -version = "4.11.4" +version = "4.12.0" description = "Read metadata from Python packages" category = "main" optional = false @@ -75,7 +75,7 @@ zipp = ">=0.5" [package.extras] docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] [[package]] name = "isodate" @@ -142,7 +142,7 @@ diagrams = ["railroad-diagrams", "jinja2"] [[package]] name = "python-dateutil" -version = "2.8.1" +version = "2.8.2" description = "Extensions to the standard Python datetime module" category = "main" optional = false @@ -171,7 +171,7 @@ tests = ["berkeleydb", "html5lib", "networkx", "pytest", "pytest-cov", "pytest-s [[package]] name = "requests" -version = "2.28.0" +version = "2.28.1" description = "Python HTTP for Humans." category = "main" optional = false @@ -179,13 +179,13 @@ python-versions = ">=3.7, <4" [package.dependencies] certifi = ">=2017.4.17" -charset-normalizer = ">=2.0.0,<2.1.0" +charset-normalizer = ">=2,<3" idna = ">=2.5,<4" urllib3 = ">=1.21.1,<1.27" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "six" @@ -244,7 +244,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "e93bb2de9cd27a0f2b92207b558bbc78d2ab19c9ca9d3053efbb7bb697f4eecd" +content-hash = "6f6a60a7f79b44331d1506947ecaa881c9bdc5e5338c47d13f899eb0b5fef302" [metadata.files] certifi = [ @@ -256,8 +256,8 @@ chardet = [ {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"}, ] charset-normalizer = [ - {file = "charset-normalizer-2.0.12.tar.gz", hash = "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597"}, - {file = "charset_normalizer-2.0.12-py3-none-any.whl", hash = "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"}, + {file = "charset-normalizer-2.1.0.tar.gz", hash = "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413"}, + {file = "charset_normalizer-2.1.0-py3-none-any.whl", hash = "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5"}, ] click = [ {file = "Click-7.0-py2.py3-none-any.whl", hash = "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13"}, @@ -272,8 +272,8 @@ idna = [ {file = "idna-2.8.tar.gz", hash = "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407"}, ] importlib-metadata = [ - {file = "importlib_metadata-4.11.4-py3-none-any.whl", hash = "sha256:c58c8eb8a762858f49e18436ff552e83914778e50e9d2f1660535ffb364552ec"}, - {file = "importlib_metadata-4.11.4.tar.gz", hash = "sha256:5d26852efe48c0a32b0509ffbc583fda1a2266545a78d104a6f4aff3db17d700"}, + {file = "importlib_metadata-4.12.0-py3-none-any.whl", hash = "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23"}, + {file = "importlib_metadata-4.12.0.tar.gz", hash = "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670"}, ] isodate = [ {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, @@ -350,16 +350,16 @@ pyparsing = [ {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, ] python-dateutil = [ - {file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"}, - {file = "python_dateutil-2.8.1-py2.py3-none-any.whl", hash = "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"}, + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, ] rdflib = [ {file = "rdflib-6.1.1-py3-none-any.whl", hash = "sha256:fc81cef513cd552d471f2926141396b633207109d0154c8e77926222c70367fe"}, {file = "rdflib-6.1.1.tar.gz", hash = "sha256:8dbfa0af2990b98471dacbc936d6494c997ede92fd8ed693fb84ee700ef6f754"}, ] requests = [ - {file = "requests-2.28.0-py3-none-any.whl", hash = "sha256:bc7861137fbce630f17b03d3ad02ad0bf978c844f3536d0edda6499dafce2b6f"}, - {file = "requests-2.28.0.tar.gz", hash = "sha256:d568723a7ebd25875d8d1eaf5dfa068cd2fc8194b2e483d7b1f7c81918dbec6b"}, + {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, + {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, ] six = [ {file = "six-1.13.0-py2.py3-none-any.whl", hash = "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd"}, diff --git a/pyproject.toml b/pyproject.toml index ca7c82e..7e40ce7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ itsdangerous="1.1.0" Jinja2="2.11.3" Markdown="3.1.1" MarkupSafe="1.1.1" -python-dateutil="2.8.1" +python-dateutil="^2.8.2" requests="^2.22.0" six="1.13.0" urllib3="1.26.5" diff --git a/requirements.txt b/requirements.txt index cda79c0..5ce1226 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ itsdangerous==1.1.0 Jinja2==2.11.3 Markdown==3.1.1 MarkupSafe==1.1.1 -python-dateutil==2.8.1 +python-dateutil==2.8.2 requests>=2.22.0, <3.0.0 six==1.13.0 urllib3==1.26.5 diff --git a/test/ramose.py b/test/ramose.py index 430681b..5812a0d 100644 --- a/test/ramose.py +++ b/test/ramose.py @@ -1235,7 +1235,10 @@ def postprocess(self, res, op_item, addon): respectively. In addition, it is possible to run multiple functions sequentially by concatenating them with "-->" in the API specification document. In this case the output of the function f_i will becomes the input result table of the function f_i+1. + + The postprocess function should output a tuple containing the result and whether the function needs to return the type of values in the result. """ + result = res if "postprocess" in op_item: @@ -1388,7 +1391,7 @@ def remove_types(self, res): def exec(self, method="get", content_type="application/json"): """ This method takes in input the the HTTP method to use for the call - and the content type to return, and execute the operation as indicated + and the content type to return, and executes the operation as indicated in the specification file, by running (in the following order): 1. the methods to preprocess the query; @@ -1471,7 +1474,9 @@ def exec(self, method="get", content_type="application/json"): class APIManager(object): + # Fixing max size for CSV + @staticmethod def __max_size_csv(): from sys import maxsize @@ -1485,7 +1490,9 @@ def __max_size_csv(): maxInt = int(maxInt/10) # Constructor: START + def __init__(self, conf_files): + """ This is the constructor of the APIManager class. It takes in input a list of API configuration files, each defined according to the Hash Format and following a particular structure, and stores all the operations