From 5c718535f0acce82dcaf2723b124b2bb4753f7ca Mon Sep 17 00:00:00 2001 From: Vicente Adolfo Bolea Sanchez Date: Tue, 21 Nov 2023 19:44:20 -0500 Subject: [PATCH] python: rewrite Python high level API in python --- flake8.cfg => .flake8 | 0 .gitignore | 4 + CMakeLists.txt | 8 + bindings/Python/CMakeLists.txt | 42 +- bindings/Python/__init__.py.in | 2 +- bindings/Python/py11File.cpp | 321 --------- bindings/Python/py11File.h | 127 ---- bindings/Python/py11File.tcc | 98 --- bindings/Python/py11FileMPI.cpp | 34 - bindings/Python/py11IO.cpp | 7 +- bindings/Python/py11IO.h | 3 +- bindings/Python/py11Variable.cpp | 14 + bindings/Python/py11Variable.h | 17 + bindings/Python/py11glue.cpp | 666 +----------------- cmake/DetectOptions.cmake | 6 +- pyproject.toml | 17 + python/CMakeLists.txt | 18 + python/adios2/__init__.py | 14 + python/adios2/adios.py | 40 ++ python/adios2/attribute.py | 23 + python/adios2/engine.py | 68 ++ python/adios2/file.py | 199 ++++++ python/adios2/file_reader.py | 185 +++++ python/adios2/io.py | 103 +++ python/adios2/operator.py | 12 + python/adios2/stream.py | 369 ++++++++++ python/adios2/variable.py | 71 ++ .../ci/cmake/ci-win2019-vs2019-serial.cmake | 1 + .../ci/cmake/ci-win2022-vs2022-serial.cmake | 1 + scripts/ci/scripts/run-flake8.sh | 2 +- scripts/developer/run-clang-format.sh | 4 +- testing/adios2/CMakeLists.txt | 4 + testing/adios2/bindings/python/CMakeLists.txt | 11 - .../bindings/python/TestBPBlocksInfo.py | 2 +- .../bindings/python/TestBPPNGHighLevelAPI.py | 92 --- .../bindings/python/TestBPReadMultisteps.py | 2 +- .../bindings/python/TestBPSelectSteps.py | 2 +- .../bindings/python/TestBPWriteRead2D.py | 2 +- .../bindings/python/TestBPWriteReadString.py | 34 +- .../bindings/python/TestBPWriteReadTypes.py | 2 +- .../python/TestBPWriteReadTypes_nompi.py | 2 +- .../python/TestBPWriteTypesHighLevelAPI.py | 365 ---------- .../TestBPWriteTypesHighLevelAPILocal.py | 88 --- .../bindings/python/TestGetException_nompi.py | 2 +- .../adios2/bindings/python/TestNullEngine.py | 2 +- testing/adios2/bindings/python/TestQuery.py | 2 +- .../bindings/python/TestQueryLocalArray.py | 2 +- testing/adios2/python/CMakeLists.txt | 34 + testing/adios2/python/TestADIOS.py | 56 ++ testing/adios2/python/TestAttribute.py | 34 + .../python/TestBPChangingShapeHighLevelAPI.py | 20 +- .../adios2/python/TestBPPNGHighLevelAPI.py | 112 +++ .../adios2/python/TestBPWriteReadString.py | 52 ++ .../python/TestBPWriteTypesHighLevelAPI.py | 354 ++++++++++ .../TestBPWriteTypesHighLevelAPILocal.py | 81 +++ .../TestBPWriteTypesHighLevelAPI_HDF5.py | 131 ++-- .../python/TestBPZfpHighLevelAPI.py | 33 +- testing/adios2/python/TestEngine.py | 95 +++ testing/adios2/python/TestFile.py | 21 + testing/adios2/python/TestIO.py | 102 +++ testing/adios2/python/TestOperator.py | 38 + testing/adios2/python/TestStream.py | 50 ++ testing/adios2/python/TestVariable.py | 66 ++ testing/adios2/python/adios2NPTypes.py | 88 +++ 64 files changed, 2515 insertions(+), 1942 deletions(-) rename flake8.cfg => .flake8 (100%) delete mode 100644 bindings/Python/py11File.cpp delete mode 100644 bindings/Python/py11File.h delete mode 100644 bindings/Python/py11File.tcc delete mode 100644 bindings/Python/py11FileMPI.cpp create mode 100644 python/CMakeLists.txt create mode 100644 python/adios2/__init__.py create mode 100644 python/adios2/adios.py create mode 100644 python/adios2/attribute.py create mode 100644 python/adios2/engine.py create mode 100644 python/adios2/file.py create mode 100644 python/adios2/file_reader.py create mode 100644 python/adios2/io.py create mode 100644 python/adios2/operator.py create mode 100644 python/adios2/stream.py create mode 100644 python/adios2/variable.py delete mode 100644 testing/adios2/bindings/python/TestBPPNGHighLevelAPI.py delete mode 100644 testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPI.py delete mode 100644 testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPILocal.py create mode 100644 testing/adios2/python/CMakeLists.txt create mode 100644 testing/adios2/python/TestADIOS.py create mode 100644 testing/adios2/python/TestAttribute.py rename testing/adios2/{bindings => }/python/TestBPChangingShapeHighLevelAPI.py (58%) create mode 100644 testing/adios2/python/TestBPPNGHighLevelAPI.py create mode 100644 testing/adios2/python/TestBPWriteReadString.py create mode 100644 testing/adios2/python/TestBPWriteTypesHighLevelAPI.py create mode 100644 testing/adios2/python/TestBPWriteTypesHighLevelAPILocal.py rename testing/adios2/{bindings => }/python/TestBPWriteTypesHighLevelAPI_HDF5.py (73%) rename testing/adios2/{bindings => }/python/TestBPZfpHighLevelAPI.py (60%) create mode 100644 testing/adios2/python/TestEngine.py create mode 100644 testing/adios2/python/TestFile.py create mode 100644 testing/adios2/python/TestIO.py create mode 100644 testing/adios2/python/TestOperator.py create mode 100644 testing/adios2/python/TestStream.py create mode 100644 testing/adios2/python/TestVariable.py create mode 100644 testing/adios2/python/adios2NPTypes.py diff --git a/flake8.cfg b/.flake8 similarity index 100% rename from flake8.cfg rename to .flake8 diff --git a/.gitignore b/.gitignore index 81ff18108b..35a8d5ab87 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,7 @@ docs/pyvenv.cfg # Visual Studio .vs/ CMakeSettings.json + +# Python wheels stuff + +*.egg-info/ diff --git a/CMakeLists.txt b/CMakeLists.txt index b2ed80530e..fccf4abb95 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -328,6 +328,13 @@ add_subdirectory(source) #------------------------------------------------------------------------------# add_subdirectory(bindings) +#------------------------------------------------------------------------------# +# Language Libraries +#------------------------------------------------------------------------------# +if(ADIOS2_HAVE_Python) + add_subdirectory(python) +endif() + #------------------------------------------------------------------------------# # Plugins #------------------------------------------------------------------------------# @@ -341,6 +348,7 @@ if(ADIOS2_BUILD_EXAMPLES) add_subdirectory(examples) endif() + #------------------------------------------------------------------------------# # Testing #------------------------------------------------------------------------------# diff --git a/bindings/Python/CMakeLists.txt b/bindings/Python/CMakeLists.txt index 404b084a37..f6a2d1842f 100644 --- a/bindings/Python/CMakeLists.txt +++ b/bindings/Python/CMakeLists.txt @@ -7,14 +7,12 @@ Python_add_library(adios2_py MODULE py11Engine.cpp py11Operator.cpp py11Query.cpp - py11File.cpp py11File.tcc py11glue.cpp ) -target_compile_definitions(adios2_py PRIVATE "ADIOS2_PYTHON_MODULE_NAME=adios2${ADIOS2_LIBRARY_SUFFIX}") +target_compile_definitions(adios2_py PRIVATE "ADIOS2_PYTHON_MODULE_NAME=adios2_bindings${ADIOS2_LIBRARY_SUFFIX}") if(ADIOS2_HAVE_MPI) target_sources(adios2_py PRIVATE py11ADIOSMPI.cpp - py11FileMPI.cpp py11IOMPI.cpp ) set(maybe_adios2_cxx11_mpi adios2_cxx11_mpi) @@ -29,44 +27,42 @@ target_link_libraries(adios2_py PRIVATE ${maybe_adios2_cxx11_mpi} adios2_cxx11 ${maybe_adios2_core_mpi} adios2_core adios2::thirdparty::pybind11 - ${maybe_mpi4py} Python::NumPy + ${maybe_mpi4py} ) configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/__init__.py.in - ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/__init__.py + ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings/__init__.py @ONLY ) set_target_properties(adios2_py PROPERTIES CXX_VISIBILITY_PRESET hidden - OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX} - ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2 - LIBRARY_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2 - RUNTIME_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2 - PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2 - COMPILE_PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2 + OUTPUT_NAME adios2_bindings${ADIOS2_LIBRARY_SUFFIX} + ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings + PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings + COMPILE_PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings ) -string(REGEX REPLACE "[^/]+" ".." relative_base "${CMAKE_INSTALL_PYTHONDIR}/adios2") -if(CMAKE_SYSTEM_NAME MATCHES "Linux") - if (NOT ADIOS2_USE_PIP) - set_target_properties(adios2_py PROPERTIES - INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}" - ) - endif() -endif() - set(install_location ${CMAKE_INSTALL_PYTHONDIR}) if (ADIOS2_USE_PIP) set(install_location ${CMAKE_INSTALL_LIBDIR}) endif() +string(REGEX REPLACE "[^/]+" ".." relative_base "${install_location}/adios2/bindings") +if(CMAKE_SYSTEM_NAME MATCHES "Linux") + set_property(TARGET adios2_py APPEND PROPERTY + INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}" + ) +endif() + install(TARGETS adios2_py - DESTINATION ${install_location} + DESTINATION ${install_location}/adios2/bindings COMPONENT adios2_python-python ) -install(FILES ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/__init__.py - DESTINATION ${install_location} +install(FILES ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings/__init__.py + DESTINATION ${install_location}/adios2/bindings COMPONENT adios2_python-python ) install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/test diff --git a/bindings/Python/__init__.py.in b/bindings/Python/__init__.py.in index 01932e43ce..fb6c0ad344 100644 --- a/bindings/Python/__init__.py.in +++ b/bindings/Python/__init__.py.in @@ -1,3 +1,3 @@ -from .adios2@ADIOS2_LIBRARY_SUFFIX@ import * +from .adios2_bindings@ADIOS2_LIBRARY_SUFFIX@ import * __version__ = "@ADIOS2_VERSION@" diff --git a/bindings/Python/py11File.cpp b/bindings/Python/py11File.cpp deleted file mode 100644 index 17b456e4d5..0000000000 --- a/bindings/Python/py11File.cpp +++ /dev/null @@ -1,321 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * py11File.cpp - * - * Created on: Mar 6, 2018 - * Author: William F Godoy godoywf@ornl.gov - */ - -#include "py11File.h" -#include "py11File.tcc" - -#include -#include - -#include "adios2/common/ADIOSMacros.h" -#include "adios2/helper/adiosFunctions.h" - -#include "py11types.h" - -namespace adios2 -{ -namespace py11 -{ - -File::File(const std::string &name, const std::string mode, const std::string engineType) -: m_Name(name), m_Mode(mode), - m_Stream(std::make_shared(name, ToMode(mode), engineType, "Python")) -{ -} - -File::File(const std::string &name, const std::string mode, const std::string &configFile, - const std::string ioInConfigFile) -: m_Name(name), m_Mode(mode), - m_Stream(std::make_shared(name, ToMode(mode), configFile, ioInConfigFile, "Python")) -{ -} - -void File::SetParameter(const std::string key, const std::string value) noexcept -{ - m_Stream->m_IO->SetParameter(key, value); -} - -void File::SetParameters(const Params ¶meters) noexcept -{ - m_Stream->m_IO->SetParameters(parameters); -} - -size_t File::AddTransport(const std::string type, const Params ¶meters) -{ - return m_Stream->m_IO->AddTransport(type, parameters); -} - -std::map -File::AvailableVariables(const std::vector &keys) noexcept -{ - const std::set keysSet = helper::VectorToSet(keys); - return m_Stream->m_IO->GetAvailableVariables(keysSet); -} - -std::map File::AvailableAttributes() noexcept -{ - return m_Stream->m_IO->GetAvailableAttributes(); -} - -void File::WriteAttribute(const std::string &name, const pybind11::array &array, - const std::string &variableName, const std::string separator, - const bool endStep) -{ - if (false) - { - } -#define declare_type(T) \ - else if (pybind11::isinstance>(array)) \ - { \ - m_Stream->WriteAttribute(name, reinterpret_cast(array.data()), \ - static_cast(array.size()), variableName, separator, \ - endStep); \ - } - ADIOS2_FOREACH_NUMPY_ATTRIBUTE_TYPE_1ARG(declare_type) -#undef declare_type - else - { - throw std::invalid_argument("ERROR: adios2 file write attribute " + name + - ", either numpy type is not supported or is not " - "c_style memory contiguous, in call to write\n"); - } -} - -void File::WriteAttribute(const std::string &name, const std::string &stringValue, - const std::string &variableName, const std::string separator, - const bool endStep) -{ - m_Stream->WriteAttribute(name, stringValue, variableName, separator, endStep); -} - -void File::WriteAttribute(const std::string &name, const std::vector &stringArray, - const std::string &variableName, const std::string separator, - const bool endStep) -{ - m_Stream->WriteAttribute(name, stringArray.data(), stringArray.size(), variableName, separator, - endStep); -} - -void File::Write(const std::string &name, const pybind11::array &array, const Dims &shape, - const Dims &start, const Dims &count, const bool endStep) -{ - if (false) - { - } -#define declare_type(T) \ - else if (pybind11::isinstance>(array)) \ - { \ - m_Stream->Write(name, reinterpret_cast(array.data()), shape, start, count, \ - vParams(), endStep); \ - } - ADIOS2_FOREACH_NUMPY_TYPE_1ARG(declare_type) -#undef declare_type - else - { - throw std::invalid_argument("ERROR: adios2 file write variable " + name + - ", either numpy type is not supported or is not " - "c_style memory contiguous, in call to write\n"); - } -} - -void File::Write(const std::string &name, const pybind11::array &array, const Dims &shape, - const Dims &start, const Dims &count, const adios2::vParams &operations, - const bool endStep) -{ - if (false) - { - } -#define declare_type(T) \ - else if (pybind11::isinstance>(array)) \ - { \ - m_Stream->Write(name, reinterpret_cast(array.data()), shape, start, count, \ - operations, endStep); \ - } - ADIOS2_FOREACH_NUMPY_TYPE_1ARG(declare_type) -#undef declare_type - else - { - throw std::invalid_argument( - "ERROR: adios2 file write variable " + name + - ", either numpy type is not supported or is not " - "c_style memory contiguous, in call to write with operations\n"); - } -} - -void File::Write(const std::string &name, const pybind11::array &array, const bool isLocalValue, - const bool endStep) -{ - if (isLocalValue) - { - Write(name, array, {adios2::LocalValueDim}, {}, {}, endStep); - } - else - { - Write(name, array, {}, {}, {}, endStep); - } -} - -void File::Write(const std::string &name, const std::string &stringValue, const bool isLocalValue, - const bool endStep) -{ - m_Stream->Write(name, stringValue, isLocalValue, endStep); -} - -bool File::GetStep() const { return const_cast(this)->m_Stream->GetStep(); } - -std::vector File::ReadString(const std::string &name, const size_t blockID) -{ - return m_Stream->Read(name, blockID); -} - -std::vector File::ReadString(const std::string &name, const size_t stepStart, - const size_t stepCount, const size_t blockID) -{ - return m_Stream->Read(name, Box(stepStart, stepCount), blockID); -} - -pybind11::array File::Read(const std::string &name, const size_t blockID) -{ - return Read(name, {}, {}, blockID); -} - -pybind11::array File::Read(const std::string &name, const Dims &start, const Dims &count, - const size_t blockID) -{ - const DataType type = m_Stream->m_IO->InquireVariableType(name); - - if (type == helper::GetDataType()) - { - const std::string value = m_Stream->Read(name, blockID).front(); - pybind11::array_t pyArray(Dims{value.size()}); - std::copy(value.begin(), value.end(), pyArray.mutable_data()); - return std::move(pyArray); - } - - return Read(name, start, count, 0, 0, blockID); -} - -pybind11::array File::Read(const std::string &name, const Dims &start, const Dims &count, - const size_t stepStart, const size_t stepCount, const size_t blockID) -{ - const DataType type = m_Stream->m_IO->InquireVariableType(name); - - if (type == DataType::None) - { - } -#define declare_type(T) \ - else if (type == helper::GetDataType()) \ - { \ - return DoRead(name, start, count, stepStart, stepCount, blockID); \ - } - ADIOS2_FOREACH_NUMPY_TYPE_1ARG(declare_type) -#undef declare_type - else - { - throw std::invalid_argument("ERROR: adios2 file read variable " + name + - ", type can't be mapped to a numpy type, in call to read\n"); - } - return pybind11::array(); -} - -pybind11::array File::ReadAttribute(const std::string &name, const std::string &variableName, - const std::string separator) -{ - const DataType type = m_Stream->m_IO->InquireAttributeType(name, variableName, separator); - - if (type == DataType::None) - { - } -#define declare_type(T) \ - else if (type == helper::GetDataType()) \ - { \ - core::Attribute *attribute = \ - m_Stream->m_IO->InquireAttribute(name, variableName, separator); \ - if (attribute->m_IsSingleValue) \ - { \ - pybind11::array_t pyArray({}); \ - pyArray.mutable_data()[0] = attribute->m_DataSingleValue; \ - } \ - pybind11::array_t pyArray(attribute->m_Elements); \ - m_Stream->ReadAttribute(name, pyArray.mutable_data(), variableName, separator); \ - return std::move(pyArray); \ - } - ADIOS2_FOREACH_NUMPY_ATTRIBUTE_TYPE_1ARG(declare_type) -#undef declare_type - else - { - throw std::invalid_argument("ERROR: adios2 file read attribute " + name + - ", type can't be mapped to a numpy type, in call to read\n"); - } - return pybind11::array(); -} - -std::vector File::ReadAttributeString(const std::string &name, - const std::string &variableName, - const std::string separator) -{ - const core::Attribute *attribute = - m_Stream->m_IO->InquireAttribute(name, variableName, separator); - - if (attribute == nullptr) - { - return std::vector(); - } - - std::vector data(attribute->m_Elements); - m_Stream->ReadAttribute(name, data.data(), variableName, separator); - return data; -} - -void File::EndStep() { m_Stream->EndStep(); } - -void File::Close() -{ - m_Stream->Close(); - m_Stream.reset(); -} - -size_t File::CurrentStep() const { return m_Stream->CurrentStep(); } - -size_t File::Steps() const { return m_Stream->Steps(); } - -// PRIVATE -adios2::Mode File::ToMode(const std::string mode) const -{ - adios2::Mode modeCpp = adios2::Mode::Undefined; - if (mode == "w") - { - modeCpp = adios2::Mode::Write; - } - else if (mode == "a") - { - modeCpp = adios2::Mode::Append; - } - else if (mode == "r") - { - modeCpp = adios2::Mode::Read; - } - else if (mode == "rra") - { - modeCpp = adios2::Mode::ReadRandomAccess; - } - else - { - throw std::invalid_argument("ERROR: adios2 mode " + mode + " for file " + m_Name + - " not supported, only \"r\", \"rra\", \"w\" and \"a\" (read, " - "readRandomAccess, write, append) " - "are valid modes, in call to open\n"); - } - - return modeCpp; -} - -} // end namespace py11 -} // end namespace adios2 diff --git a/bindings/Python/py11File.h b/bindings/Python/py11File.h deleted file mode 100644 index ef83a41def..0000000000 --- a/bindings/Python/py11File.h +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * py11File.h - * - * Created on: Mar 6, 2018 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_BINDINGS_PYTHON_PY11FILE_H_ -#define ADIOS2_BINDINGS_PYTHON_PY11FILE_H_ - -#include - -#include "adios2/common/ADIOSTypes.h" -#include "adios2/core/Stream.h" - -#if ADIOS2_USE_MPI -#include -#endif - -namespace adios2 -{ -namespace py11 -{ - -class File -{ -public: - const std::string m_Name; - const std::string m_Mode; - -#if ADIOS2_USE_MPI - File(const std::string &name, const std::string mode, MPI_Comm comm, - const std::string engineType = "BPFile"); - - File(const std::string &name, const std::string mode, MPI_Comm comm, - const std::string &configFile, const std::string ioInConfigFile); -#endif - - File(const std::string &name, const std::string mode, const std::string engineType = "BPFile"); - - File(const std::string &name, const std::string mode, const std::string &configFile, - const std::string ioInConfigFile); - - ~File() = default; - - void SetParameter(const std::string key, const std::string value) noexcept; - - void SetParameters(const Params ¶meters) noexcept; - - size_t AddTransport(const std::string type, const Params ¶meters = Params()); - - std::map - AvailableVariables(const std::vector &keys = std::vector()) noexcept; - - std::map AvailableAttributes() noexcept; - - void WriteAttribute(const std::string &name, const pybind11::array &array, - const std::string &variableName = "", const std::string separator = "/", - const bool endStep = false); - - void WriteAttribute(const std::string &name, const std::string &stringValue, - const std::string &variableName = "", const std::string separator = "/", - const bool endStep = false); - - void WriteAttribute(const std::string &name, const std::vector &stringArray, - const std::string &variableName = "", const std::string separator = "/", - const bool endStep = false); - - void Write(const std::string &name, const pybind11::array &array, const Dims &shape, - const Dims &start, const Dims &count, const bool endStep = false); - - void Write(const std::string &name, const pybind11::array &array, const Dims &shape, - const Dims &start, const Dims &count, const adios2::vParams &operations, - const bool endStep = false); - - void Write(const std::string &name, const pybind11::array &array, - const bool isLocalValue = false, const bool endStep = false); - - void Write(const std::string &name, const std::string &stringValue, - const bool isLocalValue = false, const bool endStep = false); - - bool GetStep() const; - - std::vector ReadString(const std::string &name, const size_t blockID = 0); - - std::vector ReadString(const std::string &name, const size_t stepStart, - const size_t stepCount, const size_t blockID = 0); - - pybind11::array Read(const std::string &name, const size_t blockID = 0); - - pybind11::array Read(const std::string &name, const Dims &start, const Dims &count, - const size_t blockID = 0); - - pybind11::array Read(const std::string &name, const Dims &start, const Dims &count, - const size_t stepStart, const size_t stepCount, const size_t blockID = 0); - - pybind11::array ReadAttribute(const std::string &name, const std::string &variableName = "", - const std::string separator = "/"); - - std::vector ReadAttributeString(const std::string &name, - const std::string &variableName = "", - const std::string separator = "/"); - - void EndStep(); - - void Close(); - - size_t CurrentStep() const; - - size_t Steps() const; - -private: - std::shared_ptr m_Stream; - adios2::Mode ToMode(const std::string mode) const; - - template - pybind11::array DoRead(const std::string &name, const Dims &start, const Dims &count, - const size_t stepStart, const size_t stepCount, const size_t blockID); -}; - -} // end namespace py11 -} // end namespace adios2 - -#endif /* ADIOS2_BINDINGS_PYTHON_FILEPY_H_ */ diff --git a/bindings/Python/py11File.tcc b/bindings/Python/py11File.tcc deleted file mode 100644 index 6cf41d4d4f..0000000000 --- a/bindings/Python/py11File.tcc +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * py11File.tcc - * - * Created on: May 29, 2019 - * Author: William F Godoy godoywf@ornl.gov - */ - -#ifndef ADIOS2_BINDINGS_PYTHON_PY11FILE_TCC_ -#define ADIOS2_BINDINGS_PYTHON_PY11FILE_TCC_ - -#include "py11File.h" - -namespace adios2 -{ -namespace py11 -{ - -template -pybind11::array File::DoRead(const std::string &name, const Dims &_start, const Dims &_count, - const size_t stepStart, const size_t stepCount, const size_t blockID) -{ - core::Variable &variable = *m_Stream->m_IO->InquireVariable(name); - Dims &shape = variable.m_Shape; - Dims start = _start; - Dims count = _count; - - if (variable.m_ShapeID == ShapeID::GlobalValue) - { - if (!(_start.empty() && _count.empty())) - { - throw std::invalid_argument("when reading a scalar, start and " - "count cannot be specified.\n"); - } - } - - if (variable.m_ShapeID == ShapeID::LocalArray) - { - variable.SetBlockSelection(blockID); - } - else - { - if (blockID != 0) - { - throw std::invalid_argument("blockId can only be specified when reading LocalArrays."); - } - } - - if (start.empty()) - { - // default start to be (0, 0, ...) - start = Dims(shape.size()); - } - - if (count.empty()) - { - // does the right thing for global and local arrays - count = variable.Count(); - } - - // make numpy array, shape is count, possibly with extra dim for step added - Dims shapePy; - shapePy.reserve((stepCount > 0 ? 1 : 0) + count.size()); - if (stepCount > 0) - { - shapePy.emplace_back(stepCount); - } - std::copy(count.begin(), count.end(), std::back_inserter(shapePy)); - - pybind11::array_t pyArray(shapePy); - - // set selection if requested - if (!start.empty() && !count.empty()) - { - variable.SetSelection(Box(std::move(start), std::move(count))); - } - - // set step selection if requested - if (stepCount > 0) - { - variable.SetStepSelection({stepStart, stepCount}); - } - - if (!m_Stream->m_Engine) - { - throw std::logic_error("no engine available in DoRead()"); - } - m_Stream->m_Engine->Get(variable, pyArray.mutable_data(), Mode::Sync); - - return std::move(pyArray); -} - -} // end namespace py11 -} // end namespace adios2 - -#endif diff --git a/bindings/Python/py11FileMPI.cpp b/bindings/Python/py11FileMPI.cpp deleted file mode 100644 index e38383203d..0000000000 --- a/bindings/Python/py11FileMPI.cpp +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * py11FileMPI.cpp - */ - -#include "py11File.h" - -#include "adios2/helper/adiosCommMPI.h" - -namespace adios2 -{ -namespace py11 -{ - -File::File(const std::string &name, const std::string mode, MPI_Comm comm, - const std::string engineType) -: m_Name(name), m_Mode(mode), - m_Stream(std::make_shared(name, ToMode(mode), helper::CommDupMPI(comm), engineType, - "Python")) -{ -} - -File::File(const std::string &name, const std::string mode, MPI_Comm comm, - const std::string &configFile, const std::string ioInConfigFile) -: m_Name(name), m_Mode(mode), - m_Stream(std::make_shared(name, ToMode(mode), helper::CommDupMPI(comm), configFile, - ioInConfigFile, "Python")) -{ -} - -} // end namespace py11 -} // end namespace adios2 diff --git a/bindings/Python/py11IO.cpp b/bindings/Python/py11IO.cpp index 2d57abe554..b5931219d5 100644 --- a/bindings/Python/py11IO.cpp +++ b/bindings/Python/py11IO.cpp @@ -170,12 +170,13 @@ Attribute IO::DefineAttribute(const std::string &name, const std::vectorInquireAttributeType(name)); + const DataType type(m_IO->InquireAttributeType(name, variableName, separator)); if (type == DataType::None) { @@ -183,7 +184,7 @@ Attribute IO::InquireAttribute(const std::string &name) #define declare_template_instantiation(T) \ else if (type == helper::GetDataType()) \ { \ - attribute = m_IO->InquireAttribute(name); \ + attribute = m_IO->InquireAttribute(name, variableName, separator); \ } ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_1ARG(declare_template_instantiation) #undef declare_template_instantiation diff --git a/bindings/Python/py11IO.h b/bindings/Python/py11IO.h index 35ac6e269d..9308af270d 100644 --- a/bindings/Python/py11IO.h +++ b/bindings/Python/py11IO.h @@ -71,7 +71,8 @@ class IO const std::string &variableName = "", const std::string separator = "/"); - Attribute InquireAttribute(const std::string &name); + Attribute InquireAttribute(const std::string &name, const std::string &variableName = "", + const std::string separator = "/"); bool RemoveVariable(const std::string &name); diff --git a/bindings/Python/py11Variable.cpp b/bindings/Python/py11Variable.cpp index 2a57ef6ba2..77cf775b32 100644 --- a/bindings/Python/py11Variable.cpp +++ b/bindings/Python/py11Variable.cpp @@ -80,6 +80,12 @@ size_t Variable::AddOperation(const Operator op, const Params ¶meters) return m_VariableBase->AddOperation(op.m_Type, params); } +size_t Variable::AddOperation(const std::string &op, const Params ¶meters) +{ + helper::CheckForNullptr(m_VariableBase, "in call to Variable::AddOperation"); + return m_VariableBase->AddOperation(op, parameters); +} + std::vector Variable::Operations() const { helper::CheckForNullptr(m_VariableBase, "in call to Variable::Operations"); @@ -93,6 +99,8 @@ std::vector Variable::Operations() const return operations; } +void Variable::RemoveOperations() { m_VariableBase->RemoveOperations(); } + std::string Variable::Name() const { helper::CheckForNullptr(m_VariableBase, "in call to Variable::Name"); @@ -189,6 +197,12 @@ size_t Variable::BlockID() const return m_VariableBase->m_BlockID; } +size_t Variable::SingleValue() const +{ + helper::CheckForNullptr(m_VariableBase, "in call to Variable::SingleValue"); + return m_VariableBase->m_SingleValue; +} + // size_t Variable::AddOperation(const Operator op, const Params ¶meters) {} // std::vector Variable::Operations() const {} diff --git a/bindings/Python/py11Variable.h b/bindings/Python/py11Variable.h index d04d1fc898..b9ab9c1511 100644 --- a/bindings/Python/py11Variable.h +++ b/bindings/Python/py11Variable.h @@ -95,6 +95,8 @@ class Variable size_t BlockID() const; + size_t SingleValue() const; + /** * EXPERIMENTAL: Adds operation and parameters to current Variable object * @param op operator to be added @@ -104,12 +106,27 @@ class Variable */ size_t AddOperation(const Operator op, const Params ¶meters = Params()); + /** + * EXPERIMENTAL: Adds operation and parameters to current Variable object + * @param op operator to be added + * @param parameters key/value settings particular to the Variable, not to + * be confused by op own parameters + * @return operation index handler in Operations() + */ + size_t AddOperation(const std::string &op, const Params ¶meters = Params()); + /** * EXPERIMENTAL: inspects current operators added with AddOperator * @return vector of Variable::OperatorInfo */ std::vector Operations() const; + /** + * Removes all current Operations associated with AddOperation. + * Provides the posibility to apply or not operators on a step basis. + */ + void RemoveOperations(); + /** Contains sub-block information for a particular Variable */ struct Info { diff --git a/bindings/Python/py11glue.cpp b/bindings/Python/py11glue.cpp index 128c52d941..06c9217d60 100644 --- a/bindings/Python/py11glue.cpp +++ b/bindings/Python/py11glue.cpp @@ -23,7 +23,6 @@ #include "py11ADIOS.h" #include "py11Attribute.h" #include "py11Engine.h" -#include "py11File.h" #include "py11IO.h" #include "py11Operator.h" #include "py11Query.h" @@ -78,34 +77,6 @@ struct type_caster #endif -#if ADIOS2_USE_MPI - -adios2::py11::File OpenMPI(const std::string &name, const std::string mode, - adios2::py11::MPI4PY_Comm comm, const std::string enginetype) -{ - return adios2::py11::File(name, mode, comm, enginetype); -} - -adios2::py11::File OpenConfigMPI(const std::string &name, const std::string mode, - adios2::py11::MPI4PY_Comm comm, const std::string &configfile, - const std::string ioinconfigfile) -{ - return adios2::py11::File(name, mode, comm, configfile, ioinconfigfile); -} - -#endif -adios2::py11::File Open(const std::string &name, const std::string mode, - const std::string enginetype) -{ - return adios2::py11::File(name, mode, enginetype); -} - -adios2::py11::File OpenConfig(const std::string &name, const std::string mode, - const std::string configfile, const std::string ioinconfigfile) -{ - return adios2::py11::File(name, mode, configfile, ioinconfigfile); -} - PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) { m.attr("ConstantDims") = true; @@ -147,68 +118,6 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) .value("OtherError", adios2::StepStatus::OtherError) .export_values(); -#if ADIOS2_USE_MPI - m.def("open", &OpenMPI, pybind11::arg("name"), pybind11::arg("mode"), pybind11::arg("comm"), - pybind11::arg("engine_type") = "BPFile", R"md( - Simple API MPI open, based on python IO. - Allows for passing parameters in source code. - - Parameters - name - stream name - mode - "w" : write, - "r" : read, - "a" : append (append not yet supported) - - comm (mpi4py) - MPI communicator - - engine_type - adios2 engine type, default=BPFile - - Returns - file (adios2 stream) - handler to adios File for the simple Python API - )md"); - - m.def("open", &OpenConfigMPI, pybind11::arg("name"), pybind11::arg("mode"), - pybind11::arg("comm"), pybind11::arg("config_file"), pybind11::arg("io_in_config_file"), - R"md( - Simple API MPI open, based on python IO. - Allows for passing a runtime configuration file in xml format and the - name of the io element related to the returning File. - - Parameters - name - stream name - mode - "w" : write, - "r" : read, - "a" : append (append not yet supported) - - comm (mpi4py) - MPI communicator - - config_file - adios2 runtime configuration file name, in xml format - - io_in_config_file - io element in configfile related to returning File - - Returns - file (adios2 stream) - handler to adios File for the simple Python API - )md"); - -#endif - m.def("open", &Open, "High-level API, file object open", pybind11::arg("name"), - pybind11::arg("mode"), pybind11::arg("engine_type") = "BPFile"); - - m.def("open", &OpenConfig, "High-level API, file object open with a runtime config file", - pybind11::arg("name"), pybind11::arg("mode"), pybind11::arg("config_file"), - pybind11::arg("io_in_config_file")); - pybind11::class_(m, "ADIOS") // Python 2 .def("__nonzero__", @@ -297,8 +206,12 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) .def("InquireVariable", &adios2::py11::IO::InquireVariable, pybind11::return_value_policy::move) - .def("InquireAttribute", &adios2::py11::IO::InquireAttribute, - pybind11::return_value_policy::move) + .def("InquireAttribute", + (adios2::py11::Attribute(adios2::py11::IO::*)(const std::string &, const std::string &, + const std::string)) & + adios2::py11::IO::InquireAttribute, + pybind11::arg("name"), pybind11::arg("variable_name") = "", + pybind11::arg("separator") = "/", pybind11::return_value_policy::move) .def("DefineAttribute", (adios2::py11::Attribute(adios2::py11::IO::*)( @@ -360,6 +273,23 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) .def("GetResult", &adios2::py11::Query::GetResult) .def("GetBlockIDs", &adios2::py11::Query::GetBlockIDs); + pybind11::class_(m, "Operator") + // Python 2 + .def("__nonzero__", + [](const adios2::py11::Operator &op) { + const bool opBool = op ? true : false; + return opBool; + }) + // Python 3 + .def("__bool__", + [](const adios2::py11::Operator &op) { + const bool opBool = op ? true : false; + return opBool; + }) + .def("Type", &adios2::py11::Operator::Type) + .def("SetParameter", &adios2::py11::Operator::SetParameter) + .def("Parameters", &adios2::py11::Operator::Parameters); + pybind11::class_(m, "Variable") // Python 2 .def("__nonzero__", @@ -389,8 +319,15 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) .def("Steps", &adios2::py11::Variable::Steps) .def("StepsStart", &adios2::py11::Variable::StepsStart) .def("BlockID", &adios2::py11::Variable::BlockID) - .def("AddOperation", &adios2::py11::Variable::AddOperation) - .def("Operations", &adios2::py11::Variable::Operations); + .def("SingleValue", &adios2::py11::Variable::SingleValue) + .def("AddOperation", (size_t(adios2::py11::Variable::*)(const adios2::py11::Operator, + const adios2::Params &)) & + adios2::py11::Variable::AddOperation) + .def("AddOperation", + (size_t(adios2::py11::Variable::*)(const std::string &, const adios2::Params &)) & + adios2::py11::Variable::AddOperation) + .def("Operations", &adios2::py11::Variable::Operations) + .def("RemoveOperations", &adios2::py11::Variable::RemoveOperations); pybind11::class_(m, "Attribute") // Python 2 @@ -481,543 +418,4 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) .def("LockReaderSelections", &adios2::py11::Engine::LockReaderSelections) .def("BlocksInfo", &adios2::py11::Engine::BlocksInfo); - - pybind11::class_(m, "Operator") - // Python 2 - .def("__nonzero__", - [](const adios2::py11::Operator &op) { - const bool opBool = op ? true : false; - return opBool; - }) - // Python 3 - .def("__bool__", - [](const adios2::py11::Operator &op) { - const bool opBool = op ? true : false; - return opBool; - }) - .def("Type", &adios2::py11::Operator::Type) - .def("SetParameter", &adios2::py11::Operator::SetParameter) - .def("Parameters", &adios2::py11::Operator::Parameters); - - pybind11::class_(m, "File") - .def("__repr__", - [](const adios2::py11::File &stream) { - return ""; - }) - - // enter and exit are defined for the with-as operator in Python - .def("__enter__", [](const adios2::py11::File &stream) { return stream; }) - .def("__exit__", [](adios2::py11::File &stream, pybind11::args) { stream.Close(); }) - .def( - "__iter__", [](adios2::py11::File &stream) { return stream; }, - pybind11::keep_alive<0, 1>()) - .def("__next__", - [](adios2::py11::File &stream) { - if (!stream.GetStep()) - { - throw pybind11::stop_iteration(); - } - return stream; - }) - - .def("set_parameter", &adios2::py11::File::SetParameter, pybind11::arg("key"), - pybind11::arg("value"), R"md( - Sets a single parameter. Overwrites value if key exists. - - Parameters - key - input parameter key - - value - parameter value - )md") - - .def("set_parameters", &adios2::py11::File::SetParameters, pybind11::arg("parameters"), - R"md( - Sets parameters using a dictionary. - Removes any previous parameter. - - Parameters - parameters - input key/value parameters - - value - parameter value - )md") - - .def("add_transport", &adios2::py11::File::AddTransport, - pybind11::return_value_policy::move, pybind11::arg("type"), - pybind11::arg("parameters") = adios2::Params(), R"md( - Adds a transport and its parameters to current IO. Must be - supported by current engine type. - - Parameters - type - must be a supported transport type for current engine. - - parameters - acceptable parameters for a particular transport - CAN'T use the keywords "Transport" or "transport" in key - - Returns - transport_index - handler to added transport - )md") - - .def("available_variables", &adios2::py11::File::AvailableVariables, - pybind11::return_value_policy::move, - pybind11::arg("keys") = std::vector(), R"md( - Returns a 2-level dictionary with variable information. - Read mode only. - - Parameters - keys - list of variable information keys to be extracted (case insensitive) - keys=['AvailableStepsCount','Type','Max','Min','SingleValue','Shape'] - keys=['Name'] returns only the variable names as 1st-level keys - leave empty to return all possible keys - - Returns - variables dictionary - key - variable name - value - variable information dictionary - )md") - - .def("available_attributes", &adios2::py11::File::AvailableAttributes, - pybind11::return_value_policy::move, R"md( - Returns a 2-level dictionary with attribute information. - Read mode only. - - Returns - attributes dictionary - key - attribute name - value - attribute information dictionary - )md") - - .def("write", - (void(adios2::py11::File::*)(const std::string &, const pybind11::array &, - const adios2::Dims &, const adios2::Dims &, - const adios2::Dims &, const bool)) & - adios2::py11::File::Write, - pybind11::arg("name"), pybind11::arg("array"), pybind11::arg("shape") = adios2::Dims(), - pybind11::arg("start") = adios2::Dims(), pybind11::arg("count") = adios2::Dims(), - pybind11::arg("end_step") = false, - R"md( - writes a self-describing array (numpy) variable - - Parameters - name - variable name - - array - variable data values - - shape - variable global MPI dimensions. - Pass empty numpy array for local variables. - - start - variable offset for current MPI rank. - Pass empty numpy array for local variables. - - count - variable dimension for current MPI rank. - Pass a numpy array for local variables. - - end_step - end current step, begin next step and flush (default = false). - )md") - - .def("write", - (void(adios2::py11::File::*)( - const std::string &, const pybind11::array &, const adios2::Dims &, - const adios2::Dims &, const adios2::Dims &, const adios2::vParams &, const bool)) & - adios2::py11::File::Write, - pybind11::arg("name"), pybind11::arg("array"), pybind11::arg("shape"), - pybind11::arg("start"), pybind11::arg("count"), pybind11::arg("operations"), - pybind11::arg("end_step") = false, - R"md( - writes a self-describing array (numpy) variable with operations - e.g. compression: 'zfp', 'mgard', 'sz' - - Parameters - name - variable name - - array - variable data values - - shape - variable global MPI dimensions. - Pass empty numpy array for local variables. - - start - variable offset for current MPI rank. - Pass empty numpy array for local variables. - - count - variable dimension for current MPI rank. - Pass a numpy array for local variables. - - end_step - end current step, begin next step and flush (default = false). - )md") - - .def("write", - (void(adios2::py11::File::*)(const std::string &, const pybind11::array &, const bool, - const bool)) & - adios2::py11::File::Write, - pybind11::arg("name"), pybind11::arg("array"), pybind11::arg("local_value") = false, - pybind11::arg("end_step") = false, R"md( - writes a self-describing single value array (numpy) variable - - Parameters - name - variable name - - array - variable data single value - - local_value - true: local value, false: global value - - end_step - end current step, begin next step and flush - (default = false). - )md") - - .def("write", - (void(adios2::py11::File::*)(const std::string &, const std::string &, const bool, - const bool)) & - adios2::py11::File::Write, - pybind11::arg("name"), pybind11::arg("string"), pybind11::arg("local_value") = false, - pybind11::arg("end_step") = false, R"md( - writes a self-describing single value string variable - - Parameters - name - variable name - - string - variable data single value - - local_value - true: local value, false: global value - - end_step - end current step, begin next step and flush - (default = false). - )md") - - .def("write_attribute", - (void(adios2::py11::File::*)(const std::string &, const pybind11::array &, - const std::string &, const std::string, const bool)) & - adios2::py11::File::WriteAttribute, - pybind11::arg("name"), pybind11::arg("array"), pybind11::arg("variable_name") = "", - pybind11::arg("separator") = "/", pybind11::arg("end_step") = false, R"md( - writes a self-describing single value array (numpy) variable - - Parameters - name - attribute name - - array - attribute numpy array data - - variable_name - if attribute is associated with a variable - - separator - concatenation string between variable_name and attribute - e.g. variable_name + separator + name ("var/attr") - Not used if variable_name is empty - - end_step - end current step, begin next step and flush - (default = false). - )md") - - .def("write_attribute", - (void(adios2::py11::File::*)(const std::string &, const std::string &, - const std::string &, const std::string, const bool)) & - adios2::py11::File::WriteAttribute, - pybind11::arg("name"), pybind11::arg("string_value"), - pybind11::arg("variable_name") = "", pybind11::arg("separator") = "/", - pybind11::arg("end_step") = false, R"md( - writes a self-describing single value array (numpy) variable - - Parameters - name - attribute name - - string_value - attribute single string - - variable_name - if attribute is associated with a variable - - separator - concatenation string between variable_name and attribute - e.g. variable_name + separator + name ("var/attr") - Not used if variable_name is empty - - end_step - end current step, begin next step and flush - (default = false). - )md") - - .def("write_attribute", - (void(adios2::py11::File::*)(const std::string &, const std::vector &, - const std::string &, const std::string, const bool)) & - adios2::py11::File::WriteAttribute, - pybind11::arg("name"), pybind11::arg("string_array"), - pybind11::arg("variable_name") = "", pybind11::arg("separator") = "/", - pybind11::arg("end_step") = false, R"md( - writes a self-describing single value array (numpy) variable - - Parameters - name - attribute name - - string_array - attribute string array - - variable_name - if attribute is associated with a variable - - separator - concatenation string between variable_name and attribute - e.g. variable_name + separator + name ("var/attr") - Not used if variable_name is empty - - end_step - end current step, begin next step and flush - (default = false). - )md") - - .def("read_string", - (std::vector(adios2::py11::File::*)(const std::string &, const size_t)) & - adios2::py11::File::ReadString, - pybind11::return_value_policy::take_ownership, pybind11::arg("name"), - pybind11::arg("block_id") = 0, - R"md( - Reads string value for current step - (use for streaming mode step by step) - - Parameters - name - string variable name - - block_id - required for local variables - - Returns - - list - data string values. - For global values: returns 1 element - For local values: returns n-block elements - - )md") - - .def("read_string", - (std::vector(adios2::py11::File::*)(const std::string &, const size_t, - const size_t, const size_t)) & - adios2::py11::File::ReadString, - pybind11::return_value_policy::take_ownership, pybind11::arg("name"), - pybind11::arg("step_start"), pybind11::arg("step_count"), - pybind11::arg("block_id") = 0, - R"md( - Reads string value for a certain step - (random access mode) - - Parameters - name - string variable name - - step_start - variable step start - - step_count - variable number of steps to read from step_start - - block_id - required for local variables - - Returns - string - data string values for a certain step range. - )md") - - .def("read", - (pybind11::array(adios2::py11::File::*)(const std::string &, const size_t)) & - adios2::py11::File::Read, - pybind11::return_value_policy::take_ownership, pybind11::arg("name"), - pybind11::arg("block_id") = 0, - R"md( - Reads entire variable for current step - (streaming mode step by step) - - Parameters - name - variable name - - block_id - required for local array variables - - Returns - array - values of variable name for current step. - Single values will have a shape={1} numpy array - )md") - - .def("read", - (pybind11::array(adios2::py11::File::*)(const std::string &, const adios2::Dims &, - const adios2::Dims &, const size_t)) & - adios2::py11::File::Read, - pybind11::return_value_policy::take_ownership, pybind11::arg("name"), - pybind11::arg("start") = adios2::Dims(), pybind11::arg("count") = adios2::Dims(), - pybind11::arg("block_id") = 0, - R"md( - Reads a selection piece in dimension for current step - (streaming mode step by step) - - Parameters - name - variable name - - start - variable local offset selection (defaults to (0, 0, ...) - - count - variable local dimension selection from start - defaults to whole array for GlobalArrays, or selected Block size - for LocalArrays - - block_id - required for local array variables - - Returns - array - values of variable name for current step - empty if exception is thrown - )md") - - .def("read", - (pybind11::array(adios2::py11::File::*)(const std::string &, const adios2::Dims &, - const adios2::Dims &, const size_t, - const size_t, const size_t)) & - adios2::py11::File::Read, - pybind11::return_value_policy::take_ownership, pybind11::arg("name"), - pybind11::arg("start"), pybind11::arg("count"), pybind11::arg("step_start"), - pybind11::arg("step_count"), pybind11::arg("block_id") = 0, R"md( - Random access read allowed to select steps, - only valid with File Engines - - Parameters - name - variable to be read - - start - variable offset dimensions - - count - variable local dimensions from offset - - step_start - variable step start - - step_count - variable number of steps to read from step_start - - block_id - required for local array variables - - Returns - array - resulting array from selection - )md") - - .def("read_attribute", - (pybind11::array(adios2::py11::File::*)(const std::string &, const std::string &, - const std::string)) & - adios2::py11::File::ReadAttribute, - pybind11::return_value_policy::take_ownership, pybind11::arg("name"), - pybind11::arg("variable_name") = "", pybind11::arg("separator") = "/", R"md( - Reads a numpy based attribute - - Parameters - name - attribute name - - variable_name - if attribute is associated with a variable - - separator - concatenation string between variable_name and attribute - e.g. variable_name + separator + name (var/attr) - Not used if variable_name is empty - - Returns - array - resulting array attribute data - )md") - - .def("read_attribute_string", - (std::vector(adios2::py11::File::*)( - const std::string &, const std::string &, const std::string)) & - adios2::py11::File::ReadAttributeString, - pybind11::return_value_policy::take_ownership, pybind11::arg("name"), - pybind11::arg("variable_name") = "", pybind11::arg("separator") = "/", R"md( - Read a string attribute - - Parameters - name - attribute name - - variable_name - if attribute is associated with a variable - - separator - concatenation string between variable_name and attribute - e.g. variable_name + separator + name (var/attr) - Not used if variable_name is empty - - Returns - list - resulting string list attribute data)md") - - .def("end_step", &adios2::py11::File::EndStep, R"md( - Write mode: advances to the next step. Convenient when declaring - variable attributes as advancing to the next step is not attached - to any variable. - - Read mode: in streaming mode releases the current step (no effect - in file based engines) - )md") - - .def("close", &adios2::py11::File::Close, R"md( - Closes file, thus becoming unreachable. - Not required if using open in a with-as statement. - Required in all other cases per-open to avoid resource leaks. - )md") - - .def("current_step", &adios2::py11::File::CurrentStep, R"md( - Inspect current step when using for-in loops, read mode only - - Returns - current step - )md") - - .def("steps", &adios2::py11::File::Steps, R"md( - Inspect available number of steps, for file engines, read mode only - - Returns - steps - )md"); } diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index dccfa38b33..6e5209a217 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -400,15 +400,15 @@ if(NOT SHARED_LIBS_SUPPORTED) endif() if(ADIOS2_USE_PIP) - find_package(Python 3 REQUIRED COMPONENTS Interpreter Development.Module NumPy) + find_package(Python 3 REQUIRED COMPONENTS Interpreter Development.Module) set(ADIOS2_HAVE_PIP TRUE) elseif(ADIOS2_USE_Python STREQUAL AUTO) - find_package(Python 3 COMPONENTS Interpreter Development NumPy) + find_package(Python 3 COMPONENTS Interpreter Development) if(Python_FOUND AND ADIOS2_HAVE_MPI) find_package(PythonModule COMPONENTS mpi4py mpi4py/mpi4py.h) endif() elseif(ADIOS2_USE_Python) - find_package(Python 3 REQUIRED COMPONENTS Interpreter Development NumPy) + find_package(Python 3 REQUIRED COMPONENTS Interpreter Development) if(ADIOS2_HAVE_MPI) find_package(PythonModule REQUIRED COMPONENTS mpi4py mpi4py/mpi4py.h) endif() diff --git a/pyproject.toml b/pyproject.toml index 5b39654fb7..57eefd83fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,12 @@ authors = [ description = "The Adaptable Input Output System version 2" readme = "ReadMe.md" requires-python = ">=3.8" +keywords = [ + "Python", + "Web", + "Application", + "Framework", +] classifiers = [ "Programming Language :: C++", "License :: OSI Approved :: Apache Software License", @@ -29,6 +35,12 @@ dependencies = [ "numpy", ] +[project.optional-dependencies] +dev = [ + 'black', + "pip>=21.3", +] + [project.urls] Homepage = "https://github.com/ornladios/adios2" Documentation = "https://adios2.readthedocs.io/" @@ -63,3 +75,8 @@ ADIOS2_USE_ZeroMQ = "OFF" ADIOS2_USE_ZFP = "OFF" BUILD_TESTING = "OFF" ADIOS2_INSTALL_GENERATE_CONFIG = "OFF" + +[tool.black] +line-length = 99 +target-version = ['py38', 'py39', 'py310'] +include = 'python/adios2/.*.py|testing/adios2/python/.*.py' diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt new file mode 100644 index 0000000000..0564486c09 --- /dev/null +++ b/python/CMakeLists.txt @@ -0,0 +1,18 @@ +#------------------------------------------------------------------------------# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +#------------------------------------------------------------------------------# + +add_custom_target(python_api ALL COMMAND ${CMAKE_COMMAND} -E copy_directory + ${CMAKE_CURRENT_SOURCE_DIR}/adios2 + ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2) + +set(install_location ${CMAKE_INSTALL_PYTHONDIR}) +if (ADIOS2_USE_PIP) + set(install_location ${CMAKE_INSTALL_LIBDIR}) +endif() + +install(DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/ + DESTINATION ${install_location}/adios2/ + COMPONENT adios2_python-python +) diff --git a/python/adios2/__init__.py b/python/adios2/__init__.py new file mode 100644 index 0000000000..ed1e42d313 --- /dev/null +++ b/python/adios2/__init__.py @@ -0,0 +1,14 @@ +#!/usr/bin/python3 + +import adios2.bindings + +from adios2.adios import * +from adios2.attribute import * +from adios2.engine import * +from adios2.io import * +from adios2.operator import * +from adios2.stream import * +from adios2.variable import * +from adios2.bindings import LocalValueDim + +__version__ = adios2.bindings.__version__ diff --git a/python/adios2/adios.py b/python/adios2/adios.py new file mode 100644 index 0000000000..0476f2fe04 --- /dev/null +++ b/python/adios2/adios.py @@ -0,0 +1,40 @@ +from adios2.io import IO +from adios2.operator import Operator + +import adios2.bindings as bindings + + +class ADIOS: + def __init__(self, comm=None): + if comm: + self._impl = bindings.ADIOS(comm) + else: + self._impl = bindings.ADIOS() + + self._operators = {} + self._ios = {} + + def DeclareIO(self, name): + self._ios[name] = IO(self._impl, name) + return self._ios[name] + + def AtIO(self, name): + return self._ios[name] + + def RemoveIO(self, name): + del self._ios[name] + self._impl.RemoveIO(name) + + def RemoveAllIOs(self): + self._ios = {} + self._impl.RemoveAllIOs() + + def DefineOperator(self, name, kind, parameters={}): + self._operators[name] = Operator(self._impl, name, kind, parameters) + return self._operators[name] + + def InquireOperator(self, name): + return self._operators[name] + + def FlushAll(self): + self._impl.FlushAll() diff --git a/python/adios2/attribute.py b/python/adios2/attribute.py new file mode 100644 index 0000000000..2772c65fe6 --- /dev/null +++ b/python/adios2/attribute.py @@ -0,0 +1,23 @@ +import adios2.bindings + + +class Attribute: + def __init__(self, io, name, *args, **kwargs): + self._impl = io.DefineAttribute(name, *args, **kwargs) + + def __eq__(self, other): + if isinstance(other, Attribute): + return self.Name() == other.Name() + return False + + def Name(self): + return self._impl.Name() + + def Type(self): + return self._impl.Type() + + def Data(self): + return self._impl.Data() + + def DataString(self): + return self._impl.DataString() diff --git a/python/adios2/engine.py b/python/adios2/engine.py new file mode 100644 index 0000000000..850cdcb1d4 --- /dev/null +++ b/python/adios2/engine.py @@ -0,0 +1,68 @@ +import numpy as np +import adios2.bindings as bindings + +from adios2.attribute import Attribute +from adios2.variable import Variable + + +class Engine: + def __init__(self, io, name, mode, comm=None): + if comm: + self._impl = io.Open(name, mode, comm) + else: + self._impl = io.Open(name, mode) + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.Close() + + def Close(self, transportIndex=-1): + self._impl.Close(transportIndex) + + def Steps(self): + return self._impl.Steps() + + def CurrentStep(self): + return self._impl.CurrentStep() + + def BeginStep(self, *args, **kwargs): + return self._impl.BeginStep(*args, **kwargs) + + def EndStep(self): + self._impl.EndStep() + + def BlocksInfo(self, name, step): + return self._impl.BlocksInfo(name, step) + + def Put(self, variable, content, mode=bindings.Mode.Deferred): + if isinstance(content, np.ndarray): + self._impl.Put(variable._impl, content, mode) + else: + self._impl.Put(variable._impl, content) + + def PerformPuts(self): + self._impl.PerformPuts() + + def PerformDataWrite(self): + self._impl.PerformDataWrite() + + def Get(self, variable, content=None, mode=bindings.Mode.Sync): + if isinstance(content, np.ndarray): + self._impl.Get(variable._impl, content, mode) + return None + else: + return self._impl.Get(variable._impl, mode) + + def PerformGets(self): + self._impl.PerformGets() + + def LockReaderSelections(self): + self._impl.LockReaderSelections() + + def LockWriterDefinitions(self): + self._impl.LockWriterDefinitions() + + def Flush(self, transportIndex=-1): + self._impl.Flush(transportIndex) diff --git a/python/adios2/file.py b/python/adios2/file.py new file mode 100644 index 0000000000..ea5182ae29 --- /dev/null +++ b/python/adios2/file.py @@ -0,0 +1,199 @@ +from adios2.adios import ADIOS +import adios2.bindings as bindings + +import numpy as np + + +def typeAdiosToNumPy(name): + return { + "int8_t": np.int8, + "uint8_t": np.uint8, + "int16_t": np.int16, + "uint16_t": np.uint16, + "int32_t": np.int32, + "uint32_t": np.uint32, + "int64_t": np.int64, + "uint64_t": np.uint64, + "float": np.float32, + "double": np.float64, + } + + +class FileReader: + def __init__(self, path): + self._io_name = f"FileReader:{path}:engine_type:file" + self._adios = ADIOS() + self._io = self._adios.DeclareIO(self._io_name) + self._engine = self._io.Open(path, bindings.Mode.ReadRandomAccess) + self.index = 0 + self.max_steps = 0 + + def __repr__(self): + return f"" + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + def __iter__(self): + return self + + def __next__(self): + if self.index > 0: + self.end_step() + if self.index == self.max_steps: + self.index = 0 + self.max_steps = 0 + raise StopIteration + + self.index += 1 + self.begin_step() + return self + + def set_parameters(self, **kwargs): + """ + Sets parameters using a dictionary. + Removes any previous parameter. + + Parameters + parameters + input key/value parameters + + value + parameter value + """ + self._io.SetParameters(**kwargs) + + def set_transport(self, transport, parameters={}): + """ + + Adds a transport and its parameters to current IO. Must be + supported by current engine type. + + Parameters + type + must be a supported transport type for current engine. + + parameters + acceptable parameters for a particular transport + CAN'T use the keywords "Transport" or "transport" in key + + Returns + transport_index + handler to added transport + """ + self._io.AddTransport(transport, parameters) + + def available_variables(self): + """ + + Returns a 2-level dictionary with variable information. + Read mode only. + + Parameters + keys + list of variable information keys to be extracted (case insensitive) + keys=['AvailableStepsCount','Type','Max','Min','SingleValue','Shape'] + keys=['Name'] returns only the variable names as 1st-level keys + leave empty to return all possible keys + + Returns + variables dictionary + key + variable name + value + variable information dictionary + """ + return self._io.AvailableVariables() + + def available_attributes(self): + """ + Returns a 2-level dictionary with attribute information. + Read mode only. + + Returns + attributes dictionary + key + attribute name + value + attribute information dictionary + """ + return self._io.AvailableAttributes() + + def read(self, name): + """ + Random access read allowed to select steps, + only valid with File Engines + + Parameters + name + variable to be read + + start + variable offset dimensions + + count + variable local dimensions from offset + + step_start + variable step start + + step_count + variable number of steps to read from step_start + + block_id + required for local array variables + + Returns + array + resulting array from selection + """ + variable = self._io.InquireVariable(name) + if not variable: + raise ValueError() + + if variable.Type() == "string" and variable.SingleValue() is True: + return self._engine.Get(variable) + + dtype = typeAdiosToNumPy(variable.Type()) + + output = np.zeros((variable.SelectionSize(),), dtype=dtype) + self._engine.Get(variable, output) + return output + + def read_attribute(name, variable_name, separator="/"): + """ + Reads a numpy based attribute + + Parameters + name + attribute name + + variable_name + if attribute is associated with a variable + + separator + concatenation string between variable_name and attribute + e.g. variable_name + separator + name (var/attr) + Not used if variable_name is empty + + Returns + array + resulting array attribute data + """ + pass + + def close(self): + """ + Closes file, thus becoming unreachable. + Not required if using open in a with-as statement. + Required in all other cases per-open to avoid resource leaks. + """ + self._engine.Close() + self._engine = None + self._io.FlushAll() + self._io = None + self._adios.FlushAll() + self._adios = None diff --git a/python/adios2/file_reader.py b/python/adios2/file_reader.py new file mode 100644 index 0000000000..e99095c218 --- /dev/null +++ b/python/adios2/file_reader.py @@ -0,0 +1,185 @@ +from adios2.adios import ADIOS +from adios2.Stream import typeAdiosToNumPy +import adios2.bindings as bindings + +import numpy as np + + +class FileReader: + def __init__(self, path): + self._io_name = f"file:{path}:engine_type:file" + self._adios = ADIOS() + self._io = self._adios.DeclareIO(self._io_name) + self._engine = self._io.Open(path, bindings.Mode.ReadRandomAccess) + self.index = 0 + self.max_steps = 0 + + def __repr__(self): + return f"" + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + def __iter__(self): + return self + + def __next__(self): + if self.index > 0: + self.end_step() + if self.index == self.max_steps: + self.index = 0 + self.max_steps = 0 + raise StopIteration + + self.index += 1 + self.begin_step() + return self + + def set_parameters(self, **kwargs): + """ + Sets parameters using a dictionary. + Removes any previous parameter. + + Parameters + parameters + input key/value parameters + + value + parameter value + """ + self._io.SetParameters(**kwargs) + + def set_transport(self, transport, parameters={}): + """ + + Adds a transport and its parameters to current IO. Must be + supported by current engine type. + + Parameters + type + must be a supported transport type for current engine. + + parameters + acceptable parameters for a particular transport + CAN'T use the keywords "Transport" or "transport" in key + + Returns + transport_index + handler to added transport + """ + self._io.AddTransport(transport, parameters) + + def available_variables(self): + """ + + Returns a 2-level dictionary with variable information. + Read mode only. + + Parameters + keys + list of variable information keys to be extracted (case insensitive) + keys=['AvailableStepsCount','Type','Max','Min','SingleValue','Shape'] + keys=['Name'] returns only the variable names as 1st-level keys + leave empty to return all possible keys + + Returns + variables dictionary + key + variable name + value + variable information dictionary + """ + return self._io.AvailableVariables() + + def available_attributes(self): + """ + Returns a 2-level dictionary with attribute information. + Read mode only. + + Returns + attributes dictionary + key + attribute name + value + attribute information dictionary + """ + return self._io.AvailableAttributes() + + def read(self, name): + """ + Random access read allowed to select steps, + only valid with File Engines + + Parameters + name + variable to be read + + start + variable offset dimensions + + count + variable local dimensions from offset + + step_start + variable step start + + step_count + variable number of steps to read from step_start + + block_id + required for local array variables + + Returns + array + resulting array from selection + """ + variable = self._io.InquireVariable(name) + if not variable: + raise ValueError() + + if variable.Type() == "string" and variable.SingleValue() is True: + return self._engine.Get(variable) + + dtype = typeAdiosToNumPy(variable.Type()) + + output = np.zeros((variable.SelectionSize(),), dtype=dtype) + self._engine.Get(variable, output) + return output + + def read_attribute(name, variable_name, separator="/"): + """ + Reads a numpy based attribute + + Parameters + name + attribute name + + variable_name + if attribute is associated with a variable + + separator + concatenation string between variable_name and attribute + e.g. variable_name + separator + name (var/attr) + Not used if variable_name is empty + + Returns + array + resulting array attribute data + """ + pass + + def close(self): + """ + Closes file, thus becoming unreachable. + Not required if using open in a with-as statement. + Required in all other cases per-open to avoid resource leaks. + """ + self._engine.Close() + self._engine = None + self._io.FlushAll() + self._io = None + self._adios.FlushAll() + self._adios = None diff --git a/python/adios2/io.py b/python/adios2/io.py new file mode 100644 index 0000000000..b3f575598b --- /dev/null +++ b/python/adios2/io.py @@ -0,0 +1,103 @@ +import numpy as np +import adios2.bindings as bindings +from adios2.attribute import Attribute +from adios2.variable import Variable +from adios2.engine import Engine + + +class IO: + def __init__(self, adios, name): + self._impl = adios.DeclareIO(name) + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.FlushAll() + + def DefineAttribute( + self, + name, + content=None, + variable_name="", + separator="/", + ): + return Attribute(self._impl, name, content, variable_name, separator) + + def InquireAttribute(self, name, variable_name="", separator="/"): + attr = None + attr_impl = self._impl.InquireAttribute(name, variable_name, separator) + if attr_impl: + attr = Attribute.__new__(Attribute) + attr._impl = attr_impl + return attr + + def AvailableAttributes(self): + attributes = {} + for name, attr in self._impl.AvailableAttributes(): + attributes[name] = Attribute(attr) + return attributes + + def RemoveAttribute(self, name): + self._impl.RemoveAttribute(name) + + def RemoveAllAttributes(self): + self._impl.RemoveAllAttributes() + + def DefineVariable( + self, + name, + content=None, + shape=[], + start=[], + count=[], + isConstantDims=False, + ): + if isinstance(content, np.ndarray): + return Variable(self._impl, name, content, shape, start, count, isConstantDims) + else: + return Variable(self._impl, name) + + def InquireVariable(self, name): + var = None + var_impl = self._impl.InquireVariable(name) + if var_impl: + var = Variable.__new__(Variable) + var._impl = var_impl + return var + + def AvailableVariables(self): + variables = {} + for name, attr in self._impl.AvailableVariables().items(): + variables[name] = attr + return variables + + def RemoveVariable(self, name): + self._impl.RemoveVariable(name) + + def RemoveAllVariables(self): + self._impl.RemoveAllVariables() + + def Open(self, name, mode, comm=None): + return Engine(self._impl, name, mode, comm) + + def SetEngine(self, name): + self._impl.SetEngine(name) + + def EngineType(self): + return self._impl.EngineType() + + def AddTransport(self, type, parameters=dict()): + self._impl.AddTransport(type, parameters) + + def Parameters(self): + return self._impl.Parameters() + + def SetParameter(self, key, value): + self._impl.SetParameter(key, value) + + def SetParameters(self, parameters): + self._impl.SetParameters(parameters) + + def FlushAll(self): + pass diff --git a/python/adios2/operator.py b/python/adios2/operator.py new file mode 100644 index 0000000000..f217904991 --- /dev/null +++ b/python/adios2/operator.py @@ -0,0 +1,12 @@ +import adios2.bindings + + +class Operator: + def __init__(self, adios_instance, name, type, parameters={}): + self._impl = adios_instance.DefineOperator(name, type, parameters) + + def GetParameters(self): + return self._impl.Parameters() + + def SetParameter(self, key, value): + self._impl.SetParameter(key, value) diff --git a/python/adios2/stream.py b/python/adios2/stream.py new file mode 100644 index 0000000000..07c3064d6c --- /dev/null +++ b/python/adios2/stream.py @@ -0,0 +1,369 @@ +from adios2.adios import ADIOS +import adios2.bindings as bindings + +import numpy as np + + +def open(name, mode, engine_type="BPStream", config_file=None): + """Stream object open""" + return Stream(name, mode, engine_type) + + +def typeAdiosToNumPy(name): + return { + "char": np.int8, + "int8_t": np.int8, + "uint8_t": np.uint8, + "int16_t": np.int16, + "uint16_t": np.uint16, + "int32_t": np.int32, + "uint32_t": np.uint32, + "int64_t": np.int64, + "uint64_t": np.uint64, + "float": np.float32, + "double": np.float64, + }[name] + + +class Stream: + def __init__(self, path, mode="r", comm=None, engine_type="BPStream", config_file=None): + self._io_name = f"stream:{path}:engine_type:{engine_type}" + if comm: + self._adios = ADIOS(comm) + else: + self._adios = ADIOS() + self._io = self._adios.DeclareIO(self._io_name) + self._mode = mode + + if mode == "r": + real_mode = bindings.Mode.Read + elif mode == "w": + real_mode = bindings.Mode.Write + else: + raise ValueError() + + self._engine = self._io.Open(path, real_mode) + self.index = 0 + self.max_steps = 0 + self._buffer = [] + + def __repr__(self): + return f"" + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + def __iter__(self): + return self + + def __next__(self): + if self.index > 0: + self.end_step() + if self.index == self.max_steps: + self.index = 0 + self.max_steps = 0 + raise StopIteration + + self.index += 1 + self.begin_step() + return self + + def set_parameters(self, **kwargs): + """ + Sets parameters using a dictionary. + Removes any previous parameter. + + Parameters + parameters + input key/value parameters + + value + parameter value + """ + self._io.SetParameters(**kwargs) + + def set_transport(self, transport, parameters={}): + """ + + Adds a transport and its parameters to current IO. Must be + supported by current engine type. + + Parameters + type + must be a supported transport type for current engine. + + parameters + acceptable parameters for a particular transport + CAN'T use the keywords "Transport" or "transport" in key + + Returns + transport_index + handler to added transport + """ + self._io.AddTransport(transport, parameters) + + def available_variables(self): + """ + + Returns a 2-level dictionary with variable information. + Read mode only. + + Parameters + keys + list of variable information keys to be extracted (case insensitive) + keys=['AvailableStepsCount','Type','Max','Min','SingleValue','Shape'] + keys=['Name'] returns only the variable names as 1st-level keys + leave empty to return all possible keys + + Returns + variables dictionary + key + variable name + value + variable information dictionary + """ + return self._io.AvailableVariables() + + def available_attributes(self): + """ + Returns a 2-level dictionary with attribute information. + Read mode only. + + Returns + attributes dictionary + key + attribute name + value + attribute information dictionary + """ + return self._io.AvailableAttributes() + + def define_variable(self, name): + return self._io.DefineVariable(name) + + def write(self, name, content, shape=[], start=[], count=[], operations=None): + """ + writes a self-describing array (numpy) variable + + Parameters + name + variable name + + content + variable data values + + shape + variable global MPI dimensions. + + start + variable offset for current MPI rank. + + count + variable dimension for current MPI rank. + + """ + variable = self._io.InquireVariable(name) + + if not variable: + # Sequences variables + if isinstance(content, np.ndarray): + variable = self._io.DefineVariable(name, content, shape, start, count) + elif isinstance(content, list): + content_np = np.array(content) + variable = self._io.DefineVariable(name, content_np, shape, start, count) + # Scalars variables + elif isinstance(content, str) or not hasattr(content, "__len__"): + variable = self.define_variable(name) + else: + raise ValueError + + if shape != [] and not variable.SingleValue(): + variable.SetShape(shape) + + if start != [] and count != []: + variable.SetSelection([start, count]) + + if operations: + variable.RemoveOperations() + for op in operations: + variable.AddOperationString(op[0], op[1]) + + if isinstance(content, list): + content_np = np.array(content) + self._engine.Put(variable, content_np, bindings.Mode.Sync) + self._buffer.append(content_np) + else: + self._engine.Put(variable, content, bindings.Mode.Sync) + self._buffer.append(content) + + def read(self, name, start=[], count=[]): + """ + Random access read allowed to select steps, + only valid with Stream Engines + + Parameters + name + variable to be read + + start + variable offset dimensions + + count + variable local dimensions from offset + Returns + array + resulting array from selection + """ + variable = self._io.InquireVariable(name) + if not variable: + raise ValueError() + + if variable.Type() == "string" and variable.SingleValue() is True: + return self._engine.Get(variable) + + if start != [] and count != []: + variable.SetSelection([start, count]) + + # If this is a LocalArray + if start == [] and count != []: + variable.SetBlockSelection(start) + + shape = (variable.SelectionSize(),) + if count != []: + shape = count + + dtype = typeAdiosToNumPy(variable.Type()) + + output = np.zeros(shape, dtype=dtype) + self._engine.Get(variable, output) + return output + + def write_attribute(self, name, content, variable_name="", separator="/"): + """ + writes a self-describing single value array (numpy) variable + + Parameters + name + attribute name + + array + attribute numpy array data + + variable_name + if attribute is associated with a variable + + separator + concatenation string between variable_name and attribute + e.g. variable_name + separator + name ("var/attr") + Not used if variable_name is empty + """ + attribute = self._io.InquireAttribute(name, variable_name, separator) + if not attribute: + attribute = self._io.DefineAttribute(name, content, variable_name, separator) + + def read_attribute(self, name, variable_name="", separator="/"): + """ + Reads a numpy based attribute + + Parameters + name + attribute name + + variable_name + if attribute is associated with a variable + + separator + concatenation string between variable_name and attribute + e.g. variable_name + separator + name (var/attr) + Not used if variable_name is empty + + Returns + array + resulting array attribute data + """ + attribute = self._io.InquireAttribute(name, variable_name, separator) + if not attribute: + raise KeyError() + + if attribute.Type() == "string": + return attribute.DataString() + else: + return attribute.Data() + + def read_attribute_string(self, name, variable_name="", separator="/"): + """ + Reads a numpy based attribute + + Parameters + name + attribute name + + variable_name + if attribute is associated with a variable + + separator + concatenation string between variable_name and attribute + e.g. variable_name + separator + name (var/attr) + Not used if variable_name is empty + + Returns + array + resulting array attribute data + """ + attribute = self._io.InquireAttribute(name, variable_name, separator) + if not attribute: + raise KeyError() + + return attribute.DataString() + + def begin_step(self): + self._engine.BeginStep() + + def end_step(self): + """ + Write mode: advances to the next step. Convenient when declaring + variable attributes as advancing to the next step is not attached + to any variable. + + Read mode: in streaming mode releases the current step (no effect + in file based engines) + """ + self._buffer = [] + self._engine.EndStep() + + def close(self): + """ + Closes stream, thus becoming unreachable. + Not required if using open in a with-as statement. + Required in all other cases per-open to avoid resource leaks. + """ + self._engine.Close() + self._engine = None + self._io.FlushAll() + self._io = None + self._adios.FlushAll() + self._adios = None + + def current_step(self): + """ + Inspect current step when using for-in loops, read mode only + + Returns + current step + """ + return self._engine.CurrentStep() + + def steps(self, num_steps=0): + if num_steps > 0: + self.max_steps = num_steps + else: + self.max_steps = self._engine.Steps() + + self.index = 0 + return self + + def num_steps(self, num_steps=0): + return self._engine.Steps() diff --git a/python/adios2/variable.py b/python/adios2/variable.py new file mode 100644 index 0000000000..3aec31ff41 --- /dev/null +++ b/python/adios2/variable.py @@ -0,0 +1,71 @@ +import adios2.bindings + + +class Variable: + def __init__(self, io, name, *args, **kwargs): + self._impl = io.DefineVariable(name, *args, **kwargs) + + def __eq__(self, other): + if isinstance(other, Variable): + return self.Name() == other.Name() + return False + + def BlockID(self): + return self._impl.BlockID() + + def Count(self): + return self._impl.Count() + + def SelectionSize(self): + return self._impl.SelectionSize() + + def SetBlockSelection(self, block_id): + self._impl.SetBlockSelection(block_id) + + def SetSelection(self, selection): + self._impl.SetSelection(selection) + + def SetShape(self, shape): + self._impl.SetShape(shape) + + def SetStepSelection(self, step_selection): + self._impl.SetStepSelection(step_selection) + + def Shape(self, step=0): + return self._impl.Shape(step) + + def ShapeID(self): + return self._impl.ShapeID() + + def Type(self): + return self._impl.Type() + + def SingleValue(self): + return bool(self._impl.SingleValue()) + + def Sizeof(self): + return self._impl.Sizeof() + + def Start(self): + return self._impl.Start() + + def Steps(self): + return self._impl.Steps() + + def StepsStart(self): + return self._impl.StepsStart() + + def Name(self): + return self._impl.Name() + + def AddOperationString(self, name, params={}): + return self._impl.AddOperation(name, params) + + def AddOperation(self, op, params={}): + return self._impl.AddOperation(op._impl, params) + + def Operations(self): + return self._impl.Operations() + + def RemoveOperations(self): + self._impl.RemoveOperations() diff --git a/scripts/ci/cmake/ci-win2019-vs2019-serial.cmake b/scripts/ci/cmake/ci-win2019-vs2019-serial.cmake index 945b1e083e..700c13b4b6 100644 --- a/scripts/ci/cmake/ci-win2019-vs2019-serial.cmake +++ b/scripts/ci/cmake/ci-win2019-vs2019-serial.cmake @@ -8,6 +8,7 @@ BUILD_TESTING:BOOL=ON ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_Fortran:BOOL=OFF +ADIOS2_USE_Python:BOOL=OFF ADIOS2_USE_MPI:BOOL=OFF ") diff --git a/scripts/ci/cmake/ci-win2022-vs2022-serial.cmake b/scripts/ci/cmake/ci-win2022-vs2022-serial.cmake index cae84aa729..6fcee2ed46 100644 --- a/scripts/ci/cmake/ci-win2022-vs2022-serial.cmake +++ b/scripts/ci/cmake/ci-win2022-vs2022-serial.cmake @@ -13,6 +13,7 @@ ADIOS2_BUILD_EXAMPLES:BOOL=ON ADIOS2_USE_Fortran:BOOL=OFF ADIOS2_USE_MPI:BOOL=OFF ADIOS2_USE_HDF5:STRING=ON +ADIOS2_USE_Python:BOOL=OFF ADIOS2_USE_HDF5_VOL:STRING=OFF HDF5_ROOT:PATH=C:/hdf5/HDF5-1.14.2.1-win64 ") diff --git a/scripts/ci/scripts/run-flake8.sh b/scripts/ci/scripts/run-flake8.sh index 1c5a1ad354..bacfb5894a 100755 --- a/scripts/ci/scripts/run-flake8.sh +++ b/scripts/ci/scripts/run-flake8.sh @@ -11,4 +11,4 @@ then cd ${SOURCE_DIR} fi -exec flake8 --config=flake8.cfg . +exec flake8 . diff --git a/scripts/developer/run-clang-format.sh b/scripts/developer/run-clang-format.sh index fa6c3a5ab2..7b07d72ae6 100755 --- a/scripts/developer/run-clang-format.sh +++ b/scripts/developer/run-clang-format.sh @@ -1,7 +1,9 @@ #!/bin/bash -exec sudo docker run -itt --mount type=bind,source="$(pwd)",target=/root/adios2 \ +sudo docker run -itt --mount type=bind,source="$(pwd)",target=/root/adios2 \ ghcr.io/ornladios/adios2:ci-formatting sh -c \ "git config --global --add safe.directory /root/adios2 && cd /root/adios2 && ./scripts/ci/scripts/run-clang-format.sh" + +git status --porcelain | awk '{print $2}' | xargs sudo chown "$USER:$(id -g)" diff --git a/testing/adios2/CMakeLists.txt b/testing/adios2/CMakeLists.txt index dc1ede6bca..6e94ac1102 100644 --- a/testing/adios2/CMakeLists.txt +++ b/testing/adios2/CMakeLists.txt @@ -16,3 +16,7 @@ add_subdirectory(backward_compatibility) if (ADIOS2_HAVE_Derived_Variable) add_subdirectory(derived) endif() + +if (ADIOS2_HAVE_Python) +add_subdirectory(python) +endif() diff --git a/testing/adios2/bindings/python/CMakeLists.txt b/testing/adios2/bindings/python/CMakeLists.txt index 7d4b8d4104..eb58b88763 100644 --- a/testing/adios2/bindings/python/CMakeLists.txt +++ b/testing/adios2/bindings/python/CMakeLists.txt @@ -17,13 +17,10 @@ python_add_test(NAME Bindings.Python.TypeExceptionOnGet.Serial SCRIPT TestGetExc if(ADIOS2_HAVE_MPI) add_python_mpi_test(BPWriteReadTypes) - add_python_mpi_test(BPWriteTypesHighLevelAPI) - add_python_mpi_test(BPWriteTypesHighLevelAPILocal) add_python_mpi_test(BPWriteReadString) add_python_mpi_test(BPReadMultisteps) add_python_mpi_test(BPWriteRead2D) add_python_mpi_test(BPBlocksInfo) - add_python_mpi_test(BPChangingShapeHighLevelAPI) add_python_mpi_test(NullEngine) add_python_mpi_test(Query) add_python_mpi_test(QueryLocalArray) @@ -31,13 +28,5 @@ if(ADIOS2_HAVE_MPI) #if(ADIOS2_HAVE_HDF5) # add_python_mpi_test(BPWriteTypesHighLevelAPI_HDF5) #endif() - - if(ADIOS2_HAVE_ZFP) - add_python_mpi_test(BPZfpHighLevelAPI) - endif() - - if(ADIOS2_HAVE_PNG) - add_python_mpi_test(BPPNGHighLevelAPI) - endif() endif() diff --git a/testing/adios2/bindings/python/TestBPBlocksInfo.py b/testing/adios2/bindings/python/TestBPBlocksInfo.py index 90fcc8d7c9..6968ae2ba8 100644 --- a/testing/adios2/bindings/python/TestBPBlocksInfo.py +++ b/testing/adios2/bindings/python/TestBPBlocksInfo.py @@ -13,7 +13,7 @@ from mpi4py import MPI import numpy as np -import adios2 +import adios2.bindings as adios2 # MPI comm = MPI.COMM_WORLD diff --git a/testing/adios2/bindings/python/TestBPPNGHighLevelAPI.py b/testing/adios2/bindings/python/TestBPPNGHighLevelAPI.py deleted file mode 100644 index fc3588b8f4..0000000000 --- a/testing/adios2/bindings/python/TestBPPNGHighLevelAPI.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python - -# -# Distributed under the OSI-approved Apache License, Version 2.0. See -# accompanying file Copyright.txt for details. -# -# TestBPPNGHighLevelAPI.py -# -# Created on: June 7th, 2019 -# Author: William F Godoy - -import numpy as np -import random -from mpi4py import MPI -import adios2 - - -def CompressPNG(compression_level): - - fname = "BPWRPNG_" + str(compression_level) + "_py.bp" - Nx = 10 - Ny = 50 - channels = 3 - NSteps = 1 - - # initialize values - u32s = np.zeros([Nx, Ny], np.uint32) - u8s = np.zeros([Nx, Ny, channels], np.uint8) - - value_ji = 0. - for i in range(0, Nx): - for j in range(0, Ny): - u32s[i][j] = value_ji - u8s[i][j][0] = random.randrange(256) - u8s[i][j][1] = random.randrange(256) - u8s[i][j][2] = random.randrange(256) - - value_ji += 1. - - # set global dimensions - # MPI - comm = MPI.COMM_WORLD - rank = comm.Get_rank() - size = comm.Get_size() - - shape3D = [Nx * size, Ny, 3] - start3D = [Nx * rank, 0, 0] - count3D = [Nx, Ny, 3] - - shape2D = [Nx * size, Ny] - start2D = [Nx * rank, 0] - count2D = [Nx, Ny] - - # writer - with adios2.open(fname, "w", comm) as fw: - - for s in range(0, NSteps): - fw.write("u8", u8s, shape3D, start3D, count3D, - [('PNG', {'bit_depth': '8', - 'color_type': 'PNG_COLOR_TYPE_RGB', - 'compression_level': str(compression_level)})]) - fw.write("u32", u32s, shape2D, start2D, count2D, - [('PNG', {'bit_depth': '8', - 'color_type': 'PNG_COLOR_TYPE_RGBA', - 'compression_level': str(compression_level)})], - end_step=True) - - # reader - with adios2.open(fname, "r", comm) as fr: - - for fstep in fr: - - in_u8s = fstep.read("u8", start3D, count3D) - in_u32s = fstep.read("u32", start2D, count2D) - - for i in range(0, Nx): - for j in range(0, Ny): - assert (u32s[i][j] == in_u32s[i][j]) - assert (u8s[i][j][0] == in_u8s[i][j][0]) - assert (u8s[i][j][1] == in_u8s[i][j][1]) - assert (u8s[i][j][2] == in_u8s[i][j][2]) - - -def main(): - - CompressPNG(compression_level=1) - CompressPNG(compression_level=4) - CompressPNG(compression_level=9) - - -if __name__ == "__main__": - main() diff --git a/testing/adios2/bindings/python/TestBPReadMultisteps.py b/testing/adios2/bindings/python/TestBPReadMultisteps.py index 757d5a8f2f..0b0b1133cf 100644 --- a/testing/adios2/bindings/python/TestBPReadMultisteps.py +++ b/testing/adios2/bindings/python/TestBPReadMultisteps.py @@ -12,7 +12,7 @@ from adios2NPTypes import SmallTestData from mpi4py import MPI import numpy as np -import adios2 +import adios2.bindings as adios2 def check_object(adios2_object, name): diff --git a/testing/adios2/bindings/python/TestBPSelectSteps.py b/testing/adios2/bindings/python/TestBPSelectSteps.py index 4caab59f01..803ee45ccd 100644 --- a/testing/adios2/bindings/python/TestBPSelectSteps.py +++ b/testing/adios2/bindings/python/TestBPSelectSteps.py @@ -10,7 +10,7 @@ import unittest import numpy as np from mpi4py import MPI -import adios2 +import adios2.bindings as adios2 TESTDATA_FILENAME = "steps_int32.bp" diff --git a/testing/adios2/bindings/python/TestBPWriteRead2D.py b/testing/adios2/bindings/python/TestBPWriteRead2D.py index 8d755f55ce..17a04a2c8e 100644 --- a/testing/adios2/bindings/python/TestBPWriteRead2D.py +++ b/testing/adios2/bindings/python/TestBPWriteRead2D.py @@ -12,7 +12,7 @@ from mpi4py import MPI import numpy as np -import adios2 +import adios2.bindings as adios2 # MPI comm = MPI.COMM_WORLD diff --git a/testing/adios2/bindings/python/TestBPWriteReadString.py b/testing/adios2/bindings/python/TestBPWriteReadString.py index 682340da5c..355347c7a4 100644 --- a/testing/adios2/bindings/python/TestBPWriteReadString.py +++ b/testing/adios2/bindings/python/TestBPWriteReadString.py @@ -9,7 +9,7 @@ # Author: Dmitry Ganyushin ganyushindi@ornl.gov import unittest from mpi4py import MPI -import adios2 +import adios2.bindings as adios2 N_STEPS = 3 @@ -41,38 +41,6 @@ def test_write_read_string_fullAPI(self): self.assertEqual(result, theString + str(step)) adEngine.Close() - def test_write_read_string_highAPI(self): - comm = MPI.COMM_WORLD - theString = 'hello adios' - bpFilename = 'string_test_highAPI.bp' - varname = 'mystringvar' - - with adios2.open(bpFilename, "w", comm) as fh: - - for step in range(N_STEPS): - fh.write(varname, theString + str(step), end_step=True) - - with adios2.open(bpFilename, "r", comm) as fh: - for fstep in fh: - step = fstep.current_step() - result = fstep.read_string(varname) - self.assertEqual(result, [theString + str(step)]) - - def test_read_strings_all_steps(self): - comm = MPI.COMM_WORLD - fileName = 'string_test_all.bp' - with adios2.open(fileName, "w", comm) as fh: - for i in range(N_STEPS): - fh.write("string_variable", "written {}".format(i)) - fh.end_step() - - with adios2.open(fileName, "rra", comm) as fh: - n = fh.steps() - name = "string_variable" - result = fh.read_string(name, 0, n) - expected_str = ["written {}".format(i) for i in range(n)] - self.assertEqual(result, expected_str) - if __name__ == '__main__': unittest.main() diff --git a/testing/adios2/bindings/python/TestBPWriteReadTypes.py b/testing/adios2/bindings/python/TestBPWriteReadTypes.py index b65d481712..829ace519d 100644 --- a/testing/adios2/bindings/python/TestBPWriteReadTypes.py +++ b/testing/adios2/bindings/python/TestBPWriteReadTypes.py @@ -11,7 +11,7 @@ from adios2NPTypes import SmallTestData from mpi4py import MPI -import adios2 +import adios2.bindings as adios2 def check_object(adios2_object, name): diff --git a/testing/adios2/bindings/python/TestBPWriteReadTypes_nompi.py b/testing/adios2/bindings/python/TestBPWriteReadTypes_nompi.py index 28ca3730bd..edee61f1c1 100644 --- a/testing/adios2/bindings/python/TestBPWriteReadTypes_nompi.py +++ b/testing/adios2/bindings/python/TestBPWriteReadTypes_nompi.py @@ -10,7 +10,7 @@ from adios2NPTypes import SmallTestData -import adios2 +import adios2.bindings as adios2 # Test data diff --git a/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPI.py b/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPI.py deleted file mode 100644 index d3eb4b58dc..0000000000 --- a/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPI.py +++ /dev/null @@ -1,365 +0,0 @@ -#!/usr/bin/env python - -# -# Distributed under the OSI-approved Apache License, Version 2.0. See -# accompanying file Copyright.txt for details. -# -# TestBPWriteTypes.py: test Python numpy types in ADIOS2 File -# Write/Read High-Level API -# Created on: March 12, 2018 -# Author: William F Godoy godoywf@ornl.gov - -from adios2NPTypes import SmallTestData -from mpi4py import MPI -import numpy as np -import adios2 - -comm = MPI.COMM_WORLD -rank = comm.Get_rank() -size = comm.Get_size() - -# Test data -data = SmallTestData() -nx = data.Nx - -shape = [size * nx] -start = [rank * nx] -count = [nx] - -# Writer -with adios2.open("types_np.bp", "w", comm) as fw: - - for i in range(0, 5): - - data.update(rank, i, size) - - if rank == 0 and i == 0: - fw.write("tag", "Testing ADIOS2 high-level API") - fw.write("gvarI8", np.array(data.I8[0])) - fw.write("gvarI16", np.array(data.I16[0])) - fw.write("gvarI32", np.array(data.I32[0])) - fw.write("gvarI64", np.array(data.I64[0])) - fw.write("gvarU8", np.array(data.U8[0])) - fw.write("gvarU16", np.array(data.U16[0])) - fw.write("gvarU32", np.array(data.U32[0])) - fw.write("gvarU64", np.array(data.U64[0])) - fw.write("gvarR32", np.array(data.R32[0])) - fw.write("gvarR64", np.array(data.R64[0])) - - # single value attributes - fw.write_attribute("attrStr", "Testing single string attribute") - fw.write_attribute("attrI8", np.array(data.I8[0])) - fw.write_attribute("attrI16", np.array(data.I16[0])) - fw.write_attribute("attrI32", np.array(data.I32[0])) - fw.write_attribute("attrI64", np.array(data.I64[0])) - fw.write_attribute("attrU8", np.array(data.U8[0])) - fw.write_attribute("attrU16", np.array(data.U16[0])) - fw.write_attribute("attrU32", np.array(data.U32[0])) - fw.write_attribute("attrU64", np.array(data.U64[0])) - fw.write_attribute("attrR32", np.array(data.R32[0])) - fw.write_attribute("attrR64", np.array(data.R64[0])) - - fw.write_attribute( - "attrStrArray", ["string1", "string2", "string3"]) - fw.write_attribute("attrI8Array", data.I8) - fw.write_attribute("attrI16Array", data.I16) - fw.write_attribute("attrI32Array", data.I32) - fw.write_attribute("attrI64Array", data.I64) - fw.write_attribute("attrU8Array", data.U8) - fw.write_attribute("attrU16Array", data.U16) - fw.write_attribute("attrU32Array", data.U32) - fw.write_attribute("attrU64Array", data.U64) - fw.write_attribute("attrR32Array", data.R32) - fw.write_attribute("attrR64Array", data.R64) - - fw.write("rank", np.array(rank), [adios2.LocalValueDim]) - fw.write("steps", "Step:" + str(i)) - fw.write("varI8", data.I8, shape, start, count) - fw.write("varI16", data.I16, shape, start, count) - fw.write("varI32", data.I32, shape, start, count) - fw.write("varI64", data.I64, shape, start, count) - fw.write("varU8", data.U8, shape, start, count) - fw.write("varU16", data.U16, shape, start, count) - fw.write("varU32", data.U32, shape, start, count) - fw.write("varU64", data.U64, shape, start, count) - fw.write("varR32", data.R32, shape, start, count) - fw.write("varR64", data.R64, shape, start, count) - - if rank == 0 and i == 0: - fw.write_attribute("varattrStrArray", [ - "varattr1", "varattr2", "varattr3"], "steps") - fw.write_attribute("varattrI8Array", data.I8, "varI8") - fw.write_attribute("varattrI16Array", data.I16, "varI16") - fw.write_attribute("varattrI32Array", data.I32, "varI32") - fw.write_attribute("varattrI64Array", data.I64, "varI64") - fw.write_attribute("varattrU8Array", data.U8, "varU8") - fw.write_attribute("varattrU16Array", data.U16, "varU16") - fw.write_attribute("varattrU32Array", data.U32, "varU32") - fw.write_attribute("varattrU64Array", data.U64, "varU64") - fw.write_attribute("varattrR32Array", data.R32, "varR32") - fw.write_attribute("varattrR64Array", data.R64, "varR64") - fw.write_attribute("varattrR64Value", data.R64, "varR64") - - fw.end_step() - -comm.Barrier() - -# Reader -data = SmallTestData() - -with adios2.open("types_np.bp", "r", comm) as fr: - - # file only - assert (fr.steps() == 5) - - for fr_step in fr: - - step = fr_step.current_step() - data.update(rank, step, size) - - step_vars = fr_step.available_variables() - -# for name, info in step_vars.items(): -# print("variable_name: " + name) -# for key, value in info.items(): -# print("\t" + key + ": " + value) -# print("\n") - - if step == 0: - inTag = fr_step.read_string("tag") - inI8 = fr_step.read("gvarI8") - inI16 = fr_step.read("gvarI16") - inI32 = fr_step.read("gvarI32") - inI64 = fr_step.read("gvarI64") - inU8 = fr_step.read("gvarU8") - inU16 = fr_step.read("gvarU16") - inU32 = fr_step.read("gvarU32") - inU64 = fr_step.read("gvarU64") - inR32 = fr_step.read("gvarR32") - inR64 = fr_step.read("gvarR64") - - if inTag[0] != "Testing ADIOS2 high-level API": - print("InTag: " + str(inTag)) - raise ValueError('tag variable read failed') - - if inI8 != data.I8[0]: - raise ValueError('gvarI8 read failed') - - if inI16 != data.I16[0]: - raise ValueError('gvarI16 read failed') - - if inI32 != data.I32[0]: - raise ValueError('gvarI32 read failed') - - if inI64 != data.I64[0]: - raise ValueError('gvarI64 read failed') - - if inU8 != data.U8[0]: - raise ValueError('gvarU8 read failed') - - if inU16 != data.U16[0]: - raise ValueError('gvarU16 read failed') - - if inU32 != data.U32[0]: - raise ValueError('gvarU32 read failed') - - if inU64 != data.U64[0]: - raise ValueError('gvarU64 read failed') - - if inR32 != data.R32[0]: - raise ValueError('gvarR32 read failed') - - if inR64 != data.R64[0]: - raise ValueError('gvarR64 read failed') - - # attributes - inTag = fr_step.read_attribute_string("attrStr") - inI8 = fr_step.read_attribute("attrI8") - inI16 = fr_step.read_attribute("attrI16") - inI32 = fr_step.read_attribute("attrI32") - inI64 = fr_step.read_attribute("attrI64") - inU8 = fr_step.read_attribute("attrU8") - inU16 = fr_step.read_attribute("attrU16") - inU32 = fr_step.read_attribute("attrU32") - inU64 = fr_step.read_attribute("attrU64") - inR32 = fr_step.read_attribute("attrR32") - inR64 = fr_step.read_attribute("attrR64") - - if inTag[0] != "Testing single string attribute": - raise ValueError('attr string read failed') - - if inI8[0] != data.I8[0]: - raise ValueError('attrI8 read failed') - - if inI16[0] != data.I16[0]: - raise ValueError('attrI16 read failed') - - if inI32[0] != data.I32[0]: - raise ValueError('attrI32 read failed') - - if inI64[0] != data.I64[0]: - raise ValueError('attrI64 read failed') - - if inU8[0] != data.U8[0]: - raise ValueError('attrU8 read failed') - - if inU16[0] != data.U16[0]: - raise ValueError('attrU16 read failed') - - if inU32[0] != data.U32[0]: - raise ValueError('attrU32 read failed') - - if inU64[0] != data.U64[0]: - raise ValueError('attrU64 read failed') - - if inR32[0] != data.R32[0]: - raise ValueError('attrR32 read failed') - - if inR64[0] != data.R64[0]: - raise ValueError('attrR64 read failed') - - # Array attribute - inTag = fr_step.read_attribute_string("attrStrArray") - inI8 = fr_step.read_attribute("attrI8Array") - inI16 = fr_step.read_attribute("attrI16Array") - inI32 = fr_step.read_attribute("attrI32Array") - inI64 = fr_step.read_attribute("attrI64Array") - inU8 = fr_step.read_attribute("attrU8Array") - inU16 = fr_step.read_attribute("attrU16Array") - inU32 = fr_step.read_attribute("attrU32Array") - inU64 = fr_step.read_attribute("attrU64Array") - inR32 = fr_step.read_attribute("attrR32Array") - inR64 = fr_step.read_attribute("attrR64Array") - - if inTag != ["string1", "string2", "string3"]: - raise ValueError('attrStrArray read failed') - - if (inI8 == data.I8).all() is False: - raise ValueError('attrI8 array read failed') - - if (inI16 == data.I16).all() is False: - raise ValueError('attrI16 array read failed') - - if (inI32 == data.I32).all() is False: - raise ValueError('attrI32 array read failed') - - if (inI64 == data.I64).all() is False: - raise ValueError('attrI64 array read failed') - - if (inU8 == data.U8).all() is False: - raise ValueError('attrU8 array read failed') - - if (inU16 == data.U16).all() is False: - raise ValueError('attrU16 array read failed') - - if (inU32 == data.U32).all() is False: - raise ValueError('attrU32 array read failed') - - if (inU64 == data.U64).all() is False: - raise ValueError('attrU64 array read failed') - - if (inR32 == data.R32).all() is False: - raise ValueError('attrR32 array read failed') - - if (inR64 == data.R64).all() is False: - raise ValueError('attrR64 array read failed') - - inTags = fr_step.read_attribute_string("varattrStrArray", "steps") - inI8 = fr_step.read_attribute("varattrI8Array", "varI8") - in16 = fr_step.read_attribute("varattrI16Array", "varI16") - inI32 = fr_step.read_attribute("varattrI32Array", "varI32") - inI64 = fr_step.read_attribute("varattrI64Array", "varI64") - inU8 = fr_step.read_attribute("varattrU8Array", "varU8") - inU16 = fr_step.read_attribute("varattrU16Array", "varU16") - inU32 = fr_step.read_attribute("varattrU32Array", "varU32") - inU64 = fr_step.read_attribute("varattrU64Array", "varU64") - inR32 = fr_step.read_attribute("varattrR32Array", "varR32") - inR64 = fr_step.read_attribute("varattrR64Array", "varR64") - - if inTags != ["varattr1", "varattr2", "varattr3"]: - print(inTags) - raise ValueError('var attrStrArray read failed') - - if (inI8 == data.I8).all() is False: - raise ValueError('var attrI8 array read failed') - - if (inI16 == data.I16).all() is False: - raise ValueError('var attrI16 array read failed') - - if (inI32 == data.I32).all() is False: - raise ValueError('var attrI32 array read failed') - - if (inI64 == data.I64).all() is False: - raise ValueError('var attrI64 array read failed') - - if (inU8 == data.U8).all() is False: - raise ValueError('var attrU8 array read failed') - - if (inU16 == data.U16).all() is False: - raise ValueError('var attrU16 array read failed') - - if (inU32 == data.U32).all() is False: - raise ValueError('var attrU32 array read failed') - - if (inU64 == data.U64).all() is False: - raise ValueError('var attrU64 array read failed') - - if (inR32 == data.R32).all() is False: - raise ValueError('var attrR32 array read failed') - - if (inR64 == data.R64).all() is False: - raise ValueError('var attrR64 array read failed') - - stepStr = "Step:" + str(step) - - instepStr = fr_step.read_string("steps") - if instepStr[0] != stepStr: - raise ValueError('steps variable read failed: ' + - instepStr + " " + stepStr) - - indataRanks = fr_step.read("rank", [0], [size]) - dataRanks = np.arange(0, size) - if (indataRanks == dataRanks).all() is False: - raise ValueError('Ranks read failed') - - indataI8 = fr_step.read("varI8", start, count) - indataI16 = fr_step.read("varI16", start, count) - indataI32 = fr_step.read("varI32", start, count) - indataI64 = fr_step.read("varI64", start, count) - indataU8 = fr_step.read("varU8", start, count) - indataU16 = fr_step.read("varU16", start, count) - indataU32 = fr_step.read("varU32", start, count) - indataU64 = fr_step.read("varU64", start, count) - indataR32 = fr_step.read("varR32", start, count) - indataR64 = fr_step.read("varR64", start, count) - fr_step.end_step() - - if (indataI8 == data.I8).all() is False: - raise ValueError('I8 array read failed') - - if (indataI16 == data.I16).all() is False: - raise ValueError('I16 array read failed') - - if (indataI32 == data.I32).all() is False: - raise ValueError('I32 array read failed') - - if (indataI64 == data.I64).all() is False: - raise ValueError('I64 array read failed') - - if (indataU8 == data.U8).all() is False: - raise ValueError('U8 array read failed') - - if (indataU16 == data.U16).all() is False: - raise ValueError('U16 array read failed') - - if (indataU32 == data.U32).all() is False: - raise ValueError('U32 array read failed') - - if (indataU64 == data.U64).all() is False: - raise ValueError('U64 array read failed') - - if (indataR32 == data.R32).all() is False: - raise ValueError('R32 array read failed') - - if (indataR64 == data.R64).all() is False: - raise ValueError('R64 array read failed') diff --git a/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPILocal.py b/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPILocal.py deleted file mode 100644 index 842d8ea5ac..0000000000 --- a/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPILocal.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python - -# Distributed under the OSI-approved Apache License, Version 2.0. See -# accompanying file Copyright.txt for details. -# -# TestBPWriteTypesHighLevelAPILocal.py: test Python numpy types in ADIOS2 File -# Write/Read High-Level API for Local Arrays -# Created on: March 12, 2018 -# Author: William F Godoy godoywf@ornl.gov - -from adios2NPTypes import SmallTestData -from mpi4py import MPI -import adios2 - - -def check_array(np1, np2, hint): - if (np1 == np2).all() is False: - print("InData: " + str(np1)) - print("Data: " + str(np2)) - raise ValueError('Array read failed ' + str(hint)) - - -comm = MPI.COMM_WORLD -rank = comm.Get_rank() -size = comm.Get_size() - -# Test data -data = SmallTestData() -nx = data.Nx - -shape = [] -start = [] -count = [nx] - -# Writer -with adios2.open("types_np_local.bp", "w", comm) as fw: - - for i in range(0, 5): - - data.update(rank, i, size) - - fw.write("varI8", data.I8, shape, start, count) - fw.write("varI16", data.I16, shape, start, count) - fw.write("varI32", data.I32, shape, start, count) - fw.write("varI64", data.I64, shape, start, count) - fw.write("varU8", data.U8, shape, start, count) - fw.write("varU16", data.U16, shape, start, count) - fw.write("varU32", data.U32, shape, start, count) - fw.write("varU64", data.U64, shape, start, count) - fw.write("varR32", data.R32, shape, start, count) - fw.write("varR64", data.R64, shape, start, count) - fw.end_step() - - -# Reader -data = SmallTestData() - -with adios2.open("types_np_local.bp", "r", comm) as fr: - - for fr_step in fr: - - step = fr_step.current_step() - - for b in range(0, size): - - data.update(b, step, size) - - indataI8 = fr_step.read("varI8", b) - indataI16 = fr_step.read("varI16", b) - indataI32 = fr_step.read("varI32", b) - indataI64 = fr_step.read("varI64", b) - indataU8 = fr_step.read("varU8", b) - indataU16 = fr_step.read("varU16", b) - indataU32 = fr_step.read("varU32", b) - indataU64 = fr_step.read("varU64", b) - indataR32 = fr_step.read("varR32", b) - indataR64 = fr_step.read("varR64", b) - - check_array(indataI8, data.I8, 'I8') - check_array(indataI16, data.I16, 'I16') - check_array(indataI32, data.I32, 'I32') - check_array(indataI64, data.I64, 'I64') - check_array(indataU8, data.U8, 'U8') - check_array(indataU16, data.U16, 'U16') - check_array(indataU32, data.U32, 'U32') - check_array(indataU64, data.U64, 'U64') - check_array(indataR32, data.R32, 'R32') - check_array(indataR64, data.R64, 'R64') diff --git a/testing/adios2/bindings/python/TestGetException_nompi.py b/testing/adios2/bindings/python/TestGetException_nompi.py index 77248d92fa..6e2d8a333d 100644 --- a/testing/adios2/bindings/python/TestGetException_nompi.py +++ b/testing/adios2/bindings/python/TestGetException_nompi.py @@ -1,7 +1,7 @@ import numpy as np import logging -import adios2 +import adios2.bindings as adios2 if __name__ == '__main__': __spec__ = None diff --git a/testing/adios2/bindings/python/TestNullEngine.py b/testing/adios2/bindings/python/TestNullEngine.py index dfe1c0069c..ac5fb1796a 100644 --- a/testing/adios2/bindings/python/TestNullEngine.py +++ b/testing/adios2/bindings/python/TestNullEngine.py @@ -11,7 +11,7 @@ from mpi4py import MPI import numpy as np -import adios2 +import adios2.bindings as adios2 # MPI comm = MPI.COMM_WORLD diff --git a/testing/adios2/bindings/python/TestQuery.py b/testing/adios2/bindings/python/TestQuery.py index e406ce3cf5..d0ee554d0f 100644 --- a/testing/adios2/bindings/python/TestQuery.py +++ b/testing/adios2/bindings/python/TestQuery.py @@ -1,7 +1,7 @@ # from mpi4py import MPI import numpy as np -import adios2 +import adios2.bindings as adios2 import sys # MPI diff --git a/testing/adios2/bindings/python/TestQueryLocalArray.py b/testing/adios2/bindings/python/TestQueryLocalArray.py index 8025d540d7..4eab11acde 100644 --- a/testing/adios2/bindings/python/TestQueryLocalArray.py +++ b/testing/adios2/bindings/python/TestQueryLocalArray.py @@ -1,7 +1,7 @@ # from mpi4py import MPI import numpy as np -import adios2 +import adios2.bindings as adios2 import sys # MPI diff --git a/testing/adios2/python/CMakeLists.txt b/testing/adios2/python/CMakeLists.txt new file mode 100644 index 0000000000..3e9ff46447 --- /dev/null +++ b/testing/adios2/python/CMakeLists.txt @@ -0,0 +1,34 @@ +#------------------------------------------------------------------------------# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +#------------------------------------------------------------------------------# + +function(add_python_mpi_test testname) + python_add_test(NAME Api.Python.${testname}.MPI SCRIPT Test${testname}.py + EXEC_WRAPPER ${MPIEXEC_COMMAND} + ) + set_tests_properties(Api.Python.${testname}.MPI PROPERTIES + PROCESSORS "${MPIEXEC_MAX_NUMPROCS}" + ) +endfunction() + +python_add_test(NAME Api.Python.ADIOS SCRIPT TestADIOS.py) +python_add_test(NAME Api.Python.Engine SCRIPT TestEngine.py) +python_add_test(NAME Api.Python.IO SCRIPT TestIO.py) +python_add_test(NAME Api.Python.Operator SCRIPT TestOperator.py) +python_add_test(NAME Api.Python.Variable SCRIPT TestVariable.py) +python_add_test(NAME Api.Python.Attribute SCRIPT TestAttribute.py) +python_add_test(NAME Api.Python.Stream SCRIPT TestStream.py) + +if(ADIOS2_HAVE_MPI) + add_python_mpi_test(BPWriteReadString) + add_python_mpi_test(BPWriteTypesHighLevelAPI) + add_python_mpi_test(BPWriteTypesHighLevelAPILocal) + add_python_mpi_test(BPChangingShapeHighLevelAPI) + if(ADIOS2_HAVE_PNG) + add_python_mpi_test(BPPNGHighLevelAPI) + endif() + if(ADIOS2_HAVE_ZFP) + add_python_mpi_test(BPZfpHighLevelAPI) + endif() +endif() diff --git a/testing/adios2/python/TestADIOS.py b/testing/adios2/python/TestADIOS.py new file mode 100644 index 0000000000..a7ddea2b26 --- /dev/null +++ b/testing/adios2/python/TestADIOS.py @@ -0,0 +1,56 @@ +from adios2.adios import ADIOS +import unittest + + +class TestADIOS(unittest.TestCase): + def test_define_operator(self): + adios = ADIOS() + op = adios.DefineOperator("op", "null") + self.assertNotEqual(op, None) + + def test_inquiry_operator(self): + adios = ADIOS() + op1 = adios.DefineOperator("op1", "null") + op2 = adios.DefineOperator("op2", "null") + op_x = adios.InquireOperator("op2") + self.assertNotEqual(op1, op2) + self.assertEqual(op2, op_x) + self.assertNotEqual(op1, op_x) + with self.assertRaises(KeyError): + adios.InquireOperator("NonExisting") + + def test_declare_io(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + self.assertNotEqual(writer, None) + + def test_at_io(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + reader = adios.DeclareIO("BPReader") + x = adios.AtIO("BPReader") + self.assertNotEqual(writer, reader) + self.assertEqual(reader, x) + self.assertNotEqual(writer, x) + with self.assertRaises(KeyError): + adios.AtIO("NonExisting") + + def test_remove_io(self): + adios = ADIOS() + adios.DeclareIO("BPWriter") + adios.RemoveIO("BPWriter") + with self.assertRaises(KeyError): + adios.AtIO("BPWriter") + + def test_remove_all_io(self): + adios = ADIOS() + adios.DeclareIO("BPWriter") + adios.DeclareIO("BPReader") + adios.RemoveAllIOs() + with self.assertRaises(KeyError): + adios.AtIO("BPWriter") + adios.AtIO("BPReader") + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/adios2/python/TestAttribute.py b/testing/adios2/python/TestAttribute.py new file mode 100644 index 0000000000..9e5c286058 --- /dev/null +++ b/testing/adios2/python/TestAttribute.py @@ -0,0 +1,34 @@ +from adios2.adios import ADIOS + +import adios2.bindings as bindings + +import unittest +import numpy as np + + +class TestAttribute(unittest.TestCase): + def test_create_write(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + ts = writer.DefineAttribute("timestamp", "20231122") + self.assertEqual(ts.Name(), "timestamp") + self.assertEqual(ts.DataString(), ["20231122"]) + + def test_create_reader(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + ts = writer.DefineAttribute("timestamp", "20231122") + self.assertEqual(ts.Name(), "timestamp") + self.assertEqual(ts.DataString(), ["20231122"]) + + def test_create_write_ndarray(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + arr = np.array([2023, 11, 22]) + ts = writer.DefineAttribute("timestamp", arr) + self.assertEqual(ts.Name(), "timestamp") + self.assertTrue(np.array_equal(ts.Data(), [2023, 11, 22])) + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/adios2/bindings/python/TestBPChangingShapeHighLevelAPI.py b/testing/adios2/python/TestBPChangingShapeHighLevelAPI.py similarity index 58% rename from testing/adios2/bindings/python/TestBPChangingShapeHighLevelAPI.py rename to testing/adios2/python/TestBPChangingShapeHighLevelAPI.py index 0fbd3d89b2..6497650341 100644 --- a/testing/adios2/bindings/python/TestBPChangingShapeHighLevelAPI.py +++ b/testing/adios2/python/TestBPChangingShapeHighLevelAPI.py @@ -11,7 +11,7 @@ import numpy as np from mpi4py import MPI -import adios2 +from adios2 import Stream comm = MPI.COMM_WORLD rank = comm.Get_rank() @@ -25,13 +25,17 @@ count = [[nx[0]], [nx[1]]] # Write different sized arrays as separate steps -with adios2.open('out.bp', 'w', comm) as f: - f.write('z', data[0], shape[0], start[0], count[0], end_step=True) - f.write('z', data[1], shape[1], start[1], count[1], end_step=True) +with Stream("out.bp", "w", comm) as s: + s.begin_step() + s.write("z", data[0], shape[0], start[0], count[0]) + s.end_step() + s.begin_step() + s.write("z", data[1], shape[1], start[1], count[1]) + s.end_step() # Read back arrays -with adios2.open('out.bp', 'r', comm) as f: - for f_step in f: - shape_z = int(f_step.available_variables()['z']['Shape']) +with Stream("out.bp", "r", comm) as s: + for step in s.steps(): + shape_z = int(step.available_variables()["z"]["Shape"]) print(shape_z) - assert (shape_z == int(shape[f_step.current_step()][0])) + assert shape_z == int(shape[step.current_step()][0]) diff --git a/testing/adios2/python/TestBPPNGHighLevelAPI.py b/testing/adios2/python/TestBPPNGHighLevelAPI.py new file mode 100644 index 0000000000..c5a6426751 --- /dev/null +++ b/testing/adios2/python/TestBPPNGHighLevelAPI.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python + +# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +# +# TestBPPNGHighLevelAPI.py +# +# Created on: June 7th, 2019 +# Author: William F Godoy + +import numpy as np +import random +from mpi4py import MPI +from adios2 import Stream + + +def CompressPNG(compression_level): + fname = "BPWRPNG_" + str(compression_level) + "_py.bp" + Nx = 10 + Ny = 50 + channels = 3 + NSteps = 1 + + # initialize values + u32s = np.zeros([Nx, Ny], np.uint32) + u8s = np.zeros([Nx, Ny, channels], np.uint8) + + value_ji = 0.0 + for i in range(0, Nx): + for j in range(0, Ny): + u32s[i][j] = value_ji + u8s[i][j][0] = random.randrange(256) + u8s[i][j][1] = random.randrange(256) + u8s[i][j][2] = random.randrange(256) + + value_ji += 1.0 + + # set global dimensions + # MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + size = comm.Get_size() + + shape3D = [Nx * size, Ny, 3] + start3D = [Nx * rank, 0, 0] + count3D = [Nx, Ny, 3] + + shape2D = [Nx * size, Ny] + start2D = [Nx * rank, 0] + count2D = [Nx, Ny] + + # writer + with Stream(fname, "w", comm) as s: + for step in s.steps(NSteps): + s.write( + "u8", + u8s, + shape3D, + start3D, + count3D, + [ + ( + "PNG", + { + "bit_depth": "8", + "color_type": "PNG_COLOR_TYPE_RGB", + "compression_level": str(compression_level), + }, + ) + ], + ) + s.write( + "u32", + u32s, + shape2D, + start2D, + count2D, + [ + ( + "PNG", + { + "bit_depth": "8", + "color_type": "PNG_COLOR_TYPE_RGBA", + "compression_level": str(compression_level), + }, + ) + ], + ) + + # reader + with Stream(fname, "r", comm) as s: + for step in s.steps(): + in_u8s = step.read("u8", start3D, count3D) + in_u32s = step.read("u32", start2D, count2D) + + for i in range(0, Nx): + for j in range(0, Ny): + assert u32s[i][j] == in_u32s[i][j] + assert u8s[i][j][0] == in_u8s[i][j][0] + assert u8s[i][j][1] == in_u8s[i][j][1] + assert u8s[i][j][2] == in_u8s[i][j][2] + + +def main(): + CompressPNG(compression_level=1) + CompressPNG(compression_level=4) + CompressPNG(compression_level=9) + + +if __name__ == "__main__": + main() diff --git a/testing/adios2/python/TestBPWriteReadString.py b/testing/adios2/python/TestBPWriteReadString.py new file mode 100644 index 0000000000..a9b819b02a --- /dev/null +++ b/testing/adios2/python/TestBPWriteReadString.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +# +# TestBPWriteReadString.py: test writing/reading Python string type +# in ADIOS2 File Write +# Created on: Oct 19, 2020 +# Author: Dmitry Ganyushin ganyushindi@ornl.gov +import unittest +from mpi4py import MPI +from adios2 import Stream + +N_STEPS = 3 + + +class TestAdiosWriteReadStringfullAPI(unittest.TestCase): + def test_write_read_string_highAPI(self): + comm = MPI.COMM_WORLD + theString = "hello adios" + bpFilename = "string_test_highAPI.bp" + varname = "mystringvar" + + with Stream(bpFilename, "w", comm) as s: + for step in s.steps(N_STEPS): + s.write(varname, theString + str(step.current_step())) + + with Stream(bpFilename, "r", comm) as s: + for _ in s.steps(): + step = s.current_step() + result = s.read(varname) + self.assertEqual(result, theString + str(step)) + + def test_read_strings_all_steps(self): + comm = MPI.COMM_WORLD + fileName = "string_test_all.bp" + with Stream(fileName, "w", comm) as s: + i = 0 + for _ in s.steps(N_STEPS): + s.write("string_variable", "written {}".format(i)) + i += 1 + + # with Stream(fileName, "rra", comm) as s: + # n = s.num_steps() + # name = "string_variable" + # result = s.read_string(name, 0, n) + # expected_str = ["written {}".format(i) for i in range(n)] + # self.assertEqual(result, expected_str) + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py b/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py new file mode 100644 index 0000000000..5e6d7fb445 --- /dev/null +++ b/testing/adios2/python/TestBPWriteTypesHighLevelAPI.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python + +# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +# +# TestBPWriteTypes.py: test Python numpy types in ADIOS2 File +# Write/Read High-Level API +# Created on: March 12, 2018 +# Author: William F Godoy godoywf@ornl.gov + +from adios2NPTypes import SmallTestData +from mpi4py import MPI +import numpy as np +from adios2 import Stream, LocalValueDim + +comm = MPI.COMM_WORLD +rank = comm.Get_rank() +size = comm.Get_size() + +# Test data +data = SmallTestData() +nx = data.Nx + +shape = [size * nx] +start = [rank * nx] +count = [nx] + +# Writer +with Stream("types_np.bp", "w", comm) as s: + for step in s.steps(5): + data.update(rank, step.current_step(), size) + s.write("rank", np.array(rank), shape=[LocalValueDim]) + if rank == 0 and step.current_step() == 0: + s.write("tag", "Testing ADIOS2 high-level API") + s.write("gvarI8", np.array(data.I8[0])) + s.write("gvarI16", np.array(data.I16[0])) + s.write("gvarI32", np.array(data.I32[0])) + s.write("gvarI64", np.array(data.I64[0])) + s.write("gvarU8", np.array(data.U8[0])) + s.write("gvarU16", np.array(data.U16[0])) + s.write("gvarU32", np.array(data.U32[0])) + s.write("gvarU64", np.array(data.U64[0])) + s.write("gvarR32", np.array(data.R32[0])) + s.write("gvarR64", np.array(data.R64[0])) + + # single value attributes + s.write_attribute("attrStr", "Testing single string attribute") + s.write_attribute("attrI8", np.array(data.I8[0])) + s.write_attribute("attrI16", np.array(data.I16[0])) + s.write_attribute("attrI32", np.array(data.I32[0])) + s.write_attribute("attrI64", np.array(data.I64[0])) + s.write_attribute("attrU8", np.array(data.U8[0])) + s.write_attribute("attrU16", np.array(data.U16[0])) + s.write_attribute("attrU32", np.array(data.U32[0])) + s.write_attribute("attrU64", np.array(data.U64[0])) + s.write_attribute("attrR32", np.array(data.R32[0])) + s.write_attribute("attrR64", np.array(data.R64[0])) + + s.write_attribute("attrStrArray", ["string1", "string2", "string3"]) + s.write_attribute("attrI8Array", data.I8) + s.write_attribute("attrI16Array", data.I16) + s.write_attribute("attrI32Array", data.I32) + s.write_attribute("attrI64Array", data.I64) + s.write_attribute("attrU8Array", data.U8) + s.write_attribute("attrU16Array", data.U16) + s.write_attribute("attrU32Array", data.U32) + s.write_attribute("attrU64Array", data.U64) + s.write_attribute("attrR32Array", data.R32) + s.write_attribute("attrR64Array", data.R64) + + s.write("steps", "Step:" + str(step.current_step())) + s.write("varI8", data.I8, shape, start, count) + s.write("varI16", data.I16, shape, start, count) + s.write("varI32", data.I32, shape, start, count) + s.write("varI64", data.I64, shape, start, count) + s.write("varU8", data.U8, shape, start, count) + s.write("varU16", data.U16, shape, start, count) + s.write("varU32", data.U32, shape, start, count) + s.write("varU64", data.U64, shape, start, count) + s.write("varR32", data.R32, shape, start, count) + s.write("varR64", data.R64, shape, start, count) + + if rank == 0 and step.current_step() == 0: + s.write_attribute("varattrStrArray", ["varattr1", "varattr2", "varattr3"], "steps") + s.write_attribute("varattrI8Array", data.I8, "varI8") + s.write_attribute("varattrI16Array", data.I16, "varI16") + s.write_attribute("varattrI32Array", data.I32, "varI32") + s.write_attribute("varattrI64Array", data.I64, "varI64") + s.write_attribute("varattrU8Array", data.U8, "varU8") + s.write_attribute("varattrU16Array", data.U16, "varU16") + s.write_attribute("varattrU32Array", data.U32, "varU32") + s.write_attribute("varattrU64Array", data.U64, "varU64") + s.write_attribute("varattrR32Array", data.R32, "varR32") + s.write_attribute("varattrR64Array", data.R64, "varR64") + s.write_attribute("varattrR64Value", data.R64, "varR64") + +comm.Barrier() + +# Reader +data = SmallTestData() + +with Stream("types_np.bp", "r", comm) as fr: + # file only + assert fr.num_steps() == 5 + + for fr_step in fr.steps(): + step = fr_step.current_step() + data.update(rank, step, size) + + step_vars = fr_step.available_variables() + + # for name, info in step_vars.items(): + # print("variable_name: " + name) + # for key, value in info.items(): + # print("\t" + key + ": " + value) + # print("\n") + + if step == 0: + inTag = fr_step.read("tag") + inI8 = fr_step.read("gvarI8") + inI16 = fr_step.read("gvarI16") + inI32 = fr_step.read("gvarI32") + inI64 = fr_step.read("gvarI64") + inU8 = fr_step.read("gvarU8") + inU16 = fr_step.read("gvarU16") + inU32 = fr_step.read("gvarU32") + inU64 = fr_step.read("gvarU64") + inR32 = fr_step.read("gvarR32") + inR64 = fr_step.read("gvarR64") + + if inTag != "Testing ADIOS2 high-level API": + print("InTag: " + str(inTag)) + raise ValueError("tag variable read failed") + + if inI8 != data.I8[0]: + raise ValueError("gvarI8 read failed") + + if inI16 != data.I16[0]: + raise ValueError("gvarI16 read failed") + + if inI32 != data.I32[0]: + raise ValueError("gvarI32 read failed") + + if inI64 != data.I64[0]: + raise ValueError("gvarI64 read failed") + + if inU8 != data.U8[0]: + raise ValueError("gvarU8 read failed") + + if inU16 != data.U16[0]: + raise ValueError("gvarU16 read failed") + + if inU32 != data.U32[0]: + raise ValueError("gvarU32 read failed") + + if inU64 != data.U64[0]: + raise ValueError("gvarU64 read failed") + + if inR32 != data.R32[0]: + raise ValueError("gvarR32 read failed") + + if inR64 != data.R64[0]: + raise ValueError("gvarR64 read failed") + + # attributes + inTag = fr_step.read_attribute("attrStr") + inI8 = fr_step.read_attribute("attrI8") + inI16 = fr_step.read_attribute("attrI16") + inI32 = fr_step.read_attribute("attrI32") + inI64 = fr_step.read_attribute("attrI64") + inU8 = fr_step.read_attribute("attrU8") + inU16 = fr_step.read_attribute("attrU16") + inU32 = fr_step.read_attribute("attrU32") + inU64 = fr_step.read_attribute("attrU64") + inR32 = fr_step.read_attribute("attrR32") + inR64 = fr_step.read_attribute("attrR64") + + if inTag[0] != "Testing single string attribute": + raise ValueError("attr string read failed") + + if inI8[0] != data.I8[0]: + raise ValueError("attrI8 read failed") + + if inI16[0] != data.I16[0]: + raise ValueError("attrI16 read failed") + + if inI32[0] != data.I32[0]: + raise ValueError("attrI32 read failed") + + if inI64[0] != data.I64[0]: + raise ValueError("attrI64 read failed") + + if inU8[0] != data.U8[0]: + raise ValueError("attrU8 read failed") + + if inU16[0] != data.U16[0]: + raise ValueError("attrU16 read failed") + + if inU32[0] != data.U32[0]: + raise ValueError("attrU32 read failed") + + if inU64[0] != data.U64[0]: + raise ValueError("attrU64 read failed") + + if inR32[0] != data.R32[0]: + raise ValueError("attrR32 read failed") + + if inR64[0] != data.R64[0]: + raise ValueError("attrR64 read failed") + + # Array attribute + inTag = fr_step.read_attribute_string("attrStrArray") + inI8 = fr_step.read_attribute("attrI8Array") + inI16 = fr_step.read_attribute("attrI16Array") + inI32 = fr_step.read_attribute("attrI32Array") + inI64 = fr_step.read_attribute("attrI64Array") + inU8 = fr_step.read_attribute("attrU8Array") + inU16 = fr_step.read_attribute("attrU16Array") + inU32 = fr_step.read_attribute("attrU32Array") + inU64 = fr_step.read_attribute("attrU64Array") + inR32 = fr_step.read_attribute("attrR32Array") + inR64 = fr_step.read_attribute("attrR64Array") + + if inTag != ["string1", "string2", "string3"]: + raise ValueError("attrStrArray read failed") + + if (inI8 == data.I8).all() is False: + raise ValueError("attrI8 array read failed") + + if (inI16 == data.I16).all() is False: + raise ValueError("attrI16 array read failed") + + if (inI32 == data.I32).all() is False: + raise ValueError("attrI32 array read failed") + + if (inI64 == data.I64).all() is False: + raise ValueError("attrI64 array read failed") + + if (inU8 == data.U8).all() is False: + raise ValueError("attrU8 array read failed") + + if (inU16 == data.U16).all() is False: + raise ValueError("attrU16 array read failed") + + if (inU32 == data.U32).all() is False: + raise ValueError("attrU32 array read failed") + + if (inU64 == data.U64).all() is False: + raise ValueError("attrU64 array read failed") + + if (inR32 == data.R32).all() is False: + raise ValueError("attrR32 array read failed") + + if (inR64 == data.R64).all() is False: + raise ValueError("attrR64 array read failed") + + inTags = fr_step.read_attribute_string("varattrStrArray", "steps") + inI8 = fr_step.read_attribute("varattrI8Array", "varI8") + in16 = fr_step.read_attribute("varattrI16Array", "varI16") + inI32 = fr_step.read_attribute("varattrI32Array", "varI32") + inI64 = fr_step.read_attribute("varattrI64Array", "varI64") + inU8 = fr_step.read_attribute("varattrU8Array", "varU8") + inU16 = fr_step.read_attribute("varattrU16Array", "varU16") + inU32 = fr_step.read_attribute("varattrU32Array", "varU32") + inU64 = fr_step.read_attribute("varattrU64Array", "varU64") + inR32 = fr_step.read_attribute("varattrR32Array", "varR32") + inR64 = fr_step.read_attribute("varattrR64Array", "varR64") + + if inTags != ["varattr1", "varattr2", "varattr3"]: + print(inTags) + raise ValueError("var attrStrArray read failed") + + if (inI8 == data.I8).all() is False: + raise ValueError("var attrI8 array read failed") + + if (inI16 == data.I16).all() is False: + raise ValueError("var attrI16 array read failed") + + if (inI32 == data.I32).all() is False: + raise ValueError("var attrI32 array read failed") + + if (inI64 == data.I64).all() is False: + raise ValueError("var attrI64 array read failed") + + if (inU8 == data.U8).all() is False: + raise ValueError("var attrU8 array read failed") + + if (inU16 == data.U16).all() is False: + raise ValueError("var attrU16 array read failed") + + if (inU32 == data.U32).all() is False: + raise ValueError("var attrU32 array read failed") + + if (inU64 == data.U64).all() is False: + raise ValueError("var attrU64 array read failed") + + if (inR32 == data.R32).all() is False: + raise ValueError("var attrR32 array read failed") + + if (inR64 == data.R64).all() is False: + raise ValueError("var attrR64 array read failed") + + stepStr = "Step:" + str(step) + + instepStr = fr_step.read("steps") + if instepStr != stepStr: + raise ValueError("steps variable read failed: " + instepStr + " " + stepStr) + + indataRanks = fr_step.read("rank", [0], [size]) + dataRanks = np.arange(0, size) + if (indataRanks == dataRanks).all() is False: + raise ValueError("Ranks read failed") + + indataI8 = fr_step.read("varI8", start, count) + indataI16 = fr_step.read("varI16", start, count) + indataI32 = fr_step.read("varI32", start, count) + indataI64 = fr_step.read("varI64", start, count) + indataU8 = fr_step.read("varU8", start, count) + indataU16 = fr_step.read("varU16", start, count) + indataU32 = fr_step.read("varU32", start, count) + indataU64 = fr_step.read("varU64", start, count) + indataR32 = fr_step.read("varR32", start, count) + indataR64 = fr_step.read("varR64", start, count) + + if (indataI8 == data.I8).all() is False: + raise ValueError("I8 array read failed") + + if (indataI16 == data.I16).all() is False: + raise ValueError("I16 array read failed") + + if (indataI32 == data.I32).all() is False: + raise ValueError("I32 array read failed") + + if (indataI64 == data.I64).all() is False: + raise ValueError("I64 array read failed") + + if (indataU8 == data.U8).all() is False: + raise ValueError("U8 array read failed") + + if (indataU16 == data.U16).all() is False: + raise ValueError("U16 array read failed") + + if (indataU32 == data.U32).all() is False: + raise ValueError("U32 array read failed") + + if (indataU64 == data.U64).all() is False: + raise ValueError("U64 array read failed") + + if (indataR32 == data.R32).all() is False: + raise ValueError("R32 array read failed") + + if (indataR64 == data.R64).all() is False: + raise ValueError("R64 array read failed") diff --git a/testing/adios2/python/TestBPWriteTypesHighLevelAPILocal.py b/testing/adios2/python/TestBPWriteTypesHighLevelAPILocal.py new file mode 100644 index 0000000000..2559c0b07f --- /dev/null +++ b/testing/adios2/python/TestBPWriteTypesHighLevelAPILocal.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python + +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +# +# TestBPWriteTypesHighLevelAPILocal.py: test Python numpy types in ADIOS2 File +# Write/Read High-Level API for Local Arrays +# Created on: March 12, 2018 +# Author: William F Godoy godoywf@ornl.gov + +from adios2NPTypes import SmallTestData +from mpi4py import MPI +from adios2 import Stream + + +def check_array(np1, np2, hint): + if (np1 == np2).all() is False: + print("InData: " + str(np1)) + print("Data: " + str(np2)) + raise ValueError("Array read failed " + str(hint)) + + +comm = MPI.COMM_WORLD +rank = comm.Get_rank() +size = comm.Get_size() + +# Test data +data = SmallTestData() +nx = data.Nx + +shape = [] +start = [] +count = [nx] + +# Writer +with Stream("types_np_local.bp", "w", comm) as s: + for step in s.steps(5): + data.update(rank, step.current_step(), size) + s.write("varI8", data.I8, shape, start, count) + s.write("varI16", data.I16, shape, start, count) + s.write("varI32", data.I32, shape, start, count) + s.write("varI64", data.I64, shape, start, count) + s.write("varU8", data.U8, shape, start, count) + s.write("varU16", data.U16, shape, start, count) + s.write("varU32", data.U32, shape, start, count) + s.write("varU64", data.U64, shape, start, count) + s.write("varR32", data.R32, shape, start, count) + s.write("varR64", data.R64, shape, start, count) + + +# Reader +data = SmallTestData() + +with Stream("types_np_local.bp", "r", comm) as s: + for fr_step in s.steps(): + step = fr_step.current_step() + + for b in range(0, size): + data.update(b, step, size) + + indataI8 = fr_step.read("varI8", b) + indataI16 = fr_step.read("varI16", b) + indataI32 = fr_step.read("varI32", b) + indataI64 = fr_step.read("varI64", b) + indataU8 = fr_step.read("varU8", b) + indataU16 = fr_step.read("varU16", b) + indataU32 = fr_step.read("varU32", b) + indataU64 = fr_step.read("varU64", b) + indataR32 = fr_step.read("varR32", b) + indataR64 = fr_step.read("varR64", b) + + check_array(indataI8, data.I8, "I8") + check_array(indataI16, data.I16, "I16") + check_array(indataI32, data.I32, "I32") + check_array(indataI64, data.I64, "I64") + check_array(indataU8, data.U8, "U8") + check_array(indataU16, data.U16, "U16") + check_array(indataU32, data.U32, "U32") + check_array(indataU64, data.U64, "U64") + check_array(indataR32, data.R32, "R32") + check_array(indataR64, data.R64, "R64") diff --git a/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPI_HDF5.py b/testing/adios2/python/TestBPWriteTypesHighLevelAPI_HDF5.py similarity index 73% rename from testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPI_HDF5.py rename to testing/adios2/python/TestBPWriteTypesHighLevelAPI_HDF5.py index 1ac82a605d..ff370ae740 100644 --- a/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPI_HDF5.py +++ b/testing/adios2/python/TestBPWriteTypesHighLevelAPI_HDF5.py @@ -12,7 +12,7 @@ from adios2NPTypes import SmallTestData from mpi4py import MPI import numpy as np -import adios2 +import adios2.bindings as adios2 comm = MPI.COMM_WORLD rank = comm.Get_rank() @@ -57,8 +57,7 @@ fw.write_attribute("attrR32", np.array(data.R32[0])) fw.write_attribute("attrR64", np.array(data.R64[0])) - fw.write_attribute( - "attrStrArray", ["string1", "string2", "string3"]) + fw.write_attribute("attrStrArray", ["string1", "string2", "string3"]) fw.write_attribute("attrI8Array", data.I8) fw.write_attribute("attrI16Array", data.I16) fw.write_attribute("attrI32Array", data.I32) @@ -83,8 +82,7 @@ fw.write("varR64", data.R64, shape, start, count) if rank == 0 and i == 0: - fw.write_attribute("varattrStrArray", [ - "varattr1", "varattr2", "varattr3"], "steps") + fw.write_attribute("varattrStrArray", ["varattr1", "varattr2", "varattr3"], "steps") fw.write_attribute("varattrI8Array", data.I8, "varI8") fw.write_attribute("varattrI16Array", data.I16, "varI16") fw.write_attribute("varattrI32Array", data.I32, "varI32") @@ -105,19 +103,17 @@ data = SmallTestData() with adios2.open("types_np.h5", "r", comm, "HDF5") as fr: - for fr_step in fr: - step = fr_step.current_step() data.update(rank, step, size) step_vars = fr_step.available_variables() -# for name, info in step_vars.items(): -# print("variable_name: " + name) -# for key, value in info.items(): -# print("\t" + key + ": " + value) -# print("\n") + # for name, info in step_vars.items(): + # print("variable_name: " + name) + # for key, value in info.items(): + # print("\t" + key + ": " + value) + # print("\n") if step == 0: inTag = fr_step.read_string("tag") @@ -134,37 +130,37 @@ if inTag[0] != "Testing ADIOS2 high-level API": print("InTag: " + str(inTag)) - raise ValueError('tag variable read failed') + raise ValueError("tag variable read failed") if inI8 != data.I8[0]: - raise ValueError('gvarI8 read failed') + raise ValueError("gvarI8 read failed") if inI16 != data.I16[0]: - raise ValueError('gvarI16 read failed') + raise ValueError("gvarI16 read failed") if inI32 != data.I32[0]: - raise ValueError('gvarI32 read failed') + raise ValueError("gvarI32 read failed") if inI64 != data.I64[0]: - raise ValueError('gvarI64 read failed') + raise ValueError("gvarI64 read failed") if inU8 != data.U8[0]: - raise ValueError('gvarU8 read failed') + raise ValueError("gvarU8 read failed") if inU16 != data.U16[0]: - raise ValueError('gvarU16 read failed') + raise ValueError("gvarU16 read failed") if inU32 != data.U32[0]: - raise ValueError('gvarU32 read failed') + raise ValueError("gvarU32 read failed") if inU64 != data.U64[0]: - raise ValueError('gvarU64 read failed') + raise ValueError("gvarU64 read failed") if inR32 != data.R32[0]: - raise ValueError('gvarR32 read failed') + raise ValueError("gvarR32 read failed") if inR64 != data.R64[0]: - raise ValueError('gvarR64 read failed') + raise ValueError("gvarR64 read failed") # attributes inTag = fr_step.read_attribute_string("attrStr") @@ -180,37 +176,37 @@ inR64 = fr_step.read_attribute("attrR64") if inTag[0] != "Testing single string attribute": - raise ValueError('attr string read failed') + raise ValueError("attr string read failed") if inI8[0] != data.I8[0]: - raise ValueError('attrI8 read failed') + raise ValueError("attrI8 read failed") if inI16[0] != data.I16[0]: - raise ValueError('attrI16 read failed') + raise ValueError("attrI16 read failed") if inI32[0] != data.I32[0]: - raise ValueError('attrI32 read failed') + raise ValueError("attrI32 read failed") if inI64[0] != data.I64[0]: - raise ValueError('attrI64 read failed') + raise ValueError("attrI64 read failed") if inU8[0] != data.U8[0]: - raise ValueError('attrU8 read failed') + raise ValueError("attrU8 read failed") if inU16[0] != data.U16[0]: - raise ValueError('attrU16 read failed') + raise ValueError("attrU16 read failed") if inU32[0] != data.U32[0]: - raise ValueError('attrU32 read failed') + raise ValueError("attrU32 read failed") if inU64[0] != data.U64[0]: - raise ValueError('attrU64 read failed') + raise ValueError("attrU64 read failed") if inR32[0] != data.R32[0]: - raise ValueError('attrR32 read failed') + raise ValueError("attrR32 read failed") if inR64[0] != data.R64[0]: - raise ValueError('attrR64 read failed') + raise ValueError("attrR64 read failed") # Array attribute inTag = fr_step.read_attribute_string("attrStrArray") @@ -226,37 +222,37 @@ inR64 = fr_step.read_attribute("attrR64Array") if inTag != ["string1", "string2", "string3"]: - raise ValueError('attrStrArray read failed') + raise ValueError("attrStrArray read failed") if (inI8 == data.I8).all() is False: - raise ValueError('attrI8 array read failed') + raise ValueError("attrI8 array read failed") if (inI16 == data.I16).all() is False: - raise ValueError('attrI16 array read failed') + raise ValueError("attrI16 array read failed") if (inI32 == data.I32).all() is False: - raise ValueError('attrI32 array read failed') + raise ValueError("attrI32 array read failed") if (inI64 == data.I64).all() is False: - raise ValueError('attrI64 array read failed') + raise ValueError("attrI64 array read failed") if (inU8 == data.U8).all() is False: - raise ValueError('attrU8 array read failed') + raise ValueError("attrU8 array read failed") if (inU16 == data.U16).all() is False: - raise ValueError('attrU16 array read failed') + raise ValueError("attrU16 array read failed") if (inU32 == data.U32).all() is False: - raise ValueError('attrU32 array read failed') + raise ValueError("attrU32 array read failed") if (inU64 == data.U64).all() is False: - raise ValueError('attrU64 array read failed') + raise ValueError("attrU64 array read failed") if (inR32 == data.R32).all() is False: - raise ValueError('attrR32 array read failed') + raise ValueError("attrR32 array read failed") if (inR64 == data.R64).all() is False: - raise ValueError('attrR64 array read failed') + raise ValueError("attrR64 array read failed") inTags = fr_step.read_attribute_string("varattrStrArray", "steps") inI8 = fr_step.read_attribute("varattrI8Array", "varI8") @@ -272,44 +268,43 @@ if inTags != ["varattr1", "varattr2", "varattr3"]: print(inTags) - raise ValueError('var attrStrArray read failed') + raise ValueError("var attrStrArray read failed") if (inI8 == data.I8).all() is False: - raise ValueError('var attrI8 array read failed') + raise ValueError("var attrI8 array read failed") if (inI16 == data.I16).all() is False: - raise ValueError('var attrI16 array read failed') + raise ValueError("var attrI16 array read failed") if (inI32 == data.I32).all() is False: - raise ValueError('var attrI32 array read failed') + raise ValueError("var attrI32 array read failed") if (inI64 == data.I64).all() is False: - raise ValueError('var attrI64 array read failed') + raise ValueError("var attrI64 array read failed") if (inU8 == data.U8).all() is False: - raise ValueError('var attrU8 array read failed') + raise ValueError("var attrU8 array read failed") if (inU16 == data.U16).all() is False: - raise ValueError('var attrU16 array read failed') + raise ValueError("var attrU16 array read failed") if (inU32 == data.U32).all() is False: - raise ValueError('var attrU32 array read failed') + raise ValueError("var attrU32 array read failed") if (inU64 == data.U64).all() is False: - raise ValueError('var attrU64 array read failed') + raise ValueError("var attrU64 array read failed") if (inR32 == data.R32).all() is False: - raise ValueError('var attrR32 array read failed') + raise ValueError("var attrR32 array read failed") if (inR64 == data.R64).all() is False: - raise ValueError('var attrR64 array read failed') + raise ValueError("var attrR64 array read failed") stepStr = "Step:" + str(step) instepStr = fr_step.read_string("steps") if instepStr[0] != stepStr: - raise ValueError('steps variable read failed: ' + - instepStr + " " + stepStr) + raise ValueError("steps variable read failed: " + instepStr + " " + stepStr) indataI8 = fr_step.read("varI8", start, count) indataI16 = fr_step.read("varI16", start, count) @@ -324,31 +319,31 @@ fr_step.end_step() if (indataI8 == data.I8).all() is False: - raise ValueError('I8 array read failed') + raise ValueError("I8 array read failed") if (indataI16 == data.I16).all() is False: - raise ValueError('I16 array read failed') + raise ValueError("I16 array read failed") if (indataI32 == data.I32).all() is False: - raise ValueError('I32 array read failed') + raise ValueError("I32 array read failed") if (indataI64 == data.I64).all() is False: - raise ValueError('I64 array read failed') + raise ValueError("I64 array read failed") if (indataU8 == data.U8).all() is False: - raise ValueError('U8 array read failed') + raise ValueError("U8 array read failed") if (indataU16 == data.U16).all() is False: - raise ValueError('U16 array read failed') + raise ValueError("U16 array read failed") if (indataU32 == data.U32).all() is False: - raise ValueError('U32 array read failed') + raise ValueError("U32 array read failed") if (indataU64 == data.U64).all() is False: - raise ValueError('U64 array read failed') + raise ValueError("U64 array read failed") if (indataR32 == data.R32).all() is False: - raise ValueError('R32 array read failed') + raise ValueError("R32 array read failed") if (indataR64 == data.R64).all() is False: - raise ValueError('R64 array read failed') + raise ValueError("R64 array read failed") diff --git a/testing/adios2/bindings/python/TestBPZfpHighLevelAPI.py b/testing/adios2/python/TestBPZfpHighLevelAPI.py similarity index 60% rename from testing/adios2/bindings/python/TestBPZfpHighLevelAPI.py rename to testing/adios2/python/TestBPZfpHighLevelAPI.py index d3df52e05c..0ca1600101 100644 --- a/testing/adios2/bindings/python/TestBPZfpHighLevelAPI.py +++ b/testing/adios2/python/TestBPZfpHighLevelAPI.py @@ -11,11 +11,10 @@ import numpy as np from mpi4py import MPI -import adios2 +from adios2 import Stream def CompressZfp2D(rate): - fname = "BPWRZfp2D_" + str(rate) + "_py.bp" Nx = 100 Ny = 50 @@ -25,12 +24,12 @@ def CompressZfp2D(rate): r32s = np.zeros([Ny, Nx], np.float32) r64s = np.zeros([Ny, Nx], np.float64) - value_ji = 0. + value_ji = 0.0 for j in range(0, Ny): for i in range(0, Nx): r32s[j][i] = value_ji r64s[j][i] = value_ji - value_ji += 1. + value_ji += 1.0 # set global dimensions # MPI @@ -43,30 +42,24 @@ def CompressZfp2D(rate): count = [Ny, Nx] # writer - with adios2.open(fname, "w", comm) as fw: - - for s in range(0, NSteps): - fw.write("r32", r32s, shape, start, count, - [('zfp', {'accuracy': str(rate)})]) - fw.write("r64", r64s, shape, start, count, - [('zfp', {'accuracy': str(rate)})], end_step=True) + with Stream(fname, "w", comm) as s: + for _ in s.steps(NSteps): + s.write("r32", r32s, shape, start, count, [("zfp", {"accuracy": str(rate)})]) + s.write("r64", r64s, shape, start, count, [("zfp", {"accuracy": str(rate)})]) # reader - with adios2.open(fname, "r", comm) as fr: - - for fstep in fr: - - in_r32s = fstep.read("r32", start, count) - in_r64s = fstep.read("r64", start, count) + with Stream(fname, "r", comm) as s: + for _ in s.steps(): + in_r32s = s.read("r32", start, count) + in_r64s = s.read("r64", start, count) for j in range(0, Ny): for i in range(0, Nx): - assert (abs(r32s[j][i] - in_r32s[j][i]) < 1E-4) - assert (abs(r64s[j][i] - in_r64s[j][i]) < 1E-4) + assert abs(r32s[j][i] - in_r32s[j][i]) < 1e-4 + assert abs(r64s[j][i] - in_r64s[j][i]) < 1e-4 def main(): - CompressZfp2D(rate=8) CompressZfp2D(rate=9) CompressZfp2D(rate=10) diff --git a/testing/adios2/python/TestEngine.py b/testing/adios2/python/TestEngine.py new file mode 100644 index 0000000000..bd0c6f1e24 --- /dev/null +++ b/testing/adios2/python/TestEngine.py @@ -0,0 +1,95 @@ +from adios2.adios import ADIOS + +import adios2.bindings as bindings + +import unittest +import numpy as np + + +class TestEngine(unittest.TestCase): + def test_close(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + with writer.Open("pythontestengine.bp", bindings.Mode.Write): + pass + + def test_put(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + pressure = writer.DefineVariable("pressure") + temps = writer.DefineVariable("temps", np.empty([4], dtype=np.int64)) + with writer.Open("pythontestengine.bp", bindings.Mode.Write) as engine: + engine.Put(pressure, "35PSI") + temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) + engine.Put(temps, temps_measures) + + def test_get(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + pressure = writer.DefineVariable("pressure") + temps = writer.DefineVariable( + name="temps", + content=np.empty([4], dtype=np.int64), + start=[0], + shape=[4], + count=[4], + ) + with writer.Open("pythontestengine.bp", bindings.Mode.Write) as engine: + engine.Put(pressure, "35PSI") + temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) + engine.Put(temps, temps_measures) + + with adios.DeclareIO("BPReader") as reader: + with reader.Open("pythontestengine.bp", bindings.Mode.Read) as engine: + engine.BeginStep() + pressure = reader.InquireVariable("pressure") + temps = reader.InquireVariable("temps") + pressure_reading = engine.Get(pressure) + temps_reading = np.empty([4], dtype=np.int64) + engine.Get(temps, temps_reading) + engine.EndStep() + self.assertEqual(pressure_reading, "35PSI") + self.assertTrue(np.array_equal(temps_reading, np.array([35, 40, 30, 45]))) + + def test_steps(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + pressure = writer.DefineVariable("pressure") + with writer.Open("pythontestengine.bp", bindings.Mode.Write) as engine: + for step in range(0, 10): + engine.BeginStep() + engine.Put(pressure, f"{step}PSI") + engine.EndStep() + + with adios.DeclareIO("BPReader") as reader: + with reader.Open("pythontestengine.bp", bindings.Mode.Read) as engine: + for i in range(0, engine.Steps()): + engine.BeginStep() + pressure = reader.InquireVariable("pressure") + pressure_reading = engine.Get(pressure) + self.assertEqual(pressure_reading, f"{i}PSI") + engine.EndStep() + + def test_blockinfo(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + temps = writer.DefineVariable("temps", np.empty([4], dtype=np.int64)) + with writer.Open("pythontestengine.bp", bindings.Mode.Write) as engine: + temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) + engine.Put(temps, temps_measures) + + with adios.DeclareIO("BPReader") as reader: + with reader.Open("pythontestengine.bp", bindings.Mode.Read) as engine: + engine.BeginStep() + temps = reader.InquireVariable("temps") + info = engine.BlocksInfo("temps", 0) + engine.EndStep() + + self.assertIsNot(info, None) + self.assertEqual(info[0]["Start"], "") + self.assertEqual(info[0]["Count"], "") + self.assertEqual(info[0]["WriterID"], "0") + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/adios2/python/TestFile.py b/testing/adios2/python/TestFile.py new file mode 100644 index 0000000000..133d63f96d --- /dev/null +++ b/testing/adios2/python/TestFile.py @@ -0,0 +1,21 @@ +from adios2 import Stream +from random import randint + +import unittest + + +class TestStream(unittest.TestCase): + def test_basic(self): + with Stream("pythonfiletest.bp", "w") as f: + for _ in f.steps(10): + f.write("Outlook", "Good") + f.write("temp", [randint(15, 35), randint(15, 35), randint(15, 35)]) + + with Stream("pythonfiletest.bp", "r") as f: + for _ in f.steps(): + for var_name in f.available_variables(): + print(f"var:{var_name} {f.read(var_name)}") + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/adios2/python/TestIO.py b/testing/adios2/python/TestIO.py new file mode 100644 index 0000000000..5e77f5d130 --- /dev/null +++ b/testing/adios2/python/TestIO.py @@ -0,0 +1,102 @@ +from adios2.adios import ADIOS +import adios2.bindings as bindings + +import unittest + + +class TestIO(unittest.TestCase): + def test_io_empty(self): + adios = ADIOS() + adios.DeclareIO("BPWriter") + + def test_io_define_attribute(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + ts = writer.DefineAttribute("timestamp", "20231122") + self.assertIsNot(ts, None) + + def test_io_inquire_attribute(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + ts = writer.DefineAttribute("timestamp", "20231122") + coords = writer.DefineAttribute("coords", "43N74W") + x = writer.InquireAttribute("coords") + self.assertNotEqual(ts, coords) + self.assertNotEqual(ts, x) + self.assertEqual(coords, x) + + def test_available_attribute(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + writer.DefineAttribute("timestamp", "20231122") + writer.InquireAttribute("timestamp") + self.assertIs(writer.InquireAttribute("coords"), None) + + def test_remove_attribute(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + writer.DefineAttribute("timestamp", "20231122") + writer.RemoveAttribute("timestamp") + self.assertIs(writer.InquireAttribute("timestamp"), None) + + def test_remove_all_attribute(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + writer.DefineAttribute("timestamp", "20231122") + writer.DefineAttribute("coords", "43N74W") + writer.RemoveAllAttributes() + self.assertIs(writer.InquireAttribute("timestamp"), None) + self.assertIs(writer.InquireAttribute("coords"), None) + + def test_io_define_variable(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + temp = writer.DefineVariable("temp") + self.assertNotEqual(temp, None) + + def test_io_inquire_variable(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + temp = writer.DefineVariable("temp") + presure = writer.DefineVariable("pressure") + x = writer.InquireVariable("pressure") + self.assertNotEqual(temp, presure) + self.assertNotEqual(temp, x) + self.assertEqual(presure, x) + + def test_available_variable(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + writer.DefineVariable("temp") + writer.InquireVariable("temp") + self.assertIs(writer.InquireAttribute("pressure"), None) + + def test_remove_variable(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + writer.DefineVariable("temp") + writer.RemoveVariable("temp") + self.assertIs(writer.InquireAttribute("temp"), None) + + def test_remove_all_variable(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + writer.DefineVariable("temp") + writer.DefineVariable("pressure") + writer.RemoveAllVariables() + self.assertIs(writer.InquireAttribute("pressure"), None) + self.assertIs(writer.InquireAttribute("temp"), None) + + def test_open_engine(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + writer.SetEngine("BPFile") + writer.SetParameter("threads", "2") + writer.SetParameters({"AsyncOpen": "On", "MaxOpenFilesAtOnce": "512"}) + writer.AddTransport("File", {"Library": "POSIX"}) + engine = writer.Open("pythontest.bp", bindings.Mode.Write) + self.assertNotEqual(engine, None) + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/adios2/python/TestOperator.py b/testing/adios2/python/TestOperator.py new file mode 100644 index 0000000000..4e64592659 --- /dev/null +++ b/testing/adios2/python/TestOperator.py @@ -0,0 +1,38 @@ +from adios2.adios import ADIOS + +import adios2.bindings as bindings + +import unittest +import numpy as np + + +class TestOperator(unittest.TestCase): + def test_operator_basic(self): + adios = ADIOS() + op1 = adios.DefineOperator("noop", "null") + with adios.DeclareIO("BPWriter") as writer: + temps = writer.DefineVariable("temps", np.empty([4], dtype=np.int64)) + temps.AddOperation(op1) + with writer.Open("pythontestvariable.bp", bindings.Mode.Write) as engine: + temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) + engine.Put(temps, temps_measures) + + op2 = adios.DefineOperator("noop2", "null") + with adios.DeclareIO("BPReader") as reader: + with reader.Open("pythontestvariable.bp", bindings.Mode.Read) as engine: + engine.BeginStep() + temps = reader.InquireVariable("temps") + temps.AddOperation(op2) + engine.EndStep() + + def test_operator_params(self): + adios = ADIOS() + op = adios.DefineOperator("noop", "null") + op.SetParameter("speed", "best") + op.GetParameters() + self.assertTrue("speed" in op.GetParameters()) + self.assertEqual(op.GetParameters()["speed"], "best") + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/adios2/python/TestStream.py b/testing/adios2/python/TestStream.py new file mode 100644 index 0000000000..ed7da1eca8 --- /dev/null +++ b/testing/adios2/python/TestStream.py @@ -0,0 +1,50 @@ +from adios2 import Stream, LocalValueDim +from random import randint + +import unittest + + +class TestStream(unittest.TestCase): + def test_basic(self): + with Stream("pythonfiletest.bp", "w") as s: + for _ in s.steps(10): + # Single value string + s.write("Outlook", "Good") + # Global array + s.write( + "Temp", + content=[randint(15, 35), randint(15, 35), randint(15, 35)], + shape=[3, 1], + start=[0, 0], + count=[3, 1], + ) + # Local Value + s.write("Wind", [5], shape=[LocalValueDim]) + # Local Array + s.write("Coords", [38, -46], [], [], [2]) + + with Stream("pythonfiletest.bp", "r") as s: + for _ in s.steps(): + for var_name in s.available_variables(): + print(f"var:{var_name}\t{s.read(var_name)}") + + def test_selection(self): + with Stream("pythonfiletest.bp", "w") as s: + for _ in s.steps(10): + # Global array + s.write( + "Temp", + content=[randint(15, 35), randint(15, 35), randint(15, 35)], + shape=[3, 1], + start=[0, 0], + count=[3, 1], + ) + + with Stream("pythonfiletest.bp", "r") as s: + for _ in s.steps(): + for var_name in s.available_variables(): + print(f"var:{var_name}\t{s.read(var_name)}") + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/adios2/python/TestVariable.py b/testing/adios2/python/TestVariable.py new file mode 100644 index 0000000000..19f0b3b1de --- /dev/null +++ b/testing/adios2/python/TestVariable.py @@ -0,0 +1,66 @@ +from adios2.adios import ADIOS + +import adios2.bindings as bindings + +import unittest +import numpy as np + + +class TestVariable(unittest.TestCase): + def test_create_write(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + temps = writer.DefineVariable("temps", np.empty([4], dtype=np.int64)) + with writer.Open("pythontestvariable.bp", bindings.Mode.Write) as engine: + temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) + engine.Put(temps, temps_measures) + self.assertEqual(temps.Name(), "temps") + self.assertEqual(temps.BlockID(), 0) + self.assertEqual(temps.Count(), []) + self.assertEqual(temps.Shape(), []) + self.assertEqual(temps.Sizeof(), 8) + self.assertEqual(temps.Steps(), 1) + self.assertEqual(temps.StepsStart(), 0) + + def test_create_reader(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + temps = writer.DefineVariable("temps", np.empty([4], dtype=np.int64)) + with writer.Open("pythontestvariable.bp", bindings.Mode.Write) as engine: + temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) + engine.Put(temps, temps_measures) + + with adios.DeclareIO("BPReader") as reader: + with reader.Open("pythontestvariable.bp", bindings.Mode.Read) as engine: + engine.BeginStep() + temps = reader.InquireVariable("temps") + engine.EndStep() + + self.assertEqual(temps.Name(), "temps") + self.assertEqual(temps.BlockID(), 0) + self.assertEqual(temps.Count(), []) + self.assertEqual(temps.Sizeof(), 8) + self.assertEqual(temps.Steps(), 1) + self.assertEqual(temps.StepsStart(), 0) + + def test_operators(self): + adios = ADIOS() + op1 = adios.DefineOperator("noop", "null") + with adios.DeclareIO("BPWriter") as writer: + temps = writer.DefineVariable("temps", np.empty([4], dtype=np.int64)) + temps.AddOperation(op1) + with writer.Open("pythontestvariable.bp", bindings.Mode.Write) as engine: + temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) + engine.Put(temps, temps_measures) + + op2 = adios.DefineOperator("noop2", "null") + with adios.DeclareIO("BPReader") as reader: + with reader.Open("pythontestvariable.bp", bindings.Mode.Read) as engine: + engine.BeginStep() + temps = reader.InquireVariable("temps") + temps.AddOperation(op2) + engine.EndStep() + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/adios2/python/adios2NPTypes.py b/testing/adios2/python/adios2NPTypes.py new file mode 100644 index 0000000000..d35f73e644 --- /dev/null +++ b/testing/adios2/python/adios2NPTypes.py @@ -0,0 +1,88 @@ +# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +# +# nptypes.py small test data for np types +# Created on: Feb 2, 2017 +# Author: William F Godoy godoywf@ornl.gov + +import numpy as np + + +class SmallTestData: + def __init__(self): + self.Nx = 10 + self.Str = "Hello ADIOS2 Python" + self.I8 = np.array([0, 1, -2, 3, -4, 5, -6, 7, -8, 9], dtype=np.int8) + self.I16 = np.array([512, 513, -510, 515, -508, 517, -506, 519, -504, 521], dtype=np.int16) + self.I32 = np.array( + [131072, 131073, -131070, 131075, -131068, 131077, -131066, 131079, -131064, 131081], + dtype=np.int32, + ) + self.I64 = np.array( + [ + 8589934592, + 8589934593, + -8589934590, + 8589934595, + -8589934588, + 8589934597, + -8589934586, + 8589934599, + -8589934584, + 8589934601, + ], + dtype=np.int64, + ) + + self.U8 = np.array([128, 129, 130, 131, 132, 133, 134, 135, 136, 137], dtype=np.uint8) + self.U16 = np.array( + [32768, 32769, 32770, 32771, 32772, 32773, 32774, 32775, 32776, 32777], dtype=np.uint16 + ) + self.U32 = np.array( + [ + 2147483648, + 2147483649, + 2147483650, + 2147483651, + 2147483652, + 2147483653, + 2147483654, + 2147483655, + 2147483656, + 2147483657, + ], + dtype=np.uint32, + ) + self.U64 = np.array( + [ + 9223372036854775808, + 9223372036854775809, + 9223372036854775810, + 9223372036854775811, + 9223372036854775812, + 9223372036854775813, + 9223372036854775814, + 9223372036854775815, + 9223372036854775816, + 9223372036854775817, + ], + dtype=np.uint64, + ) + + self.R32 = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype=np.float32) + self.R64 = np.array([0, -1, -2, -3, -4, -5, -6, -7, -8, -9], dtype=np.float64) + + def update(self, rank, step, size): + self.I8 += 1 + self.I16 += 1 + self.I32 += 1 + self.I64 += 1 + + self.U8 += 1 + self.U16 += 1 + self.U32 += 1 + self.U64 += 1 + + self.R32 += 1 + self.R64 += 1