diff --git a/.github/workflows/pypackaging.yml b/.github/workflows/pypackaging.yml new file mode 100644 index 0000000000..df5d2e17b2 --- /dev/null +++ b/.github/workflows/pypackaging.yml @@ -0,0 +1,100 @@ +name: Python Packaging + +on: + workflow_dispatch: + inputs: + overrideVersion: + description: Manually force a version + pull_request: + push: + branches: + - master + - release_[0-9]+ + tags: + - v[0-9]+.[0-9]+.[0-9]+* + release: + types: + - published + +env: + ADIOS2_CUSTOM_VERSION_OVERRIDE: ${{ github.event.inputs.overrideVersion }} + +jobs: + make_sdist: + name: Make SDist + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Generate common version file + run: cmake -P scripts/ci/gh-actions/config/adios-version.cmake && echo VERSION.TXT + + - name: Build SDist + run: pipx run build --sdist + + - uses: actions/upload-artifact@v3 + with: + path: dist/*.tar.gz + + build_wheels: + name: Wheel on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Generate common version file + run: cmake -P scripts/ci/gh-actions/config/adios-version.cmake && echo VERSION.TXT + + - uses: pypa/cibuildwheel@v2.16 + env: + CIBW_BUILD: cp*-manylinux_x86_64 + + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + path: wheelhouse/*.whl + + upload_pypi: + needs: [build_wheels, make_sdist] + environment: pypi + permissions: + id-token: write + runs-on: ubuntu-latest + if: github.event_name == 'release' && github.event.action == 'published' + steps: + - uses: actions/download-artifact@v3 + with: + name: artifact + path: dist + + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + upload_test_pypi: + needs: [build_wheels, make_sdist] + environment: testpypi + permissions: + id-token: write + runs-on: ubuntu-latest + # Upload to Test PyPI for every commit on main branch + if: github.event_name == 'push' && github.event.ref == 'refs/heads/master' + steps: + - uses: actions/download-artifact@v3 + with: + name: artifact + path: dist + + - name: Publish package distributions to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 64485c3b07..b2ed80530e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,19 @@ project(ADIOS2 VERSION ${ADIOS2_VERSION}) # Some boilerplate to setup nice output directories #------------------------------------------------------------------------------# include(GNUInstallDirs) + +if (ADIOS2_USE_PIP) + # Only UNIX is supported currently + if (UNIX) + # Linux bundles what libraries we have when they're put beside the modules. + set(CMAKE_INSTALL_LIBDIR "adios2") + set(CMAKE_INSTALL_INCLUDEDIR "adios2/include") + # ELF loader settings. + set(CMAKE_BUILD_WITH_INSTALL_RPATH ON) + list(APPEND CMAKE_INSTALL_RPATH "$ORIGIN") + endif () +endif () + set(CMAKE_INSTALL_CMAKEDIR ${CMAKE_INSTALL_LIBDIR}/cmake/adios2 CACHE STRING "Installation CMake subdirectory") mark_as_advanced(CMAKE_INSTALL_CMAKEDIR) @@ -172,6 +185,8 @@ adios_option(Sodium "Enable support for Sodium for encryption" AUTO) adios_option(Catalyst "Enable support for in situ visualization plugin using ParaView Catalyst" AUTO) adios_option(AWSSDK "Enable support for S3 compatible storage using AWS SDK's S3 module" OFF) adios_option(Derived_Variable "Enable support for derived variables" OFF) +adios_option(PIP "Enable support for pip packaging" OFF) +mark_as_advanced(ADIOS2_USE_PIP) include(${PROJECT_SOURCE_DIR}/cmake/DetectOptions.cmake) if(ADIOS2_HAVE_CUDA OR ADIOS2_HAVE_Kokkos_CUDA) @@ -242,7 +257,7 @@ endif() set(ADIOS2_CONFIG_OPTS - DataMan DataSpaces HDF5 HDF5_VOL MHS SST Fortran MPI Python Blosc2 BZip2 + DataMan DataSpaces HDF5 HDF5_VOL MHS SST Fortran MPI Python PIP Blosc2 BZip2 LIBPRESSIO MGARD MGARD_MDR PNG SZ ZFP DAOS IME O_DIRECT Sodium Catalyst SysVShMem UCX ZeroMQ Profiling Endian_Reverse Derived_Variable AWSSDK GPU_Support CUDA Kokkos Kokkos_CUDA Kokkos_HIP Kokkos_SYCL @@ -283,12 +298,12 @@ if(BUILD_SHARED_LIBS AND ADIOS2_RUN_INSTALL_TEST) set(ADIOS2_RUN_INSTALL_TEST FALSE) endif() else() - set(CMAKE_INSTALL_RPATH "@loader_path/${relative_base}/${CMAKE_INSTALL_LIBDIR}") + list(APPEND CMAKE_INSTALL_RPATH "@loader_path/${relative_base}/${CMAKE_INSTALL_LIBDIR}") endif() elseif(CMAKE_SYSTEM_NAME MATCHES "Linux") # Linux needs some specialized RPATH handling - set(CMAKE_INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}") + list(APPEND CMAKE_INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}") endif() endif() @@ -371,8 +386,8 @@ endif() message("") message("ADIOS2 build configuration:") message(" ADIOS Version: ${ADIOS2_VERSION}") -message(" C++ Compiler : ${CMAKE_CXX_COMPILER_ID} " - "${CMAKE_CXX_COMPILER_VERSION} " +message(" C++ Compiler : ${CMAKE_CXX_COMPILER_ID} " + "${CMAKE_CXX_COMPILER_VERSION} " "${CMAKE_CXX_COMPILER_WRAPPER}") message(" ${CMAKE_CXX_COMPILER}") message("") diff --git a/ReadMe.md b/ReadMe.md index 9e73588a36..3ff8b526b0 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -4,6 +4,7 @@ [![GitHub (pre-)release](https://img.shields.io/github/release/ornladios/adios2/all.svg)]() [![Spack Version](https://img.shields.io/spack/v/adios2.svg)](https://spack.readthedocs.io/en/latest/package_list.html#adios2) [![Conda Version](https://img.shields.io/conda/vn/conda-forge/adios2)](https://anaconda.org/conda-forge/adios2) +[![PyPI version](https://badge.fury.io/py/adios2.svg)](https://badge.fury.io/py/adios2) [![Circle CI](https://circleci.com/gh/ornladios/ADIOS2.svg?style=shield)](https://circleci.com/gh/ornladios/ADIOS2) @@ -23,7 +24,7 @@ ADIOS2 can be used on supercomputers, cloud systems, and personal computers. ADIOS2 focuses on: 1. **Performance** I/O scalability in high performance computing (HPC) applications. -2. **Adaptability** unified interfaces to allow for several modes of transport (files, memory-to-memory) +2. **Adaptability** unified interfaces to allow for several modes of transport (files, memory-to-memory) 3. **Ease of Use** two-level application programming interface (APIs) * Full APIs for HPC applications: C++11, Fortran 90, C 99, Python 2 and 3 * Simplified High-Level APIs for data analysis: Python 2 and 3, C++11, Matlab @@ -63,7 +64,7 @@ For a `cmake` configuration example see [scripts/runconf/runconf.sh](https://git * Docker images: under [scripts/docker](https://github.com/ornladios/ADIOS2/tree/master/scripts/docker) -Once ADIOS2 is installed refer to: +Once ADIOS2 is installed refer to: * [Linking ADIOS2](https://adios2.readthedocs.io/en/latest/setting_up/setting_up.html#linking-adios-2) @@ -78,7 +79,7 @@ Once ADIOS2 is installed refer to: ADIOS2 is an open source project: Questions, discussion, and contributions are welcome. Join us at: -- Mailing list: adios-ecp@kitware.com +- Mailing list: adios-ecp@kitware.com - Github Discussions: https://github.com/ornladios/ADIOS2/discussions ## Reporting Bugs @@ -103,7 +104,7 @@ See the accompanying [Copyright.txt](Copyright.txt) for more details. * scripts - Project maintenance and development scripts -* source - Internal source code for private components +* source - Internal source code for private components * adios2 - source directory for the ADIOS2 library to be installed under install-dir/lib/libadios2. * utils - source directory for the binary utilities, to be installed under install-dir/bin diff --git a/bindings/Python/CMakeLists.txt b/bindings/Python/CMakeLists.txt index 5c6121e8f6..404b084a37 100644 --- a/bindings/Python/CMakeLists.txt +++ b/bindings/Python/CMakeLists.txt @@ -49,16 +49,27 @@ set_target_properties(adios2_py PROPERTIES string(REGEX REPLACE "[^/]+" ".." relative_base "${CMAKE_INSTALL_PYTHONDIR}/adios2") if(CMAKE_SYSTEM_NAME MATCHES "Linux") - set_target_properties(adios2_py PROPERTIES - INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}" - ) + if (NOT ADIOS2_USE_PIP) + set_target_properties(adios2_py PROPERTIES + INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}" + ) + endif() +endif() + +set(install_location ${CMAKE_INSTALL_PYTHONDIR}) +if (ADIOS2_USE_PIP) + set(install_location ${CMAKE_INSTALL_LIBDIR}) endif() install(TARGETS adios2_py - DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2 + DESTINATION ${install_location} COMPONENT adios2_python-python ) install(FILES ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/__init__.py - DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2 + DESTINATION ${install_location} + COMPONENT adios2_python-python +) +install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/test + DESTINATION ${install_location} COMPONENT adios2_python-python ) diff --git a/bindings/Python/test/__init__.py b/bindings/Python/test/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bindings/Python/test/simple_read_write.py b/bindings/Python/test/simple_read_write.py new file mode 100644 index 0000000000..2f3760eb4c --- /dev/null +++ b/bindings/Python/test/simple_read_write.py @@ -0,0 +1,46 @@ +import adios2 + +import sys +import unittest + + +DATA_FILENAME = "hello-world-py.bp" + +class TestSimpleReadWrite(unittest.TestCase): + + def _write(self, ad, greeting): + """write a string to a bp file""" + io = ad.DeclareIO("hello-world-writer") + var_greeting = io.DefineVariable("Greeting") + w = io.Open(DATA_FILENAME, adios2.Mode.Write) + w.BeginStep() + w.Put(var_greeting, greeting) + w.EndStep() + w.Close() + return 0 + + def _read(self, ad): + """read a string from to a bp file""" + io = ad.DeclareIO("hello-world-reader") + r = io.Open(DATA_FILENAME, adios2.Mode.Read) + r.BeginStep() + var_greeting = io.InquireVariable("Greeting") + message = r.Get(var_greeting) + r.EndStep() + r.Close() + return message + + def test_simple_read_write(self): + """driver function""" + print("ADIOS2 version {0}".format(adios2.__version__)) + ad = adios2.ADIOS() + greeting = "Hello World from ADIOS2" + self._write(ad, greeting) + message = self._read(ad) + print("{}".format(message)) + self.assertEqual(greeting, message) + return 0 + + +if __name__ == '__main__': + unittest.main() diff --git a/cmake/ADIOSFunctions.cmake b/cmake/ADIOSFunctions.cmake index 71d6ab994c..3f8fa8a8e5 100644 --- a/cmake/ADIOSFunctions.cmake +++ b/cmake/ADIOSFunctions.cmake @@ -21,14 +21,19 @@ function(setup_version BASE) if(res EQUAL 0 AND out MATCHES "[^-]*-([^-]*)-g([a-f0-9]*)") set(ver_tweak ${CMAKE_MATCH_1}) set(ver_gitsha ${CMAKE_MATCH_2}) - endif() + endif() endif() endif() - if(ver_tweak) - set(ADIOS2_VERSION ${BASE}.${ver_tweak} PARENT_SCOPE) + if(ADIOS2_USE_PIP AND EXISTS "${CMAKE_SOURCE_DIR}/VERSION.TXT") + file(READ "VERSION.TXT" version_from_file) + set(ADIOS2_VERSION ${version_from_file} PARENT_SCOPE) else() - set(ADIOS2_VERSION ${BASE} PARENT_SCOPE) + if(ver_tweak) + set(ADIOS2_VERSION ${BASE}.${ver_tweak} PARENT_SCOPE) + else() + set(ADIOS2_VERSION ${BASE} PARENT_SCOPE) + endif() endif() if(ver_gitsha) diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index 4dd7f5522a..dccfa38b33 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -399,7 +399,10 @@ if(NOT SHARED_LIBS_SUPPORTED) endif() endif() -if(ADIOS2_USE_Python STREQUAL AUTO) +if(ADIOS2_USE_PIP) + find_package(Python 3 REQUIRED COMPONENTS Interpreter Development.Module NumPy) + set(ADIOS2_HAVE_PIP TRUE) +elseif(ADIOS2_USE_Python STREQUAL AUTO) find_package(Python 3 COMPONENTS Interpreter Development NumPy) if(Python_FOUND AND ADIOS2_HAVE_MPI) find_package(PythonModule COMPONENTS mpi4py mpi4py/mpi4py.h) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..5b39654fb7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,65 @@ +[build-system] +requires = ["scikit-build-core", "numpy", "setuptools_scm>=8"] +build-backend = "scikit_build_core.build" + +[project] +name = "adios2" +dynamic = ["version"] +authors = [ + { name="Caitlin Ross", email="caitlin.ross@kitware.com" }, + { name="Chuck Atkins", email="chuck.atkins@kitware.com" }, + { name="Greg S. Eisenhauer", email="eisen@cc.gatech.edu" }, + { name="Junmin Gu", email="jgu@lbl.gov" }, + { name="Norbert Podhorszki", email="pnorbert@ornl.gov" }, + { name="Ruonan (Jason) Wang", email="wangr1@ornl.gov" }, + { name="Scott Wittenburg", email="scott.wittenburg@kitware.com" }, + { name="Spiros Tsalikis", email="spiros.tsalikis@kitware.com" }, + { name="V. A. Bolea Sanchez", email="vicente.bolea@kitware.com" }, + { name="William F. Godoy", email="godoywf@ornl.gov" }, +] +description = "The Adaptable Input Output System version 2" +readme = "ReadMe.md" +requires-python = ">=3.8" +classifiers = [ + "Programming Language :: C++", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", +] +dependencies = [ + "numpy", +] + +[project.urls] +Homepage = "https://github.com/ornladios/adios2" +Documentation = "https://adios2.readthedocs.io/" +"Bug Tracker" = "https://github.com/ornladios/adios2/issues" +Discussions = "https://github.com/ornladios/ADIOS2/discussions" +Changelog = "https://github.com/ornladios/ADIOS2/releases" + +[tool.cibuildwheel] +# Trigger an install of the package, and run a basic test +test-command = "python -m unittest adios2.test.simple_read_write.TestSimpleReadWrite" + +[tool.scikit-build] +wheel.packages = ["adios2"] + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "VERSION.TXT" +regex = "^(?P.+?)$" + +[tool.scikit-build.cmake.define] +ADIOS2_USE_PIP = "ON" +ADIOS2_USE_Python = "ON" +ADIOS2_USE_Fortran = "OFF" +ADIOS2_USE_MPI = "OFF" +ADIOS2_USE_HDF5 = "OFF" +ADIOS2_USE_HDF5_VOL = "OFF" +ADIOS2_USE_BZip2 = "OFF" +ADIOS2_USE_Blosc2 = "OFF" +ADIOS2_USE_DataMan = "OFF" +ADIOS2_USE_SZ = "OFF" +ADIOS2_USE_ZeroMQ = "OFF" +ADIOS2_USE_ZFP = "OFF" +BUILD_TESTING = "OFF" +ADIOS2_INSTALL_GENERATE_CONFIG = "OFF" diff --git a/scripts/ci/gh-actions/config/adios-version.cmake b/scripts/ci/gh-actions/config/adios-version.cmake new file mode 100644 index 0000000000..d3f4e76852 --- /dev/null +++ b/scripts/ci/gh-actions/config/adios-version.cmake @@ -0,0 +1,49 @@ +cmake_minimum_required(VERSION 3.12) + +# Write a file containing the ADIOS2 version to use in pip packaging. This is +# how we work around the issue that scikit-build-core does not yet have a way +# to get the package version from CMake. +# +# There are several issues on scikit-build-core similar to these: +# +# - https://github.com/scikit-build/scikit-build-core/issues/172 +# - https://github.com/scikit-build/scikit-build-core/issues/176 +# +# Once there is a general solution for that, we can dispense with +# invoking this to write a custom version file to be consumed both +# by ADIOSFunctions.cmake and pyproject.toml. + +set(adios2_pip_package_version "UNKNOWN") + +if (DEFINED ENV{ADIOS2_CUSTOM_VERSION_OVERRIDE} AND NOT "$ENV{ADIOS2_CUSTOM_VERSION_OVERRIDE}" STREQUAL "") + # Use the override version from env var + set(adios2_pip_package_version $ENV{ADIOS2_CUSTOM_VERSION_OVERRIDE}) +else() + # Compute the version using git describe + if(NOT GIT_COMMAND) + find_program(GIT_COMMAND git) + endif() + if(GIT_COMMAND) + execute_process( + COMMAND git describe + RESULT_VARIABLE res + OUTPUT_VARIABLE out + ERROR_QUIET + ) + if(res EQUAL 0) + if (out MATCHES "^v([^-]*)$") + set(ver_tag ${CMAKE_MATCH_1}) + set(adios2_pip_package_version "${ver_tag}") + elseif (out MATCHES "^v([^-]*)-([^-]*)-g[a-f0-9]*") + set(ver_tag ${CMAKE_MATCH_1}) + set(ver_ncommits ${CMAKE_MATCH_2}) + set(adios2_pip_package_version "${ver_tag}.${ver_ncommits}") + endif() + endif() + if (NOT res EQUAL 0 OR out MATCHES "fatal: not a git repository") + message(FATAL_ERROR "Must be in a git repository to run this script") + endif() + endif() +endif() + +file(WRITE "VERSION.TXT" ${adios2_pip_package_version}) diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 46e5e3e6d8..82753d45ce 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -4,7 +4,10 @@ #------------------------------------------------------------------------------# add_subdirectory(adios2) -add_subdirectory(utils) + +if (NOT ADIOS2_USE_PIP) + add_subdirectory(utils) +endif() if(ADIOS2_HAVE_HDF5_VOL) add_subdirectory(h5vol) diff --git a/source/adios2/toolkit/remote/CMakeLists.txt b/source/adios2/toolkit/remote/CMakeLists.txt index b3b49d6159..9b613d4b45 100644 --- a/source/adios2/toolkit/remote/CMakeLists.txt +++ b/source/adios2/toolkit/remote/CMakeLists.txt @@ -3,19 +3,21 @@ # accompanying file Copyright.txt for details. #------------------------------------------------------------------------------# -add_executable(remote_server ./remote_server.cpp remote_common.cpp) +if (NOT ADIOS2_USE_PIP) + add_executable(remote_server ./remote_server.cpp remote_common.cpp) -target_link_libraries(remote_server PUBLIC EVPath::EVPath adios2_core adios2sys - PRIVATE $<$:shlwapi>) + target_link_libraries(remote_server PUBLIC EVPath::EVPath adios2_core adios2sys + PRIVATE $<$:shlwapi>) -get_property(pugixml_headers_path - TARGET pugixml - PROPERTY INTERFACE_INCLUDE_DIRECTORIES -) + get_property(pugixml_headers_path + TARGET pugixml + PROPERTY INTERFACE_INCLUDE_DIRECTORIES + ) -target_include_directories(remote_server PRIVATE ${PROJECT_BINARY_DIR} ${pugixml_headers_path}) + target_include_directories(remote_server PRIVATE ${PROJECT_BINARY_DIR} ${pugixml_headers_path}) -set_property(TARGET remote_server PROPERTY OUTPUT_NAME remote_server${ADIOS2_EXECUTABLE_SUFFIX}) -install(TARGETS remote_server EXPORT adios2 - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT adios2_tools-runtime -) + set_property(TARGET remote_server PROPERTY OUTPUT_NAME remote_server${ADIOS2_EXECUTABLE_SUFFIX}) + install(TARGETS remote_server EXPORT adios2 + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT adios2_tools-runtime + ) +endif () diff --git a/source/adios2/toolkit/sst/CMakeLists.txt b/source/adios2/toolkit/sst/CMakeLists.txt index c77fc0459b..3c539b16a9 100644 --- a/source/adios2/toolkit/sst/CMakeLists.txt +++ b/source/adios2/toolkit/sst/CMakeLists.txt @@ -104,4 +104,6 @@ if(CMAKE_VERSION VERSION_LESS 3.14) endforeach() endif() -add_subdirectory(util) +if (NOT ADIOS2_USE_PIP) + add_subdirectory(util) +endif () diff --git a/thirdparty/EVPath/CMakeLists.txt b/thirdparty/EVPath/CMakeLists.txt index 42de1109e6..2be480f49f 100644 --- a/thirdparty/EVPath/CMakeLists.txt +++ b/thirdparty/EVPath/CMakeLists.txt @@ -17,6 +17,12 @@ set(EVPATH_LIBRARY_COMPONENT adios2_evpath-libraries) set(EVPATH_ARCHIVE_COMPONENT adios2_evpath-development) set(EVPATH_HEADER_COMPONENT adios2_evpath-development) +if (ADIOS2_USE_PIP) + # If these are not correct for python packaging, auditwheel will fail + string(REGEX REPLACE "[^/]+" ".." relative_base "${EVPATH_INSTALL_MODULE_DIR}") + list(APPEND CMAKE_INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}") +endif () + add_subdirectory(EVPath) set(EVPath_DIR ${CMAKE_CURRENT_BINARY_DIR}/EVPath CACHE INTERNAL "") setup_libversion_dir(EVPath)