From 6bdd31d130005ba9beb19d28bde1af2f2102cbd4 Mon Sep 17 00:00:00 2001 From: Alex Nitz Date: Tue, 30 Jul 2024 02:47:18 +0100 Subject: [PATCH] add python 3.12 to the build and drop python 3.8 (#4716) * drop python3.7, 3.8, 3.9 * add missing quotes * banish distutils * banish p39 in more places * tox needs some things early * move here * maybe? * try el9 * must use 311 for el9 * this might be the right name * updates * try this * more stuff * update * do not use imp module * more places where imp was used * bring back 39 * more reverts * update * update * fixes * this should be here * cc * fix web url * add back 3.9 to basic tests * centralize python12 backwards compatibility function * cc * simplify setup.py, ensure math is linked * cc --- .github/workflows/basic-tests.yml | 2 +- .github/workflows/distribution.yml | 8 +- .github/workflows/inference-workflow.yml | 2 +- .github/workflows/mac-test.yml | 3 +- .github/workflows/search-workflow.yml | 2 +- .github/workflows/tmpltbank-workflow.yml | 2 +- .github/workflows/tut-test.yml | 2 +- .github/workflows/workflow-tests.yml | 2 +- .../pycbc_cut_merge_triggers_to_tmpltbank | 4 +- bin/all_sky_search/pycbc_reduce_template_bank | 4 +- bin/pycbc_banksim_match_combine | 5 +- pycbc/__init__.py | 15 +++ pyproject.toml | 14 +-- requirements-igwn.txt | 1 + setup.py | 103 ++++-------------- tools/docker_build_dist.sh | 1 - tox.ini | 5 +- 17 files changed, 62 insertions(+), 113 deletions(-) diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index d962026e09a..55e9c24ee9a 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -13,7 +13,7 @@ jobs: max-parallel: 60 matrix: os: [ubuntu-20.04] - python-version: [3.8, 3.9, '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11', '3.12'] test-type: [unittest, search, docs] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/distribution.yml b/.github/workflows/distribution.yml index 122f01ae03d..1b54b501a3d 100644 --- a/.github/workflows/distribution.yml +++ b/.github/workflows/distribution.yml @@ -20,13 +20,13 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: '3.10' - name: Install cibuildwheel run: python -m pip install cibuildwheel - name: Build wheels run: python -m cibuildwheel --output-dir wheelhouse env: - CIBW_BUILD: cp38-* cp39-* cp310-* cp311-* + CIBW_BUILD: cp39-* cp310-* cp311-* cp312-* CIBW_SKIP: "*musllinux*" CIBW_ARCHS_MACOS: x86_64 arm64 - uses: actions/upload-artifact@v2 @@ -40,10 +40,10 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Python 3.8 + - name: Set up Python 3.10 uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: '3.10' - uses: actions/download-artifact@v2 with: path: ./ diff --git a/.github/workflows/inference-workflow.yml b/.github/workflows/inference-workflow.yml index d95e1c5b714..d31076da5e6 100644 --- a/.github/workflows/inference-workflow.yml +++ b/.github/workflows/inference-workflow.yml @@ -10,7 +10,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: '3.10' - name: install condor run: | wget -qO - https://research.cs.wisc.edu/htcondor/ubuntu/HTCondor-Release.gpg.key | sudo apt-key add - diff --git a/.github/workflows/mac-test.yml b/.github/workflows/mac-test.yml index f441f648d5c..7501eb9d9b5 100644 --- a/.github/workflows/mac-test.yml +++ b/.github/workflows/mac-test.yml @@ -13,7 +13,7 @@ jobs: max-parallel: 4 matrix: os: [macos-12] - python-version: [3.8, 3.9, '3.10', '3.11'] + python-version: ['3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v1 - name: Set up Python ${{ matrix.python-version }} @@ -25,5 +25,4 @@ jobs: pip install --upgrade pip setuptools "tox<4.0.0" - name: run basic pycbc test suite run: | - sudo chmod -R 777 /usr/local/miniconda/ tox -e py-unittest diff --git a/.github/workflows/search-workflow.yml b/.github/workflows/search-workflow.yml index 9383c5066cb..f7f8c744ad5 100644 --- a/.github/workflows/search-workflow.yml +++ b/.github/workflows/search-workflow.yml @@ -15,7 +15,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: '3.10' - name: install condor run: | wget -qO - https://research.cs.wisc.edu/htcondor/ubuntu/HTCondor-Release.gpg.key | sudo apt-key add - diff --git a/.github/workflows/tmpltbank-workflow.yml b/.github/workflows/tmpltbank-workflow.yml index 66a4f1ba3b2..e971e97b2b7 100644 --- a/.github/workflows/tmpltbank-workflow.yml +++ b/.github/workflows/tmpltbank-workflow.yml @@ -14,7 +14,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: '3.10' - name: install condor run: | wget -qO - https://research.cs.wisc.edu/htcondor/ubuntu/HTCondor-Release.gpg.key | sudo apt-key add - diff --git a/.github/workflows/tut-test.yml b/.github/workflows/tut-test.yml index 5abd59c9d75..11928d632f6 100644 --- a/.github/workflows/tut-test.yml +++ b/.github/workflows/tut-test.yml @@ -13,7 +13,7 @@ jobs: max-parallel: 60 matrix: os: [ubuntu-20.04] - python-version: [3.8, 3.9, '3.10', '3.11'] + python-version: ['3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/workflow-tests.yml b/.github/workflows/workflow-tests.yml index 86410066aff..79643cc8793 100644 --- a/.github/workflows/workflow-tests.yml +++ b/.github/workflows/workflow-tests.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: '3.10' - name: install condor run: | wget -qO - https://research.cs.wisc.edu/htcondor/ubuntu/HTCondor-Release.gpg.key | sudo apt-key add - diff --git a/bin/all_sky_search/pycbc_cut_merge_triggers_to_tmpltbank b/bin/all_sky_search/pycbc_cut_merge_triggers_to_tmpltbank index 0848aa949c2..fafbb03bf12 100644 --- a/bin/all_sky_search/pycbc_cut_merge_triggers_to_tmpltbank +++ b/bin/all_sky_search/pycbc_cut_merge_triggers_to_tmpltbank @@ -21,12 +21,12 @@ Reduce a MERGE triggers file to a reduced template bank """ import logging -import imp import argparse import numpy import h5py import pycbc from pycbc.io import HFile +from pycbc import load_source parser = argparse.ArgumentParser(description=__doc__) pycbc.add_common_pycbc_options(parser) @@ -50,7 +50,7 @@ pycbc.init_logging(opt.verbose) bank_fd = HFile(opt.full_template_bank, 'r') -modl = imp.load_source('filter_func', opt.filter_func_file) +modl = load_source('filter_func', opt.filter_func_file) func = modl.filter_tmpltbank bool_arr = func(bank_fd) diff --git a/bin/all_sky_search/pycbc_reduce_template_bank b/bin/all_sky_search/pycbc_reduce_template_bank index 9345ac95dc1..c8010edb36f 100644 --- a/bin/all_sky_search/pycbc_reduce_template_bank +++ b/bin/all_sky_search/pycbc_reduce_template_bank @@ -23,10 +23,10 @@ Reduce a template bank using some input parameter cuts import numpy import logging -import imp import argparse import pycbc from pycbc.io import HFile +from pycbc import load_source parser = argparse.ArgumentParser(description=__doc__) pycbc.add_common_pycbc_options(parser) @@ -48,7 +48,7 @@ pycbc.init_logging(opt.verbose) bank_fd = HFile(opt.input_bank, 'r') -modl = imp.load_source('filter_func', opt.filter_func_file) +modl = load_source('filter_func', opt.filter_func_file) func = modl.filter_tmpltbank bool_arr = func(bank_fd) diff --git a/bin/pycbc_banksim_match_combine b/bin/pycbc_banksim_match_combine index 111329e8e7e..652fd0b8813 100644 --- a/bin/pycbc_banksim_match_combine +++ b/bin/pycbc_banksim_match_combine @@ -22,7 +22,6 @@ a set of injection files. The *order* of the injection files *must* match the bank files, and the number of injections in each must correspond one-to-one. """ -import imp import argparse import numpy as np @@ -33,7 +32,7 @@ from pycbc import pnutils from pycbc.waveform import TemplateBank from pycbc.io.ligolw import LIGOLWContentHandler from pycbc.io.hdf import HFile - +from pycbc import load_source __author__ = "Ian Harry " __version__ = pycbc.version.git_verbose_msg @@ -137,7 +136,7 @@ for val in trig_par_list: f['trig_params/{}'.format(val)] = trig_params[val] if options.filter_func_file: - modl = imp.load_source('filter_func', options.filter_func_file) + modl = load_source('filter_func', options.filter_func_file) func = modl.filter_injections bool_arr = func(inj_params['mass1'], inj_params['mass2'], inj_params['spin1z'], inj_params['spin2z']) diff --git a/pycbc/__init__.py b/pycbc/__init__.py index 02d8b95e961..d6db0026e0a 100644 --- a/pycbc/__init__.py +++ b/pycbc/__init__.py @@ -32,6 +32,8 @@ import logging import random import string +import importlib.util +import importlib.machinery from datetime import datetime as dt try: @@ -227,3 +229,16 @@ def gps_now(): from astropy.time import Time return float(Time.now().gps) + +# This is needed as a backwards compatibility. The function was removed in +# python 3.12. +def load_source(modname, filename): + loader = importlib.machinery.SourceFileLoader(modname, filename) + spec = importlib.util.spec_from_file_location(modname, filename, + loader=loader) + module = importlib.util.module_from_spec(spec) + # The module is always executed and not cached in sys.modules. + # Uncomment the following line to cache the module. + # sys.modules[module.__name__] = module + loader.exec_module(module) + return module diff --git a/pyproject.toml b/pyproject.toml index af1b3aa113d..1fcf4c3ca7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,16 +1,6 @@ [build-system] requires = ["setuptools", "wheel", - "cython>=3.0.0", - "numpy==1.16.0; python_version <= '3.7'", - "numpy==1.17.3; python_version == '3.8'", - "numpy==1.19.3; python_version == '3.9'", - "numpy==1.21.4; python_version =='3.10'", - "numpy==1.23.4; python_version=='3.11'", - "numpy; python_version >= '3.12'", + "cython>=0.29.21", + "numpy>=2.0.0", ] - -# To ensure the best compatibility, try to match the numpy reqs -# where possible to the following used by scipy and the minimum -# of our normal numpy requirements in setup.py -# https://github.com/scipy/oldest-supported-numpy/blob/master/setup.cfg diff --git a/requirements-igwn.txt b/requirements-igwn.txt index 9486aa71255..f6199d03283 100644 --- a/requirements-igwn.txt +++ b/requirements-igwn.txt @@ -1,4 +1,5 @@ # For LDG service access +ligo-proxy-utils ciecplib[kerberos] >= 0.7.0 dqsegdb2 >= 1.1.4 amqplib diff --git a/setup.py b/setup.py index d5f769271c9..f39558046d5 100755 --- a/setup.py +++ b/setup.py @@ -20,11 +20,9 @@ """ import sys -import os, subprocess, shutil +import os, subprocess import platform -from distutils.command.clean import clean as _clean - from setuptools import Extension, setup, Command from setuptools.command.build_ext import build_ext as _build_ext from setuptools import find_packages @@ -83,27 +81,6 @@ def run(self): _build_ext.run(self) - -# Add swig-generated files to the list of things to clean, so they -# get regenerated each time. -class clean(_clean): - def finalize_options (self): - _clean.finalize_options(self) - self.clean_files = [] - self.clean_folders = ['docs/_build'] - def run(self): - _clean.run(self) - for f in self.clean_files: - try: - os.unlink(f) - print('removed ' + f) - except: - pass - - for fol in self.clean_folders: - shutil.rmtree(fol, ignore_errors=True) - print('removed ' + fol) - def get_version_info(): """Get VCS info and write version info to version.py. """ @@ -192,7 +169,6 @@ def run(self): cmdclass = { 'build_docs': build_docs, 'build_gh_pages': build_gh_pages, - 'clean': clean, 'build_ext': cbuild_ext } @@ -209,12 +185,21 @@ def run(self): # do the actual work of building the package VERSION = get_version_info() -cythonext = ['waveform.spa_tmplt', - 'waveform.utils', - 'types.array', - 'filter.matchedfilter', - 'vetoes.chisq'] +cythonext = ['pycbc.waveform.spa_tmplt_cpu', + 'pycbc.waveform.utils_cpu', + 'pycbc.types.array_cpu', + 'pycbc.filter.matchedfilter_cpu', + 'pycbc.vetoes.chisq_cpu', + "pycbc.fft.fftw_pruned_cython", + "pycbc.events.eventmgr_cython", + "pycbc.events.simd_threshold_cython", + "pycbc.filter.simd_correlate_cython", + "pycbc.waveform.decompress_cpu_cython", + "pycbc.inference.models.relbin_cpu", + ] ext = [] + +libraries = ['m'] # Some platforms / toolchains don't implicitly link this cython_compile_args = ['-O3', '-w', '-ffast-math', '-ffinite-math-only'] @@ -233,57 +218,18 @@ def run(self): cython_compile_args += ["-stdlib=libc++"] cython_link_args += ["-stdlib=libc++"] + for name in cythonext: - e = Extension("pycbc.%s_cpu" % name, - ["pycbc/%s_cpu.pyx" % name.replace('.', '/')], + fname = name.replace('.', '/') + e = Extension(name, + [f"{fname}.pyx"], + language='c++', extra_compile_args=cython_compile_args, extra_link_args=cython_link_args, + libraries=libraries, compiler_directives={'embedsignature': True}) ext.append(e) -# Not all modules work like this: -e = Extension("pycbc.fft.fftw_pruned_cython", - ["pycbc/fft/fftw_pruned_cython.pyx"], - extra_compile_args=cython_compile_args, - extra_link_args=cython_link_args, - compiler_directives={'embedsignature': True}) -ext.append(e) -e = Extension("pycbc.events.eventmgr_cython", - ["pycbc/events/eventmgr_cython.pyx"], - extra_compile_args=cython_compile_args, - extra_link_args=cython_link_args, - compiler_directives={'embedsignature': True}) -ext.append(e) -e = Extension("pycbc.events.simd_threshold_cython", - ["pycbc/events/simd_threshold_cython.pyx"], - language='c++', - extra_compile_args=cython_compile_args, - extra_link_args=cython_link_args, - compiler_directives={'embedsignature': True}) -ext.append(e) -e = Extension("pycbc.filter.simd_correlate_cython", - ["pycbc/filter/simd_correlate_cython.pyx"], - language='c++', - extra_compile_args=cython_compile_args, - extra_link_args=cython_link_args, - compiler_directives={'embedsignature': True}) -ext.append(e) -e = Extension("pycbc.waveform.decompress_cpu_cython", - ["pycbc/waveform/decompress_cpu_cython.pyx"], - language='c++', - extra_compile_args=cython_compile_args, - extra_link_args=cython_link_args, - compiler_directives={'embedsignature': True}) -ext.append(e) -e = Extension("pycbc.inference.models.relbin_cpu", - ["pycbc/inference/models/relbin_cpu.pyx"], - language='c++', - extra_compile_args=cython_compile_args, - extra_link_args=cython_link_args, - compiler_directives={'embedsignature': True}) -ext.append(e) - - setup( name = 'PyCBC', version = VERSION, @@ -292,7 +238,7 @@ def run(self): long_description_content_type='text/markdown', author = 'The PyCBC team', author_email = 'alex.nitz@gmail.org', - url = 'http://www.pycbc.org/', + url = 'http://pycbc.org/', download_url = f'https://github.com/gwastro/pycbc/tarball/v{VERSION}', keywords = [ 'ligo', @@ -313,14 +259,13 @@ def run(self): 'pycbc.neutron_stars': find_files('pycbc/neutron_stars') }, ext_modules = ext, - python_requires='>=3.7', + python_requires='>=3.9', classifiers=[ 'Programming Language :: Python', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Intended Audience :: Science/Research', 'Natural Language :: English', 'Topic :: Scientific/Engineering', diff --git a/tools/docker_build_dist.sh b/tools/docker_build_dist.sh index 3acaa64cbe7..14d26c33f29 100755 --- a/tools/docker_build_dist.sh +++ b/tools/docker_build_dist.sh @@ -43,7 +43,6 @@ if [ "x${PYCBC_CONTAINER}" == "xpycbc_rhel_virtualenv" ]; then yum clean all yum makecache yum -y install openssl-devel - yum -y install ligo-proxy-utils yum -y install python3-virtualenv yum -y install hdf5-static libxml2-static zlib-static libstdc++-static cfitsio-static glibc-static swig fftw-static gsl-static --skip-broken diff --git a/tox.ini b/tox.ini index 4d0538c3968..ee0d9866c71 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,8 @@ recreate = true envlist = py-unittest indexserver = preinstall = https://pypi.python.org/simple -requires=tox-conda +requires=tox-conda + setuptools [base] deps = @@ -14,7 +15,7 @@ deps = [testenv] allowlist_externals = bash passenv=LAL_DATA_PATH -conda_deps=openssl=1.1 +conda_deps=openssl conda_channels=conda-forge platform = lin: linux mac: darwin