diff --git a/.github/workflows/bsd.yml b/.github/workflows/bsd.yml index 433e1d778..5358412a3 100644 --- a/.github/workflows/bsd.yml +++ b/.github/workflows/bsd.yml @@ -20,8 +20,7 @@ jobs: with: usesh: true run: | - scripts/internal/install-sysdeps.sh - PIP_BREAK_SYSTEM_PACKAGES=1 gmake install-pydeps install print-sysinfo test test-memleaks + PIP_BREAK_SYSTEM_PACKAGES=1 make install-sysdeps install-pydeps-test install print-sysinfo test test-memleaks openbsd: # if: false @@ -33,8 +32,7 @@ jobs: with: usesh: true run: | - scripts/internal/install-sysdeps.sh - PIP_BREAK_SYSTEM_PACKAGES=1 gmake install-pydeps install print-sysinfo test test-memleaks + PIP_BREAK_SYSTEM_PACKAGES=1 make install-sysdeps install-pydeps-test install print-sysinfo test test-memleaks netbsd: # if: false @@ -46,5 +44,4 @@ jobs: with: usesh: true run: | - scripts/internal/install-sysdeps.sh - PIP_BREAK_SYSTEM_PACKAGES=1 gmake PYTHON=python3.11 install-pydeps install print-sysinfo test test-memleaks + PIP_BREAK_SYSTEM_PACKAGES=1 make PYTHON=python3.11 install-sysdeps install-pydeps-test install print-sysinfo test test-memleaks diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b2bc54b88..356f4d080 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,7 +59,7 @@ jobs: CIBW_PRERELEASE_PYTHONS: True CIBW_TEST_EXTRAS: test CIBW_TEST_COMMAND: - make -C {project} PYTHON="env python" PSUTIL_SCRIPTS_DIR="{project}/scripts" install-sysdeps install-pydeps install print-sysinfo test test-memleaks + make -C {project} PYTHON="env python" PSUTIL_SCRIPTS_DIR="{project}/scripts" install-sysdeps install-pydeps-test install print-sysinfo test test-memleaks - name: Upload wheels uses: actions/upload-artifact@v4 @@ -87,7 +87,7 @@ jobs: CIBW_BUILD: 'cp27-*' CIBW_TEST_EXTRAS: test CIBW_TEST_COMMAND: - make -C {project} PYTHON="env python" PSUTIL_SCRIPTS_DIR="{project}/scripts" install-sysdeps install-pydeps install print-sysinfo test test-memleaks + make -C {project} PYTHON="env python" PSUTIL_SCRIPTS_DIR="{project}/scripts" install-sysdeps install-pydeps-test install print-sysinfo test test-memleaks steps: - uses: actions/checkout@v4 diff --git a/HISTORY.rst b/HISTORY.rst index 7741888ff..b9ef61dee 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,17 +5,20 @@ XXXX-XX-XX +**Enhancements** + +- 2448_: add ``make install-sysdeps`` target to install the necessary system + dependencies (python-dev, gcc, etc.) on all supported UNIX flavors. +- 2449_: add ``make install-pydeps-test`` and ``make install-pydeps-dev`` + targets. They can be used to install dependencies meant for running tests and + for local development. They can also be installed via ``pip install .[test]`` + and ``pip install .[dev]``. + **Bug fixes** - 2427_: psutil (segfault) on import in the free-threaded (no GIL) version of Python 3.13. (patch by Sam Gross) -**Enhancements** - -- 2448_: add ``make install-sysdeps`` target to install all necessary system - dependencies (python-dev, gcc, etc.) on different UNIX flavors. Also rename - ``make setup-dev-env`` to ``make install-pydeps``. - 6.0.0 ====== diff --git a/INSTALL.rst b/INSTALL.rst index c0812c74d..a425400fd 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -18,18 +18,14 @@ Compile psutil from sources UNIX ---- -On all UNIX systems you can use the -`install-sysdeps.sh `__ -script to install the system dependencies necessary to compile psutil. You can -invoke this script from the Makefile as:: +On all UNIX systems you can use the `install-sysdeps.sh +`__ +script. This will install the system dependencies necessary to compile psutil +from sources. You can invoke this script from the Makefile as:: make install-sysdeps -If you're on a BSD platform you need to use ``gmake`` instead of ``make``:: - - gmake install-sysdeps - -After system deps are installed you can build & compile psutil with:: +After system deps are installed, you can compile & install psutil with:: make build make install @@ -59,11 +55,10 @@ Alpine:: Windows ------- -In order to build / install psutil from sources on Windows you need Visual Studio -(MinGW is not supported). -Here's a couple of guides describing how to do it: `link `__ -and `link `__. -Once VS is installed do:: +In order to build / install psutil from sources on Windows you need to install +`Visua Studio 2017 `__ +or later (see cPython `devguide `__'s instructions). +MinGW is not supported. Once Visual Studio is installed do:: pip install --no-binary :all: psutil diff --git a/Makefile b/Makefile index b5733a20a..5fc72a5fb 100644 --- a/Makefile +++ b/Makefile @@ -13,31 +13,6 @@ PY3_DEPS = \ pytest \ pytest-xdist -# deps for local development -ifdef LINUX - ifndef CIBUILDWHEEL - PY3_DEPS += \ - black \ - check-manifest \ - coverage \ - packaging \ - pylint \ - pyperf \ - pypinfo \ - pytest-cov \ - requests \ - rstcheck \ - ruff \ - setuptools \ - sphinx \ - sphinx_rtd_theme \ - toml-sort \ - twine \ - virtualenv \ - wheel - endif -endif - # python 2 deps PY2_DEPS = \ futures \ @@ -64,12 +39,16 @@ BUILD_OPTS = `$(PYTHON) -c \ print('--parallel %s' % cpus if cpus > 1 else '')"` # In not in a virtualenv, add --user options for install commands. -INSTALL_OPTS = `$(PYTHON) -c \ +SETUP_INSTALL_OPTS = `$(PYTHON) -c \ "import sys; print('' if hasattr(sys, 'real_prefix') or hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix else '--user')"` +PIP_INSTALL_OPTS = --trusted-host files.pythonhosted.org --trusted-host pypi.org --upgrade # if make is invoked with no arg, default to `make help` .DEFAULT_GOAL := help +# install git hook +_ := $(shell mkdir -p .git/hooks/ && ln -sf ../../scripts/internal/git_pre_commit.py .git/hooks/pre-commit && chmod +x .git/hooks/pre-commit) + # =================================================================== # Install # =================================================================== @@ -110,8 +89,7 @@ build: ## Compile (in parallel) without installing. install: ## Install this package as current user in "edit" mode. ${MAKE} build - $(PYTHON_ENV_VARS) $(PYTHON) setup.py develop $(INSTALL_OPTS) - $(PYTHON_ENV_VARS) $(PYTHON) -c "import psutil" # make sure it actually worked + $(PYTHON_ENV_VARS) $(PYTHON) setup.py develop $(SETUP_INSTALL_OPTS) uninstall: ## Uninstall this package via pip. cd ..; $(PYTHON_ENV_VARS) $(PYTHON) -m pip uninstall -y -v psutil || true @@ -140,13 +118,22 @@ install-pip: ## Install pip (no-op if already installed). sys.exit(code);" install-sysdeps: - scripts/internal/install-sysdeps.sh + ./scripts/internal/install-sysdeps.sh + +install-pydeps-test: ## Install python deps necessary to run unit tests. + ${MAKE} install-pip + $(PYTHON) -m pip install $(PIP_INSTALL_OPTS) pip # upgrade pip to latest version + $(PYTHON) -m pip install $(PIP_INSTALL_OPTS) `$(PYTHON) -c "import setup; print(' '.join(setup.TEST_DEPS))"` -install-pydeps: ## Install GIT hooks, pip, test deps (also upgrades them). +install-pydeps-dev: ## Install python deps meant for local development. ${MAKE} install-git-hooks ${MAKE} install-pip - $(PYTHON) -m pip install $(INSTALL_OPTS) --trusted-host files.pythonhosted.org --trusted-host pypi.org --upgrade pip - $(PYTHON) -m pip install $(INSTALL_OPTS) --trusted-host files.pythonhosted.org --trusted-host pypi.org --upgrade $(PY_DEPS) + $(PYTHON) -m pip install $(PIP_INSTALL_OPTS) pip # upgrade pip to latest version + $(PYTHON) -m pip install $(PIP_INSTALL_OPTS) `$(PYTHON) -c "import setup; print(' '.join(setup.TEST_DEPS + setup.DEV_DEPS))"` + +install-git-hooks: ## Install GIT pre-commit hook. + ln -sf ../../scripts/internal/git_pre_commit.py .git/hooks/pre-commit + chmod +x .git/hooks/pre-commit # =================================================================== # Tests @@ -265,14 +252,6 @@ fix-all: ## Run all code fixers. ${MAKE} fix-black ${MAKE} fix-toml -# =================================================================== -# GIT -# =================================================================== - -install-git-hooks: ## Install GIT pre-commit hook. - ln -sf ../../scripts/internal/git_pre_commit.py .git/hooks/pre-commit - chmod +x .git/hooks/pre-commit - # =================================================================== # Distribution # =================================================================== diff --git a/appveyor.yml b/appveyor.yml index 553dbf9cd..7488400d9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -36,14 +36,14 @@ init: install: - "%WITH_COMPILER% %PYTHON%/python.exe -m pip --version" - - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py install-pydeps" + - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py install-pydeps-test" - "%WITH_COMPILER% %PYTHON%/python.exe -m pip freeze" - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py install" + - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py print-sysinfo" build: off test_script: - - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py print-sysinfo" - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py test" - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py test-memleaks" diff --git a/docs/DEVGUIDE.rst b/docs/DEVGUIDE.rst index 4a0b43aef..456714603 100644 --- a/docs/DEVGUIDE.rst +++ b/docs/DEVGUIDE.rst @@ -12,32 +12,36 @@ Once you have a compiler installed run: .. code-block:: bash git clone git@github.com:giampaolo/psutil.git - make install-pydeps # install useful dev libs (ruff, coverage, ...) + make install-sysdeps # install gcc and python headers + make install-pydeps-test # install python deps necessary to run unit tests make build make install make test -- ``make`` (see `Makefile`_) is the designated tool to run tests, build, install - try new features you're developing, etc. This also includes Windows (see - `make.bat`_ ). - Some useful commands are: +- ``make`` (and the accompanying `Makefile`_) is the designated tool to build, + install, run tests and do pretty much anything that involves development. + This also includes Windows, meaning that you can run `make command` similarly + as if you were on UNIX (see `make.bat`_ and `winmake.py`_). Some useful + commands are: .. code-block:: bash - make test-parallel # faster - make test-memleaks - make test-coverage - make lint-all # Run Python and C linter - make fix-all # Fix linting errors + make clean # remove build files + make install-pydeps-dev # install dev deps (ruff, black, ...) + make test # run tests + make test-parallel # run tests in parallel (faster) + make test-memleaks # run memory leak tests + make test-coverage # run test coverage + make lint-all # run linters + make fix-all # fix linters errors make uninstall make help - - To run a specific unit test: .. code-block:: bash - make test ARGS=psutil.tests.test_system.TestDiskAPIs + make test ARGS=psutil/tests/test_system.py::TestDiskAPIs::test_disk_usage - If you're working on a new feature and you wish to compile & test it "on the fly" from a test script, this is a quick & dirty way to do it: @@ -48,7 +52,8 @@ Once you have a compiler installed run: make test test_script.py # on Windows - Do not use ``sudo``. ``make install`` installs psutil as a limited user in - "edit" mode, meaning you can edit psutil code on the fly while you develop. + "edit" / development mode, meaning you can edit psutil code on the fly while + you develop. - If you want to target a specific Python version: @@ -60,10 +65,12 @@ Once you have a compiler installed run: Coding style ------------ -- Oython code strictly follows `PEP-8`_ styling guides and this is enforced by - a commit GIT hook installed via ``make install-git-hooks`` which will reject - commits if code is not PEP-8 complieant. -- C code should follow `PEP-7`_ styling guides. +- Python code strictly follows `PEP-8`_ styling guide. In addition we use + ``black`` and ``ruff`` code formatters / linters. This is enforced by a GIT + commit hook, installed via ``make install-git-hooks``, which will reject the + commit if code is not compliant. +- C code should follow `PEP-7`_ styling guides, but things are more relaxed. +- Linters are enforced also for ``.rst`` and ``.toml`` files. Code organization ----------------- @@ -101,7 +108,7 @@ Make a pull request - Fork psutil (go to https://github.com/giampaolo/psutil and click on "fork") - Git clone the fork locally: ``git clone git@github.com:YOUR-USERNAME/psutil.git`` -- Create a branch:``git checkout -b new-feature`` +- Create a branch: ``git checkout -b new-feature`` - Commit your changes: ``git commit -am 'add some feature'`` - Push the branch: ``git push origin new-feature`` - Create a new PR via the GitHub web interface and sign-off your work (see @@ -118,13 +125,14 @@ Documentation ------------- - doc source code is written in a single file: ``docs/index.rst``. -- doc can be built with ``make install-pydeps; cd docs; make html``. +- doc can be built with ``make install-pydeps-dev; cd docs; make html``. - public doc is hosted at https://psutil.readthedocs.io. .. _`CREDITS`: https://github.com/giampaolo/psutil/blob/master/CREDITS .. _`CONTRIBUTING.md`: https://github.com/giampaolo/psutil/blob/master/CONTRIBUTING.md .. _`HISTORY.rst`: https://github.com/giampaolo/psutil/blob/master/HISTORY.rst .. _`make.bat`: https://github.com/giampaolo/psutil/blob/master/make.bat +.. _`winmake.py`: https://github.com/giampaolo/psutil/blob/master/scripts/internal/winmake.py .. _`Makefile`: https://github.com/giampaolo/psutil/blob/master/Makefile .. _`PEP-7`: https://www.python.org/dev/peps/pep-0007/ .. _`PEP-8`: https://www.python.org/dev/peps/pep-0008/ diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 113ff0d55..935df40be 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -47,7 +47,7 @@ import win32api # requires "pip install pywin32" import win32con import win32process - import wmi # requires "pip install wmi" / "make install-pydeps" + import wmi # requires "pip install wmi" / "make install-pydeps-test" if WINDOWS: from psutil._pswindows import convert_oserror diff --git a/scripts/internal/install-sysdeps.sh b/scripts/internal/install-sysdeps.sh index 9894722f0..003a7d081 100755 --- a/scripts/internal/install-sysdeps.sh +++ b/scripts/internal/install-sysdeps.sh @@ -47,13 +47,13 @@ main() { elif [ $HAS_APK ]; then $SUDO apk add python3-dev gcc musl-dev linux-headers coreutils procps elif [ $FREEBSD ]; then - $SUDO pkg install -y gmake python3 gcc + $SUDO pkg install -y python3 gcc elif [ $NETBSD ]; then $SUDO /usr/sbin/pkg_add -v pkgin $SUDO pkgin update - $SUDO pkgin -y install gmake python311-* gcc12-* + $SUDO pkgin -y install python311-* gcc12-* elif [ $OPENBSD ]; then - $SUDO pkg_add gmake gcc python3 + $SUDO pkg_add gcc python3 else echo "Unsupported platform: $UNAME_S" fi diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 13fb57a72..3995a46f2 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -36,46 +36,18 @@ HERE = os.path.abspath(os.path.dirname(__file__)) ROOT_DIR = os.path.realpath(os.path.join(HERE, "..", "..")) PYPY = '__pypy__' in sys.builtin_module_names +WINDOWS = os.name == "nt" if PY3: GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" else: GET_PIP_URL = "https://bootstrap.pypa.io/pip/2.7/get-pip.py" +sys.path.insert(0, ROOT_DIR) # so that we can import setup.py -# mandatory deps -if PY3: - DEPS = [ - "setuptools", - "pytest", - "pytest-xdist", - "wheel", - ] -else: - DEPS = [ - "enum34", - "futures", - "ipaddress", - "mock==1.0.1", - "pytest-xdist", - "pytest==4.6.11", - "setuptools", - "wheel", - ] - -# deps for local development -if not APPVEYOR: - DEPS += [ - "coverage", - "pdbpp", - "pyperf", - "pyreadline", - "requests", - "wheel", - ] - -if not PYPY: - DEPS.append("pywin32") - DEPS.append("wmi") +import setup # NOQA + +TEST_DEPS = setup.TEST_DEPS +DEV_DEPS = setup.DEV_DEPS _cmds = {} if PY3: @@ -123,6 +95,8 @@ def stderr_handle(): def win_colorprint(s, color=LIGHTBLUE): + if not WINDOWS: + return print(s) color += 8 # bold handle = stderr_handle() SetConsoleTextAttribute = ctypes.windll.Kernel32.SetConsoleTextAttribute @@ -390,11 +364,18 @@ def clean(): safe_rmtree("tmp") -def install_pydeps(): +def install_pydeps_test(): + """Install useful deps.""" + install_pip() + install_git_hooks() + sh("%s -m pip install -U %s" % (PYTHON, " ".join(TEST_DEPS))) + + +def install_pydeps_dev(): """Install useful deps.""" install_pip() install_git_hooks() - sh("%s -m pip install -U %s" % (PYTHON, " ".join(DEPS))) + sh("%s -m pip install -U %s" % (PYTHON, " ".join(DEV_DEPS))) def test(args=""): @@ -585,10 +566,11 @@ def parse_args(): sp.add_parser('install', help="build + install in develop/edit mode") sp.add_parser('install-git-hooks', help="install GIT pre-commit hook") sp.add_parser('install-pip', help="install pip") + sp.add_parser('install-pydeps-dev', help="install dev python deps") + sp.add_parser('install-pydeps-test', help="install python test deps") sp.add_parser('print-access-denied', help="print AD exceptions") sp.add_parser('print-api-speed', help="benchmark all API calls") sp.add_parser('print-sysinfo', help="print system info") - sp.add_parser('install-pydeps', help="install python deps") test = sp.add_parser('test', help="[ARG] run tests") test_by_name = sp.add_parser('test-by-name', help=" run test by name") sp.add_parser('test-connections', help="run connections tests") diff --git a/setup.py b/setup.py index a88239c7c..126f66703 100755 --- a/setup.py +++ b/setup.py @@ -68,6 +68,52 @@ CP37_PLUS = PY37_PLUS and sys.implementation.name == "cpython" Py_GIL_DISABLED = sysconfig.get_config_var("Py_GIL_DISABLED") +# Test deps, installable via `pip install .[test]`. +if PY3: + TEST_DEPS = [ + "pytest", + "pytest-xdist", + "setuptools", + ] +else: + TEST_DEPS = [ + "futures", + "ipaddress", + "enum34", + "mock==1.0.1", + "pytest-xdist", + "pytest==4.6.11", + "setuptools", + ] +if WINDOWS and not PYPY: + TEST_DEPS.append("pywin32") + TEST_DEPS.append("wheel") + TEST_DEPS.append("wmi") + +# Development deps, installable via `pip install .[dev]`. +DEV_DEPS = [ + "black", + "check-manifest", + "coverage", + "packaging", + "pylint", + "pyperf", + "pypinfo", + "pytest-cov", + "requests", + "rstcheck", + "ruff", + "sphinx", + "sphinx_rtd_theme", + "toml-sort", + "twine", + "virtualenv", + "wheel", +] +if WINDOWS: + DEV_DEPS.append("pyreadline") + DEV_DEPS.append("pdbpp") + macros = [] if POSIX: macros.append(("PSUTIL_POSIX", 1)) @@ -88,19 +134,6 @@ sources.append('psutil/_psutil_posix.c') -extras_require = { - "test": [ - "enum34; python_version <= '3.4'", - "ipaddress; python_version < '3.0'", - "mock; python_version < '3.0'", - ] -} -if not PYPY: - extras_require['test'].extend( - ["pywin32; sys.platform == 'win32'", "wmi; sys.platform == 'win32'"] - ) - - def get_version(): INIT = os.path.join(HERE, 'psutil/__init__.py') with open(INIT) as f: @@ -512,6 +545,10 @@ def main(): ], ) if setuptools is not None: + extras_require = { + "dev": DEV_DEPS, + "test": TEST_DEPS, + } kwargs.update( python_requires=( ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"