Merge pull request #1313 from HaoZeke/handlePipLocal

ENH,BUG: Handle `pip` matrices better

.github/workflows/ci.yml

@@ -2,6 +2,10 @@ name: CI
 
 on: [push, pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test:
     name: test

.github/workflows/pre-commit.yml

@@ -5,6 +5,10 @@ on:
   push:
     branches: [master]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   pre-commit:
     runs-on: ubuntu-latest

CHANGES.rst

@@ -8,6 +8,7 @@ New Features
 - Benchmarks can be skipped during their execution (after setup) by raising ``SkipNotImplemented`` (#1307)
 - Added ``default_benchmark_timeout`` to the configuration object, can also be
   passed via ``-a timeout=NUMBER`` (#1308)
+- ``ASV_RUNNER_PATH`` can be set from the terminal to test newer versions of ``asv_runner`` (#1312)
 
 API Changes
 ^^^^^^^^^^^
@@ -17,6 +18,8 @@ API Changes
 
 Bug Fixes
 ^^^^^^^^^
+- Fixed ``install_timeout`` for ``conda`` (#1310)
+- Fixed handling of local ``pip`` matrix (#1312)
 - Fixed the deadlock when mamba is used with an environment file. (#1300)
 - Fixed environment file usage with mamba and recognizes default
   ``environment.yml``. (#1303)

asv/environment.py

@@ -12,6 +12,7 @@
 import sys
 import itertools
 import subprocess
+import importlib
 from pathlib import Path
 
 from .console import log
@@ -508,7 +509,27 @@ def __init__(self, conf, python, requirements, tagged_env_vars):
         # These are needed for asv to build and run the project, not part of
         # benchmark name mangling
         self._base_requirements = {}
-        self._base_requirements["pip+asv_runner"] = ""
+        # gh-1314
+        asv_runner_path = os.getenv("ASV_RUNNER_PATH", "")
+        module_path = Path(asv_runner_path) / "asv_runner"
+
+        # Check if the path points to a directory containing the "asv_runner" module
+        if module_path.is_dir() and (module_path / "__init__.py").is_file():
+            spec = importlib.util.spec_from_file_location("asv_runner",
+                                                          module_path / "__init__.py")
+            # Attempt to load the module
+            asv_runner_module = importlib.util.module_from_spec(spec)
+            try:
+                spec.loader.exec_module(asv_runner_module)
+                self._base_requirements["pip+asv_runner"] = asv_runner_path
+            except Exception as e:
+                self._base_requirements["pip+asv_runner"] = ""
+                log.warning(f"Failed to load module from ASV_RUNNER_PATH: {e}")
+        else:
+            self._base_requirements["pip+asv_runner"] = ""
+            if asv_runner_path:
+                log.warning("ASV_RUNNER_PATH does not point"
+                            "to a directory containing the 'asv_runner' module")
         if not util.ON_PYPY:
             # XXX: What if pypy installed asv tries to benchmark a cpython
             # python?
@@ -519,6 +540,12 @@ def __init__(self, conf, python, requirements, tagged_env_vars):
         if (Path.cwd() / "pdm.lock").exists():
             self._base_requirements["pdm"] = ""
 
+        # Update the _base_requirements if needed
+        for key in list(self._requirements.keys()):
+            if key in self._base_requirements:
+                self._base_requirements[key] = self._requirements[key]
+                del self._requirements[key]
+
         self._build_command = conf.build_command
         self._install_command = conf.install_command
         self._uninstall_command = conf.uninstall_command
@@ -970,6 +997,8 @@ def run_executable(self, executable, args, **kwargs):
                              PIP_USER=str("false"),
                              PATH=str(os.pathsep.join(paths)))
         exe = self.find_executable(executable)
+        if kwargs.get("timeout", None) is None:
+            kwargs["timeout"] = self._install_timeout
         return util.check_output([exe] + args, **kwargs)
 
     def load_info_file(self, path):

asv/plugins/conda.py

@@ -87,6 +87,8 @@ def __init__(self, conf, python, requirements, tagged_env_vars):
         self._python = python
         self._requirements = requirements
         self._conda_channels = conf.conda_channels
+        if "conda-forge" not in conf.conda_channels:
+            self._conda_channels += ["conda-forge"]
         self._conda_environment_file = conf.conda_environment_file
 
         if conf.conda_environment_file == "IGNORE":
@@ -143,13 +145,8 @@ def _setup(self):
             # categorize & write dependencies based on pip vs. conda
             env_file.writelines((f'   - {s}\n' for s in conda_args))
             if pip_args:
-                # and now specify the packages that are to be installed in
-                # the pip subsection
                 env_file.write('   - pip:\n')
-                env_file.writelines((f'     - {s}\n' for s in pip_args))
-
             env_file.close()
-
             try:
                 env_file_name = self._conda_environment_file or env_file.name
                 self._run_conda(['env', 'create', '-f', env_file_name,
@@ -174,18 +171,24 @@ def _setup(self):
                 raise
         finally:
             os.unlink(env_file.name)
+        if not len(pip_args) == 0:
+            pip_calls = []
+            for pkgname, pipval in pip_args:
+                pip_calls.append(util.construct_pip_call(self._run_pip, pkgname, pipval))
+            for pipcall in pip_calls:
+                pipcall()
 
     def _get_requirements(self):
         conda_args = []
         pip_args = []
 
         for key, val in {**self._requirements,
                          **self._base_requirements}.items():
-            if key.startswith('pip+'):
+            if key.startswith("pip+"):
                 if val:
-                    pip_args.append(f"{key[4:]}=={val}")
+                    pip_args.append((key[4:], val))
                 else:
-                    pip_args.append(key[4:])
+                    pip_args.append((key[4:], None))
             else:
                 if val:
                     conda_args.append(f"{key}={val}")
@@ -204,7 +207,9 @@ def _run_conda(self, args, env=None):
             raise util.UserError(str(e))
 
         with _conda_lock():
-            return util.check_output([conda] + args, env=env)
+            return util.check_output([conda] + args,
+                                     timeout=self._install_timeout,
+                                     env=env)
 
     def run(self, args, **kwargs):
         log.debug(f"Running '{' '.join(args)}' in {self.name}")
@@ -224,3 +229,8 @@ def run_executable(self, executable, args, **kwargs):
 
         with lock():
             return super(Conda, self).run_executable(executable, args, **kwargs)
+
+    def _run_pip(self, args, **kwargs):
+        # Run pip via python -m pip, so that it works on Windows when
+        # upgrading pip itself, and avoids shebang length limit on Linux
+        return self.run_executable("python", ["-mpip"] + list(args), **kwargs)

asv/plugins/mamba.py

@@ -83,7 +83,10 @@ def _matches(cls, python):
         if not re.match(r'^[0-9].*$', python):
             return False
         else:
-            mamba_path = str(Path(os.getenv("CONDA_EXE")).parent / "mamba")
+            if os.getenv("CONDA_EXE"):
+                mamba_path = str(Path(os.getenv("CONDA_EXE")).parent / "mamba")
+            else:
+                return False
             try:
                 return util.search_channels(mamba_path, "python", python)
             except util.ProcessError:
@@ -122,8 +125,11 @@ def _setup(self):
             transaction = solver.solve(mamba_pkgs)
             transaction.execute(libmambapy.PrefixData(self._path))
         if not len(pip_args) == 0:
-            pargs = ["install", "-v", "--upgrade-strategy", "only-if-needed"]
-            self._run_pip(pargs + pip_args)
+            pip_calls = []
+            for pkgname, pipval in pip_args:
+                pip_calls.append(util.construct_pip_call(self._run_pip, pkgname, pipval))
+            for pipcall in pip_calls:
+                pipcall()
 
     def _get_requirements(self):
         mamba_args = []
@@ -133,9 +139,9 @@ def _get_requirements(self):
                          **self._base_requirements}.items():
             if key.startswith("pip+"):
                 if val:
-                    pip_args.append(f"{key[4:]}=={val}")
+                    pip_args.append((key[4:], val))
                 else:
-                    pip_args.append(key[4:])
+                    pip_args.append((key[4:], None))
             else:
                 if val:
                     mamba_args.append(f"{key}={val}")

asv/plugins/virtualenv.py

@@ -147,19 +147,21 @@ def _install_requirements(self):
         env.update(self.build_env_vars)
 
         self._run_pip(pip_args, env=env)
+        pip_calls = []
+        pip_args = []
 
-        args = ['install', '-v', '--upgrade']
         for key, val in {**self._requirements,
                          **self._base_requirements}.items():
-            pkg = key
-            if key.startswith('pip+'):
-                pkg = key[4:]
-
-            if val:
-                args.append(f"{pkg}=={val}")
-            else:
-                args.append(pkg)
-        self._run_pip(args, timeout=self._install_timeout, env=env)
+            if key.startswith("pip+"):
+                if val:
+                    pip_args.append((key[4:], val))
+                else:
+                    pip_args.append((key[4:], None))
+
+        for pkgname, pipval in pip_args:
+            pip_calls.append(util.construct_pip_call(self._run_pip, pkgname, pipval))
+        for pipcall in pip_calls:
+            pipcall()
 
     def _run_pip(self, args, **kwargs):
         # Run pip via python -m pip, so that it works on Windows when

asv/util.py

@@ -23,6 +23,8 @@
 import operator
 import collections
 import multiprocessing
+import functools
+from pathlib import Path
 
 import json5
 from asv_runner.util import human_time, human_float, _human_time_units
@@ -1297,6 +1299,22 @@ def search_channels(cli_path, pkg, version):
     return True
 
 
+def construct_pip_call(pip_caller, pkgname, pipval=None):
+    pargs = ['install', '-v', '--upgrade']
+    if pipval:
+        ptokens = pipval.split()
+        flags = [x for x in ptokens if x.startswith('-')]
+        paths = [x for x in ptokens if Path(x).is_dir()]
+        pargs += flags
+        if paths:
+            pargs += paths
+        else:
+            pargs += [f"{pkgname}=={pipval}"]
+    else:
+        pargs += [pkgname]
+    return functools.partial(pip_caller, pargs)
+
+
 if hasattr(sys, 'pypy_version_info'):
     ON_PYPY = True
 else:

docs/source/asv.conf.json.rst

@@ -267,6 +267,11 @@ the project being benchmarked may specify in its ``setup.py`` file.
     name with ``pip+``. For example, ``emcee`` is only available from ``pip``,
     so the package name to be used is ``pip+emcee``.
 
+    .. versionadded::0.6
+
+      ``pip`` dependencies can now accept local (fully qualified) directories,
+      and also take flags (e.g. ``-e``)
+
 The ``env`` and ``env_nobuild`` dictionaries can be used to set also
 environment variables::
 

docs/source/dev.rst

@@ -5,6 +5,7 @@ This section describes some things that may be of interest to
 developers and other people interested in internals of ``asv``.
 
 .. note::
+
    From version 0.6 on-wards, functionality in ``asv`` has been split into the
    section needed by the code being benchmarked (``asv_runner``) and the rest of
    ``asv``. This means that the ``asv`` documentation covers setting up

docs/source/env_vars.rst

@@ -25,6 +25,14 @@ behavior are also set:
 - ``PYTHONNOUSERSITE``: ``True`` (for conda environments only)
 - ``PYTHONPATH``: unset (if really needed, can be overridden by setting ``ASV_PYTHONPATH``)
 
+.. note::
+
+    .. versionadded::0.6
+
+      ``ASV_RUNNER_PATH`` may be set to provide a local installation of
+      ``asv_runner``, mostly used for the CI to ensure changes to ``asv_runner``
+      do not break ``asv``
+
 
 Custom environment variables
 ----------------------------

docs/source/writing_benchmarks.rst

@@ -187,9 +187,10 @@ value is the maximum of the timeouts of the benchmarks using it.
 
 .. note::
 
-   From version 0.6 onwards, the configuration option
-   ``default_benchmark_timeout`` can also be set for a project-wide
-   timeout.
+    .. versionchanged:: 0.6
+
+        The configuration option ``default_benchmark_timeout``
+        can also be set for a project-wide timeout.
 
 .. _benchmark-attributes: