feat: Add support for prepending conda channels via the CLI. (#67)

README.md

@@ -333,6 +333,10 @@ The `--file_key` argument is passed the `test` key name from the `files` configu
 
 The `--file_key`, `--output`, and `--matrix` flags must be used together. `--matrix` may be an empty string if the file that should be generated does not depend on any specific matrix variations.
 
+The `--prepend-channels` argument accepts a semi-colon separated list of additional channels to use, like `rapids-dependency-file-generator --prepend-channels "my_channel;my_other_channel"`.
+If both `--output` and `--prepend-channels` are provided, the output format must be conda.
+Prepending channels can be useful for adding local channels with packages to be tested in CI workflows.
+
 Running `rapids-dependency-file-generator -h` will show the most up-to-date CLI arguments.
 
 ## Examples

src/rapids_dependency_file_generator/cli.py

@@ -4,7 +4,7 @@
 import yaml
 
 from ._version import __version__ as version
-from .constants import OutputTypes, default_dependency_file_path
+from .constants import OutputTypes, default_channels, default_dependency_file_path
 from .rapids_dependency_file_generator import (
     delete_existing_files,
     make_dependency_files,
@@ -51,6 +51,16 @@ def validate_args(argv):
         ),
     )
 
+    parser.add_argument(
+        "--prepend-channels",
+        help=(
+            "A string representing a list of conda channels to prepend to the list of "
+            "channels. Channels should be separated by a semicolon, such as "
+            '`--prepend-channels "my_channel;my_other_channel"`. This option is '
+            f"only valid with --output {OutputTypes.CONDA} or no --output"
+        ),
+    )
+
     args = parser.parse_args(argv)
     dependent_arg_keys = ["file_key", "output", "matrix"]
     dependent_arg_values = [getattr(args, key) is None for key in dependent_arg_keys]
@@ -60,6 +70,11 @@ def validate_args(argv):
             + "".join([f"\n  --{x}" for x in dependent_arg_keys])
         )
 
+    if args.prepend_channels and args.output and args.output != str(OutputTypes.CONDA):
+        raise ValueError(
+            f"--prepend-channels is only valid with --output {OutputTypes.CONDA}"
+        )
+
     # If --clean was passed without arguments, default to cleaning from the root of the
     # tree where the config file is.
     if args.clean == "":
@@ -99,6 +114,13 @@ def main(argv=None):
             }
         }
 
+    if args.prepend_channels:
+        prepend_channels = args.prepend_channels.split(";")
+        parsed_config["channels"] = prepend_channels + parsed_config.get(
+            "channels", default_channels
+        )
+
     if args.clean:
         delete_existing_files(args.clean)
+
     make_dependency_files(parsed_config, args.config, to_stdout)

tests/examples/prepend-channels/dependencies.yaml

@@ -0,0 +1,45 @@
+files:
+  build:
+    output: conda
+    conda_dir: output/actual
+    matrix:
+      cuda: ["11.5", "11.6"]
+      arch: [x86_64]
+    includes:
+      - build
+channels:
+  - rapidsai
+  - conda-forge
+dependencies:
+  build:
+    common:
+      - output_types: [conda, requirements]
+        packages:
+          - clang=11.1.0
+          - spdlog>=1.8.5,<1.9
+      - output_types: conda
+        packages:
+          - pip
+          - pip:
+              - git+https://github.com/python-streamz/streamz.git@master
+    specific:
+      - output_types: [conda, requirements]
+        matrices:
+          - matrix:
+              cuda: "11.5"
+            packages:
+              - cuda-python>=11.5,<11.7.1
+          - matrix:
+              cuda: "11.6"
+            packages:
+              - cuda-python>=11.6,<11.7.1
+      - output_types: conda
+        matrices:
+          - matrix:
+              cuda: "11.5"
+            packages:
+              - cudatoolkit=11.5
+          - matrix:
+              cuda: "11.6"
+            packages:
+              - cudatoolkit=11.6

tests/examples/prepend-channels/output/expected/build_cuda-115_arch-x86_64.yaml

@@ -0,0 +1,16 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
+channels:
+- my_channel
+- my_other_channel
+- rapidsai
+- conda-forge
+dependencies:
+- clang=11.1.0
+- cuda-python>=11.5,<11.7.1
+- cudatoolkit=11.5
+- pip
+- spdlog>=1.8.5,<1.9
+- pip:
+  - git+https://github.com/python-streamz/streamz.git@master
+name: build_cuda-115_arch-x86_64

tests/examples/prepend-channels/output/expected/build_cuda-116_arch-x86_64.yaml

@@ -0,0 +1,16 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
+channels:
+- my_channel
+- my_other_channel
+- rapidsai
+- conda-forge
+dependencies:
+- clang=11.1.0
+- cuda-python>=11.6,<11.7.1
+- cudatoolkit=11.6
+- pip
+- spdlog>=1.8.5,<1.9
+- pip:
+  - git+https://github.com/python-streamz/streamz.git@master
+name: build_cuda-116_arch-x86_64

tests/test_cli.py

@@ -24,6 +24,21 @@ def test_validate_args():
     with pytest.raises(Exception):
         validate_args(["--output", "conda", "--matrix", "cuda=11.5;arch=x86_64"])
 
+    # Prepending channels with an output type that is not conda
+    with pytest.raises(Exception):
+        validate_args(
+            [
+                "--output",
+                "requirements",
+                "--matrix",
+                "cuda=11.5;arch=x86_64",
+                "--file_key",
+                "all",
+                "--prepend-channels",
+                "my_channel;my_other_channel",
+            ]
+        )
+
     # Valid
     validate_args(
         [
@@ -35,3 +50,25 @@ def test_validate_args():
             "all",
         ]
     )
+
+    # Valid, with prepended channels
+    validate_args(
+        [
+            "--prepend-channels",
+            "my_channel;my_other_channel",
+        ]
+    )
+
+    # Valid, with output/matrix/file_key and prepended channels
+    validate_args(
+        [
+            "--output",
+            "conda",
+            "--matrix",
+            "cuda=11.5;arch=x86_64",
+            "--file_key",
+            "all",
+            "--prepend-channels",
+            "my_channel;my_other_channel",
+        ]
+    )

tests/test_examples.py

@@ -63,14 +63,18 @@ def test_examples(example_dir):
                 new_path.parent.mkdir(parents=True, exist_ok=True)
                 shutil.copyfile(full_path, new_path)
 
-    main(
-        [
-            "--config",
-            str(dep_file_path),
-            "--clean",
-            str(example_dir.joinpath("output", "actual")),
-        ]
-    )
+    cli_args = [
+        "--config",
+        str(dep_file_path),
+        "--clean",
+        str(example_dir.joinpath("output", "actual")),
+    ]
+
+    # Prepend channels for the prepend_channels tests
+    if example_dir.name in ("prepend-channels"):
+        cli_args = ["--prepend-channels", "my_channel;my_other_channel"] + cli_args
+
+    main(cli_args)
 
     expected_file_set = make_file_set(expected_dir)
     actual_file_set = make_file_set(actual_dir)