Add support for fetching remote configuration

requirements.in

@@ -1,2 +1,4 @@
 click
 colorama
+pyyaml
+requests

requirements.txt

@@ -2,7 +2,13 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --upgrade requirements.txt
+#    pip-compile 
 #
+certifi==2019.3.9         # via requests
+chardet==3.0.4            # via requests
 click==7.0
 colorama==0.4.1
+idna==2.8                 # via requests
+pyyaml==5.1
+requests==2.21.0
+urllib3==1.24.3           # via requests

src/checks.py

@@ -8,23 +8,14 @@
 from typing import Callable, Dict, List, Optional, Tuple, Union
 
 import apache_2_license
+from config import Config
 from helpers import sh, step
 from report import Report, Result, ResultKind, color_result
 
 
 @dataclass
-class State:
-    project: str
-    module: Optional[str]
-    version: str
+class State(Config):
     work_dir: str
-    incubating: bool
-    zipname_template: str
-    sourcedir_template: str
-    github_reponame_template: str
-    gpg_key: str
-    git_hash: str
-    build_and_test_command: Optional[str]
 
     def _generate_optional_placeholders(
         self, key: str, value: str, condition: bool
@@ -73,7 +64,7 @@ def _pattern_placeholders(self) -> Dict[str, str]:
     @classmethod
     def list_placeholder_keys(cls) -> List[str]:
         # There's probably a better way to do this, but it'll do for now
-        instance = cls("", "", "", "", False, "", "", "", "", "", None)
+        instance = cls(*([None] * 12))  # type: ignore
         return list(instance._pattern_placeholders.keys())
 
     def _format_template(self, template: str) -> str:
@@ -170,6 +161,7 @@ def make_check(fun: CheckFun) -> Check:
 
 def run_checks(state: State, checks: List[Check]) -> Report:
     results = []
+
     for check in checks:
         step(f"Running check: {check.name}")
         try:

src/config.py

@@ -0,0 +1,17 @@
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class Config:
+    repo: str
+    project: str
+    module: Optional[str]
+    version: str
+    incubating: bool
+    zipname_template: str
+    sourcedir_template: str
+    github_reponame_template: str
+    gpg_key: str
+    git_hash: str
+    build_and_test_command: Optional[str]

src/main.py

@@ -2,13 +2,16 @@
 import os
 import sys
 import tempfile
-from typing import Optional
+from typing import Any, Dict, Optional, Union
 
 import click
 import colorama
+import requests
+import yaml
 from colorama import Fore, Style
 
 from checks import State, checks, run_checks
+from config import Config
 from helpers import header, sh, step
 from report import print_report
 
@@ -20,15 +23,127 @@
 USER_AGENT = "gh:openzipkin-contrib/apache-release-verification"
 
 
-@click.command()
-@click.option("--project", default="zipkin")
-@click.option("--module")
+def _load_yaml(x: Any) -> Dict:
+    return {key.replace("-", "_"): value for key, value in yaml.safe_load(x).items()}
+
+
+def local_config_callback(
+    ctx: click.Context,
+    _param: Union[click.Option, click.Parameter],
+    value: Optional[str],
+) -> Optional[str]:
+    if value is None:
+        logging.debug("local_config_callback: value is None, not loading anything")
+        return None
+    with open(value) as f:
+        data = _load_yaml(f)
+    logging.debug(f"local_config_callback: loaded data from {value}: {data}")
+    original = ctx.default_map or {}
+    ctx.default_map = {**original, **data}
+    return value
+
+
+def remote_config_provider(is_default: bool, url: str) -> Dict:
+    if not url.startswith("http://") and not url.startswith("https://"):
+        url = (
+            "https://openzipkin-contrib.github.io/apache-release-verification/"
+            f"presets/{url}.yaml"
+        )
+    logging.debug(f"remote_config_provider: Loading remote config from {url}")
+    resp = requests.get(url, headers={"User-Agent": USER_AGENT})
+    try:
+        resp.raise_for_status()
+        data = _load_yaml(resp.content)
+        logging.debug(f"remote_config_provider: Loaded data: {data}")
+        return data
+    except requests.exceptions.HTTPError:
+        if is_default:
+            return {}
+        else:
+            raise
+
+
+def remote_config_callback(
+    ctx: click.Context,
+    _param: Union[click.Option, click.Parameter],
+    value: Optional[str],
+) -> Optional[str]:
+    is_default = False
+    if value is None:
+        is_default = True
+        project = ctx.params["project"]
+        module = ctx.params["module"]
+        if project is not None and module is not None:
+            value = f"{project}/{module}"
+            logging.debug(f"remote_config_callback: inferred URL {value}")
+        else:
+            logging.debug(
+                "remote_config_callback: no value specified, and project or "
+                "module is None, not fetching remote config"
+            )
+    if value is not None:
+        original = ctx.default_map or {}
+        ctx.default_map = {**original, **remote_config_provider(is_default, value)}
+    return value
+
+
+def configure_logging(verbose: bool):
+    if verbose:
+        level = logging.DEBUG
+    else:
+        level = logging.INFO
+    logging.basicConfig(level=level, format="%(message)s")
+
+
+def configure_logging_callback(
+    _ctx: click.Context, _param: Union[click.Option, click.Parameter], verbose: bool
+) -> bool:
+    configure_logging(verbose)
+    return verbose
+
+
+@click.command(context_settings=dict(max_content_width=120))
+@click.option(
+    "-v",
+    "--verbose",
+    is_flag=True,
+    expose_value=False,
+    # We don't actually use this; it's evaluated in the __main__ block.
+    # See comment there for details.
+)
+@click.option("--project", default="zipkin", is_eager=True)
+@click.option("--module", is_eager=True)
+@click.option(
+    "--config",
+    default=None,
+    callback=local_config_callback,
+    expose_value=False,
+    is_eager=True,
+    help="Path to a local .yml file to load options from.",
+)
+@click.option(
+    "--remote-config",
+    default=None,
+    callback=remote_config_callback,
+    expose_value=False,
+    is_eager=True,
+    help="Remote file to load options from. Can be a full HTTP(S) URL, or a "
+    "simple string PROJECT/MODULE, which will be expanded to load from "
+    "the central repository at https://openzipkin-contrib.github.io/"
+    "apache-release-verification/presets/PROJECT/MODULE.yaml. Defaults "
+    "to $PROJECT/$MODULE",
+)
 @click.option("--version", required=True)
 @click.option("--gpg-key", required=True, help="ID of GPG key used to sign the release")
 @click.option(
     "--git-hash", required=True, help="Git hash of the commit the release is built from"
 )
-@click.option("--repo", default="dev", help="dev, release, or test")
+@click.option(
+    "--repo",
+    type=click.Choice(["dev", "release", "test"]),
+    default="dev",
+    help="dev, release, or test",
+)
 @click.option(
     "--incubating/--not-incubating",
     is_flag=True,
@@ -62,60 +177,27 @@
     "test the release. Executed with the exctracted source release archive "
     "as the working directory.",
 )
-@click.option("-v", "--verbose", is_flag=True)
-def main(
-    project: str,
-    module: Optional[str],
-    version: str,
-    git_hash: str,
-    gpg_key: str,
-    repo: str,
-    incubating: bool,
-    zipname_template: str,
-    sourcedir_template: str,
-    github_reponame_template: str,
-    build_and_test_command: Optional[str],
-    verbose: bool,
-) -> None:
-    configure_logging(verbose)
-    logging.debug(
-        f"Arguments: project={project} module={module} version={version} "
-        f"incubating={incubating} verbose={verbose} "
-        f"zipname_template={zipname_template} sourcedir_template={sourcedir_template} "
-        f"github_reponame_template={github_reponame_template} "
-        f"build_and_test_command={build_and_test_command} "
-        f"gpg_key={gpg_key} git_hash={git_hash}"
-    )
+def main(**kwargs) -> None:
+    config = Config(**kwargs)
+    logging.debug(f"Resolved config: {config}")
 
-    header_msg = f"Verifying release candidate for {project}"
-    if module:
-        header_msg += f"/{module}"
-    header_msg += f" {version}"
+    header_msg = f"Verifying release candidate for {config.project}"
+    if config.module:
+        header_msg += f"/{config.module}"
+    header_msg += f" {config.version}"
     header(header_msg)
     logging.info(f"{Fore.YELLOW}{DISCLAIMER}{Style.RESET_ALL}")
 
     workdir = make_and_enter_workdir()
     logging.info(f"Working directory: {workdir}")
 
-    base_url = generate_base_url(repo, project, incubating)
+    base_url = generate_base_url(config.repo, config.project, config.incubating)
     logging.debug(f"Base URL: {base_url}")
 
-    fetch_project(base_url, module, version, incubating)
+    fetch_project(base_url, config.module, config.version, config.incubating)
     fetch_keys(base_url)
 
-    state = State(
-        project=project,
-        module=module,
-        version=version,
-        work_dir=workdir,
-        incubating=incubating,
-        zipname_template=zipname_template,
-        sourcedir_template=sourcedir_template,
-        github_reponame_template=github_reponame_template,
-        gpg_key=gpg_key,
-        git_hash=git_hash,
-        build_and_test_command=build_and_test_command,
-    )
+    state = State(work_dir=workdir, **config.__dict__)
 
     # TODO this is the place to filter checks here with optional arguments
     report = run_checks(state, checks=checks)
@@ -130,14 +212,6 @@ def main(
         sys.exit(1)
 
 
-def configure_logging(verbose: bool) -> None:
-    if verbose:
-        level = logging.DEBUG
-    else:
-        level = logging.INFO
-    logging.basicConfig(level=level, format="%(message)s")
-
-
 def make_and_enter_workdir() -> str:
     workdir = tempfile.mkdtemp()
     os.chdir(workdir)
@@ -181,4 +255,15 @@ def fetch_keys(base_url: str) -> None:
 
 if __name__ == "__main__":
     colorama.init()
+
+    # There is only a single level of eagerness in Click, and we use that to
+    # load config options from local or remote files. But we need to handle
+    # --verbose before that happens, so that we can log from the related
+    # functions. So... you know, this is it.
+    if "-v" in sys.argv or "--verbose" in sys.argv:
+        configure_logging(True)
+    else:
+        configure_logging(False)
+
+    # Now we can execute the actual program
     main()