Merge pull request #16 from pangeo-forge/file-based-config

Support file-based config

.github/workflows/test-action.yaml

@@ -8,14 +8,24 @@ on:
 
 jobs:
   test:
-    name: (recipes@${{ matrix.recipes-version }})
+    name: (recipes@${{ matrix.recipes-version }}, config=${{ matrix.config }})
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        recipes-version: ["0.10.3"]  # , "main"]
+        recipes-version: [
+          # "main",
+          # "0.10.3",
+          "0.10.4",
+        ]
+        config: [
+          "inline",
+          "file",
+        ]
     steps:
       - uses: actions/checkout@v4
+
+      # ----- Shared setup --------------------------------------------------------------
       - name: "Clone test feedstock"
         # Fetches test feedstock (containing example test recipes) from pangeo-forge-recipes
         run: |
@@ -31,7 +41,41 @@ jobs:
         run: |
           grep -E 'recipes|gpcp' pangeo-forge-recipes/examples/feedstock/meta.yaml > temp.yaml \
           && mv temp.yaml pangeo-forge-recipes/examples/feedstock/meta.yaml
+      
+      # ----- File based config ---------------------------------------------------------
+      - name: Write local-config.json
+        if: matrix.config == 'file'
+        run: |
+          cat << EOF > ./local-config.json
+          {
+          "BaseCommand": {
+            "feedstock_subdir": "pangeo-forge-recipes/examples/feedstock"
+          },
+          "Bake": {
+            "prune": true,
+            "bakery_class": "pangeo_forge_runner.bakery.local.LocalDirectBakery"
+          },
+          "TargetStorage": {
+            "fsspec_class": "fsspec.implementations.local.LocalFileSystem",
+            "root_path": "./target"
+          },
+          "InputCacheStorage": {
+            "fsspec_class": "fsspec.implementations.local.LocalFileSystem",
+            "root_path": "./cache"
+          }
+          }
+          EOF
+      - name: "Deploy recipes"
+        if: matrix.config == 'file'
+        uses: ./
+        with:
+          # select_recipe_by_label: true
+          pangeo_forge_runner_config: ./local-config.json
+
+
+      # ---- Inline config --------------------------------------------------------------
       - name: "Deploy recipes"
+        if: matrix.config == 'inline'
         uses: ./
         with:
           # select_recipe_by_label: true

action/deploy_recipe.py

@@ -50,9 +50,30 @@ def main():
     run_attempt = os.environ["GITHUB_RUN_ATTEMPT"]
 
     # user input
-    config = json.loads(os.environ["INPUT_PANGEO_FORGE_RUNNER_CONFIG"])
+    config_string = os.environ["INPUT_PANGEO_FORGE_RUNNER_CONFIG"]
     select_recipe_by_label = os.environ["INPUT_SELECT_RECIPE_BY_LABEL"]
 
+    # parse config
+    print(f"pangeo-forge-runner-config provided as {config_string}")
+    try:
+        if os.path.exists(config_string):
+            # we allow local paths pointing to json files
+            print(f"Loading json from file '{config_string}'...")
+            with open(config_string) as f:
+                config = json.load(f)
+        else:
+            # or json strings passed inline in the workflow yaml
+            print(f"{config_string} does not exist as a file. Loading json from string...")
+            config = json.loads(config_string)
+    except json.JSONDecodeError as e:
+        raise ValueError(
+            f"{config_string} failed to parse to JSON. If you meant to pass a JSON string, "
+            "please confirm that it is correctly formatted. If you meant to pass a filename, "
+            "please confirm this path exists. Note, pangeo-forge/deploy-recipe-action should "
+            "always be invoked after actions/checkout, therefore the provided path must be "
+            "given relative to the repo root."
+        ) from e
+
     # log variables to stdout
     print(f"{head_ref = }")
     print(f"{sha = }")

tests/test_main.py

@@ -24,8 +24,23 @@ def select_recipe_by_label(request):
     return request.param
 
 
+@pytest.fixture(params=["inline", "file", "broken_inline", "broken_file"])
+def pangeo_forge_runner_config(request, tmp_path_factory: pytest.TempPathFactory):
+    if "inline" in request.param:
+        return "{}" if not "broken" in request.param else "{broken: json}"
+    elif "file" in request.param:
+        if not "broken" in request.param:
+            tmp_config_dir = tmp_path_factory.mktemp("config")
+            outpath = tmp_config_dir / "config.json"
+            with open(tmp_config_dir / "config.json", mode="w") as f:
+                json.dump({}, f)
+            return outpath.absolute().as_posix()
+        else:
+            return "non/existent/path.json"
+
+
 @pytest.fixture
-def env(select_recipe_by_label, head_ref):
+def env(select_recipe_by_label, head_ref, pangeo_forge_runner_config):
     return {
         "GITHUB_REPOSITORY": "my/repo",
         "GITHUB_API_URL": "https://api.github.com",
@@ -36,7 +51,7 @@ def env(select_recipe_by_label, head_ref):
         "GITHUB_REPOSITORY_ID": "1234567890",
         "GITHUB_RUN_ID": "0987654321",
         "GITHUB_RUN_ATTEMPT": "1",
-        "INPUT_PANGEO_FORGE_RUNNER_CONFIG": '{}',
+        "INPUT_PANGEO_FORGE_RUNNER_CONFIG": pangeo_forge_runner_config,
         "INPUT_SELECT_RECIPE_BY_LABEL": select_recipe_by_label,
     }
 
@@ -101,6 +116,14 @@ def mock_tempfile_name():
     return "mock-temp-file.json"
 
 
+def get_config_asdict(config: str) -> dict:
+    """Config could be a JSON file path or JSON string, either way load it to dict."""
+    if os.path.exists(config):
+        with open(config) as f:
+            return json.load(f)
+    return json.loads(config)
+
+
 @patch("deploy_recipe.os.listdir")
 @patch("deploy_recipe.subprocess.run")
 @patch("deploy_recipe.requests.get")
@@ -134,24 +157,46 @@ def test_main(
     listdir_return_value, pip_install_raises = listdir_return_value_with_pip_install_raises
     listdir.return_value = listdir_return_value
 
-    if pip_install_raises:
-        config: dict = json.loads(env["INPUT_PANGEO_FORGE_RUNNER_CONFIG"])
-        config.update({"BaseCommand": {"feedstock_subdir": "broken-requirements-feedstock"}})
-        env["INPUT_PANGEO_FORGE_RUNNER_CONFIG"] = json.dumps(config)
+    config_is_broken = (
+        env["INPUT_PANGEO_FORGE_RUNNER_CONFIG"] == "{broken: json}"
+        or env["INPUT_PANGEO_FORGE_RUNNER_CONFIG"] == "non/existent/path.json"
+    )
+
+    if pip_install_raises and not config_is_broken:
+        update_with = {"BaseCommand": {"feedstock_subdir": "broken-requirements-feedstock"}}
+        config = get_config_asdict(env["INPUT_PANGEO_FORGE_RUNNER_CONFIG"])
+        config.update(update_with)
+
+        if os.path.exists(env["INPUT_PANGEO_FORGE_RUNNER_CONFIG"]):
+            with open(env["INPUT_PANGEO_FORGE_RUNNER_CONFIG"], "w") as f:
+                json.dump(config, f)
+        else:
+            env["INPUT_PANGEO_FORGE_RUNNER_CONFIG"] = json.dumps(config)
 
     with patch.dict(os.environ, env):
-        if "broken-requirements-feedstock" in env["INPUT_PANGEO_FORGE_RUNNER_CONFIG"]:
+        if (
+            not config_is_broken
+            and "broken-requirements-feedstock" in str(
+                get_config_asdict(env["INPUT_PANGEO_FORGE_RUNNER_CONFIG"])
+            )
+        ):
             with pytest.raises(ValueError):
                 main()
             # if pip install fails, we should bail early & never invoke `bake`. re: `call_args_list`:
             # https://docs.python.org/3/library/unittest.mock.html#unittest.mock.Mock.call_args_list
             for call in [args[0][0] for args in subprocess_run.call_args_list]:
                 assert "bake" not in call
+        elif env["INPUT_PANGEO_FORGE_RUNNER_CONFIG"] == "{broken: json}":
+            with pytest.raises(ValueError):
+                main()
+        elif env["INPUT_PANGEO_FORGE_RUNNER_CONFIG"] == "non/existent/path.json":
+            with pytest.raises(ValueError):
+                main()
         else:
             main()
 
             if any([path.endswith("requirements.txt") for path in listdir_return_value]):
-                config = json.loads(env["INPUT_PANGEO_FORGE_RUNNER_CONFIG"])
+                config = get_config_asdict(env["INPUT_PANGEO_FORGE_RUNNER_CONFIG"])
                 feedstock_subdir = (
                     config["BaseCommand"]["feedstock-subdir"]
                     if "BaseCommand" in config