Merge pull request #2885 from SwissDataScienceCenter/release/v1.2.4

chore: release v1.2.4

CHANGES.rst

@@ -18,6 +18,26 @@
 Changes
 =======
 
+`1.2.4 <https://github.com/SwissDataScienceCenter/renku-python/compare/v1.2.3...v1.2.4>`__ (2022-05-06)
+-------------------------------------------------------------------------------------------------------
+
+Bug Fixes
+~~~~~~~~~
+
+-  **core:** fix using float values in renku workflow iterate
+   (`#2875 <https://github.com/SwissDataScienceCenter/renku-python/issues/2875>`__)
+   (`07934a8 <https://github.com/SwissDataScienceCenter/renku-python/commit/07934a8df49a4b8a7a4c25eddaae93b97943ac59>`__)
+-  **service:** set oauth token when using gitlab APIs
+   (`#2884 <https://github.com/SwissDataScienceCenter/renku-python/issues/2884>`__)
+   (`11a69d7 <https://github.com/SwissDataScienceCenter/renku-python/commit/11a69d71fc08854a03bf3e524f0d68d3e86a5685>`__)
+
+Features
+~~~~~~~~
+
+-  **core:** preserve staged files when editing renku config
+   (`#2871 <https://github.com/SwissDataScienceCenter/renku-python/issues/2871>`__)
+   (`3c3cc66 <https://github.com/SwissDataScienceCenter/renku-python/commit/3c3cc66a426c71d742d13b5fb394791d8425a5c6>`__)
+
 `1.2.3 <https://github.com/SwissDataScienceCenter/renku-python/compare/v1.2.2...v1.2.3>`__ (2022-04-29)
 -------------------------------------------------------------------------------------------------------
 

helm-chart/renku-core/Chart.yaml

@@ -3,4 +3,4 @@ appVersion: "1.0"
 description: A Helm chart for Kubernetes
 name: renku-core
 icon: https://avatars0.githubusercontent.com/u/53332360?s=400&u=a4311d22842343604ef61a8c8a1e5793209a67e9&v=4
-version: 1.2.3
+version: 1.2.4

helm-chart/renku-core/values.yaml

@@ -109,7 +109,7 @@ versions:
     fullnameOverride: ""
     image:
       repository: renku/renku-core
-      tag: "v1.2.3"
+      tag: "v1.2.4"
       pullPolicy: IfNotPresent
   v8:
     name: v8

renku/command/command_builder/command.py

@@ -414,6 +414,7 @@ def with_commit(
         commit_if_empty: bool = False,
         raise_if_empty: bool = False,
         commit_only: Optional[bool] = None,
+        skip_staging: bool = False,
     ) -> "Command":
         """Create a commit.
 
@@ -426,7 +427,7 @@ def with_commit(
         """
         from renku.command.command_builder.repo import Commit
 
-        return Commit(self, message, commit_if_empty, raise_if_empty, commit_only)
+        return Commit(self, message, commit_if_empty, raise_if_empty, commit_only, skip_staging)
 
     @check_finalized
     def lock_project(self) -> "Command":

renku/command/command_builder/repo.py

@@ -35,6 +35,7 @@ def __init__(
         commit_if_empty: Optional[bool] = False,
         raise_if_empty: Optional[bool] = False,
         commit_only: Optional[bool] = None,
+        skip_staging: bool = False,
     ) -> None:
         """__init__ of Commit.
 
@@ -49,6 +50,7 @@ def __init__(
         self._commit_if_empty = commit_if_empty
         self._raise_if_empty = raise_if_empty
         self._commit_filter_paths = commit_only
+        self._skip_staging: bool = skip_staging
 
     def _pre_hook(self, builder: Command, context: dict, *args, **kwargs) -> None:
         """Hook to create a commit transaction.
@@ -65,7 +67,9 @@ def _pre_hook(self, builder: Command, context: dict, *args, **kwargs) -> None:
         from renku.core.management.git import prepare_commit
 
         self.diff_before = prepare_commit(
-            context["client_dispatcher"].current_client, commit_only=self._commit_filter_paths
+            context["client_dispatcher"].current_client,
+            commit_only=self._commit_filter_paths,
+            skip_staging=self._skip_staging,
         )
 
     def _post_hook(self, builder: Command, context: dict, result: CommandResult, *args, **kwargs):
@@ -90,6 +94,7 @@ def _post_hook(self, builder: Command, context: dict, result: CommandResult, *ar
                 commit_empty=self._commit_if_empty,
                 raise_if_empty=self._raise_if_empty,
                 commit_message=self._message,
+                skip_staging=self._skip_staging,
             )
         except errors.RenkuException as e:
             result.error = e

renku/command/config.py

@@ -100,7 +100,7 @@ def update_config():
         Command()
         .command(_update_config)
         .require_migration()
-        .with_commit(commit_if_empty=False, commit_only=CONFIG_LOCAL_PATH)
+        .with_commit(commit_if_empty=False, commit_only=CONFIG_LOCAL_PATH, skip_staging=True)
         .with_database()
     )
 

renku/command/git.py

@@ -24,7 +24,6 @@
 from renku.core import errors
 
 GIT_KEY = "renku.git"
-GIT_ISOLATION = "renku.worktree"
 
 
 def set_git_home(value: Path):
@@ -57,25 +56,3 @@ def get_git_home(path=".") -> Path:
         return Repository(path, search_parent_directories=True).path
     except errors.GitError:
         raise ValueError(f"Cannot find a git repository at '{path}'")
-
-
-def set_git_isolation(value):
-    """Set Git isolation.
-
-    Args:
-        value: Git isolation level.
-
-    Returns:
-        The value passed.
-    """
-    ctx = click.get_current_context()
-    ctx.meta[GIT_ISOLATION] = value
-
-    return value
-
-
-def get_git_isolation():
-    """Get Git isolation from the current context."""
-    ctx = click.get_current_context(silent=True)
-    if ctx and GIT_ISOLATION in ctx.meta:
-        return ctx.meta[GIT_ISOLATION]

renku/command/options.py

@@ -19,17 +19,6 @@
 
 import click
 
-from .git import set_git_isolation
-
-option_isolation = click.option(
-    "--isolation",
-    is_flag=True,
-    default=False,
-    callback=lambda ctx, param, value: set_git_isolation(value),
-    help="Set up the isolation for invoking of the given command.",
-)
-
-
 option_external_storage_requested = click.option(
     "external_storage_requested",
     "--external-storage/--no-external-storage",

renku/core/management/git.py

@@ -38,14 +38,14 @@
 STARTED_AT = int(time.time() * 1e3)
 
 
-def prepare_commit(
-    client,
-    commit_only=None,
-    skip_dirty_checks=False,
-):
+def prepare_commit(client, commit_only=None, skip_dirty_checks=False, skip_staging: bool = False):
     """Gather information about repo needed for committing later on."""
     diff_before = set()
 
+    if skip_staging:
+        if not isinstance(commit_only, list) or len(commit_only) == 0:
+            raise errors.OperationError("Cannot use ``skip_staging`` without specifying files to commit.")
+
     if commit_only == COMMIT_DIFF_STRATEGY:
         if len(client.repository.staged_changes) > 0 or len(client.repository.unstaged_changes) > 0:
             client.repository.reset()
@@ -73,6 +73,7 @@ def finalize_commit(
     raise_if_empty=False,
     commit_message=None,
     abbreviate_message=True,
+    skip_staging: bool = False,
 ):
     """Commit modified/added paths."""
     from renku.infrastructure.repository import Actor
@@ -127,8 +128,10 @@ def finalize_commit(
     if abbreviate_message:
         commit_message = shorten_message(commit_message)
 
+    # NOTE: Only commit specified paths when skipping staging area
+    paths = commit_only if skip_staging else []
     # Ignore pre-commit hooks since we have already done everything.
-    client.repository.commit(commit_message + client.transaction_id, committer=committer, no_verify=True)
+    client.repository.commit(commit_message + client.transaction_id, committer=committer, no_verify=True, paths=paths)
 
 
 def prepare_worktree(

renku/core/workflow/converters/cwl.py

@@ -267,7 +267,9 @@ def _convert_step(
                 dirents.append(path)
                 jsrequirement = True
 
-            environment_variables.append(cwl.EnvironmentDef(f"{RENKU_ENV_PREFIX}{output_.name}", output_.actual_value))
+            environment_variables.append(
+                cwl.EnvironmentDef(f"{RENKU_ENV_PREFIX}{output_.name}", str(output_.actual_value))
+            )
             outp, arg = CWLExporter._convert_output(output_)
             tool_object.outputs.append(outp)
             if arg:
@@ -280,15 +282,17 @@ def _convert_step(
                 cwl.Dirent(entry="$(inputs.{})".format(tool_input.id), entryname=input_.actual_value, writable=False)
             )
 
-            environment_variables.append(cwl.EnvironmentDef(f"{RENKU_ENV_PREFIX}{input_.name}", input_.actual_value))
+            environment_variables.append(
+                cwl.EnvironmentDef(f"{RENKU_ENV_PREFIX}{input_.name}", str(input_.actual_value))
+            )
             tool_object.inputs.append(tool_input)
             if input_.mapped_to:
                 tool_object.stdin = "$(inputs.{}.path)".format(tool_input.id)
                 jsrequirement = True
 
         for parameter in workflow.parameters:
             environment_variables.append(
-                cwl.EnvironmentDef(f"{RENKU_ENV_PREFIX}{parameter.name}", parameter.actual_value)
+                cwl.EnvironmentDef(f"{RENKU_ENV_PREFIX}{parameter.name}", str(parameter.actual_value))
             )
             tool_object.inputs.append(CWLExporter._convert_parameter(parameter))
 

renku/data/shacl_shape.json

@@ -1132,6 +1132,11 @@
                         "@id": "xsd:decimal"
                      }
                   },
+                  {
+                     "datatype": {
+                        "@id": "xsd:double"
+                     }
+                  },
                   {
                      "datatype": {
                         "@id": "xsd:integer"
@@ -1359,6 +1364,11 @@
                "nodeKind": "sh:Literal",
                "path": "schema:value",
                "or": [
+                  {
+                     "datatype": {
+                        "@id": "xsd:double"
+                     }
+                  },
                   {
                      "datatype": {
                         "@id": "xsd:decimal"

renku/infrastructure/repository.py

@@ -207,6 +207,7 @@ def commit(
         committer: "Actor" = None,
         no_verify: bool = False,
         no_edit: bool = False,
+        paths: Optional[List[Union[Path, str]]] = None,
     ) -> "Commit":
         """Commit added files to the VCS."""
         if self._repository is None:
@@ -217,7 +218,12 @@ def commit(
         if committer:
             env.update({"GIT_COMMITTER_NAME": committer.name, "GIT_COMMITTER_EMAIL": committer.email})
 
-        self.run_git_command("commit", message=message, no_verify=no_verify, amend=amend, no_edit=no_edit, env=env)
+        # NOTE: Only commit specified paths
+        args = ["--"] + [str(p) for p in paths] if paths else []
+
+        self.run_git_command(
+            "commit", *args, message=message, no_verify=no_verify, amend=amend, no_edit=no_edit, env=env
+        )
 
         return Commit.from_commit(self._repository, self._repository.head.commit)
 

renku/ui/cli/run.py

@@ -254,7 +254,6 @@
 
 import click
 
-from renku.command.options import option_isolation
 from renku.ui.cli.utils.callback import ClickCallback
 
 
@@ -276,7 +275,7 @@
     callback=lambda _, __, values: [int(value) % 256 for value in values],
     help="Allowed command exit-code.",
 )
-@option_isolation
+@click.option("--isolation", is_flag=True, default=False, help="Invoke the given command in isolation.")
 @click.argument("command_line", nargs=-1, required=True, type=click.UNPROCESSED)
 @click.option("--verbose", is_flag=True, default=False, help="Print generated plan after the execution.")
 def run(

renku/ui/service/gateways/gitlab_api_provider.py

@@ -58,7 +58,7 @@ def download_files_from_api(
 
         git_data = GitURL.parse(remote)
         try:
-            gl = gitlab.Gitlab(git_data.instance_url, private_token=token)
+            gl = gitlab.Gitlab(git_data.instance_url, oauth_token=token)
             project = gl.projects.get(f"{git_data.owner}/{git_data.name}")
         except gitlab.GitlabAuthenticationError:
             # NOTE: Invalid or expired tokens fail even on public projects. Let's give it a try without tokens

tests/cli/test_config.py

@@ -16,6 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Test ``config`` command."""
+import os
 import subprocess
 import sys
 from threading import Thread
@@ -272,3 +273,22 @@ def test_config_interpolation_is_disabled(client, runner, value):
 
     assert 0 == result.exit_code, format_result_exception(result)
     assert f"{value}\n" == result.output
+
+
+def test_config_commit(client, runner, data_repository, global_config_dir):
+    """Test config changes only commits the renku config file."""
+    commit_sha_before = client.repository.head.commit.hexsha
+
+    (client.path / "untracked").write_text("untracked")
+    (client.path / "staged").write_text("staged")
+    client.repository.add("staged")
+
+    result = runner.invoke(cli, ["config", "set", "key", "value"])
+
+    assert 0 == result.exit_code, format_result_exception(result)
+    assert {os.path.join(".renku", "renku.ini")} == {f.a_path for f in client.repository.head.commit.get_changes()}
+    assert {"untracked"} == set(client.repository.untracked_files)
+    assert {"staged"} == {f.a_path for f in client.repository.staged_changes}
+
+    commit_sha_after = client.repository.head.commit.hexsha
+    assert commit_sha_after != commit_sha_before