Merge branch 'main' into feature/#149-add-support-for-import-linter-t…

…o-lint-tasks

.github/workflows/report.yml

@@ -51,3 +51,4 @@ jobs:
           poetry run coverage report -- --format markdown >> $GITHUB_STEP_SUMMARY
           echo  -e "\n\n# Static Code Analysis\n" >> $GITHUB_STEP_SUMMARY
           cat .lint.txt >> $GITHUB_STEP_SUMMARY
+          poetry run tbx security pretty-print .security.json >> $GITHUB_STEP_SUMMARY

doc/changes/unreleased.md

@@ -1,5 +1,7 @@
 # Unreleased
 
-## ✨ Features
+## ✨ Added
 
-* #149: Added nox task to lint imports
+* #149: Added nox task to lint imports
+* #248: Added security results to workflow summary
+* #233: Added nox task to verify dependency declarations

doc/user_guide/getting_started.rst

@@ -193,6 +193,7 @@ You are ready to use the toolbox. With *nox -l* you can list all available tasks
     - lint:code -> Runs the static code analyzer on the project
     - lint:typing -> Runs the type checker on the project
     - lint:security -> Runs the security linter on the project
+    - lint:dependencies -> Checks if only valid sources of dependencies are used
     - docs:multiversion -> Builds the multiversion project documentation
     - docs:build -> Builds the project documentation
     - docs:open -> Opens the built project documentation

exasol/toolbox/nox/_lint.py

@@ -1,6 +1,10 @@
 from __future__ import annotations
 
-from typing import Iterable
+from typing import (
+    Iterable,
+    List,
+    Dict
+)
 import argparse
 from pathlib import Path
 
@@ -10,6 +14,11 @@
 from exasol.toolbox.nox._shared import python_files
 from noxconfig import PROJECT_CONFIG
 
+from pathlib import Path
+import rich.console
+import tomlkit
+import sys
+
 
 def _pylint(session: Session, files: Iterable[str]) -> None:
     session.run(
@@ -67,6 +76,7 @@ def _security_lint(session: Session, files: Iterable[str]) -> None:
     )
 
 
+
 def _import_lint(session: Session, path: Path) -> None:
     session.run(
         "poetry",
@@ -76,6 +86,60 @@ def _import_lint(session: Session, path: Path) -> None:
         path
     )
 
+class Dependencies:
+    def __init__(self, illegal: Dict[str, List[str]] | None):
+        self._illegal = illegal or {}
+
+    @staticmethod
+    def parse(pyproject_toml: str) -> "Dependencies":
+        def _source_filter(version) -> bool:
+            ILLEGAL_SPECIFIERS = ['url', 'git', 'path']
+            return any(
+                specifier in version
+                for specifier in ILLEGAL_SPECIFIERS
+            )
+
+        def find_illegal(part) -> List[str]:
+            return [
+                f"{name} = {version}"
+                for name, version in part.items()
+                if _source_filter(version)
+            ]
+
+        illegal: Dict[str, List[str]] = {}
+        toml = tomlkit.loads(pyproject_toml)
+        poetry = toml.get("tool", {}).get("poetry", {})
+
+        part = poetry.get("dependencies", {})
+        if illegal_group := find_illegal(part):
+            illegal["tool.poetry.dependencies"] = illegal_group
+
+        part = poetry.get("dev", {}).get("dependencies", {})
+        if illegal_group := find_illegal(part):
+            illegal["tool.poetry.dev.dependencies"] = illegal_group
+
+        part = poetry.get("group", {})
+        for group, content in part.items():
+            illegal_group = find_illegal(content.get("dependencies", {}))
+            if illegal_group:
+                illegal[f"tool.poetry.group.{group}.dependencies"] = illegal_group
+        return Dependencies(illegal)
+
+    @property
+    def illegal(self) -> Dict[str, List[str]]:
+        return self._illegal
+
+
+def report_illegal(illegal: Dict[str, List[str]], console: rich.console.Console):
+    count = sum(len(deps) for deps in illegal.values())
+    suffix = "y" if count == 1 else "ies"
+    console.print(f"{count} illegal dependenc{suffix}\n", style="red")
+    for section, dependencies in illegal.items():
+        console.print(f"\\[{section}]", style="red")
+        for dependency in dependencies:
+            console.print(dependency, style="red")
+        console.print("")
+
 
 @nox.session(name="lint:code", python=False)
 def lint(session: Session) -> None:
@@ -98,6 +162,16 @@ def security_lint(session: Session) -> None:
     _security_lint(session, list(filter(lambda file: "test" not in file, py_files)))
 
 
+@nox.session(name="lint:dependencies", python=False)
+def dependency_check(session: Session) -> None:
+    """Checks if only valid sources of dependencies are used"""
+    content = Path(PROJECT_CONFIG.root, "pyproject.toml").read_text()
+    dependencies = Dependencies.parse(content)
+    console = rich.console.Console()
+    if illegal := dependencies.illegal:
+        report_illegal(illegal, console)
+        sys.exit(1)
+
 @nox.session(name="lint:import", python=False)
 def import_lint(session: Session) -> None:
     """(experimental) Runs import linter on the project"""
@@ -124,5 +198,4 @@ def import_lint(session: Session) -> None:
         session.error(
             "Please make sure you have a configuration file for the importlinter"
         )
-    _import_lint(session=session, path=path)
-
+    _import_lint(session=session, path=path)

exasol/toolbox/nox/tasks.py

@@ -68,5 +68,6 @@ def check(session: Session) -> None:
     python_files,
 )
 
+
 # isort: on
 # fmt: on

exasol/toolbox/templates/github/workflows/report.yml

@@ -51,3 +51,4 @@ jobs:
           poetry run coverage report -- --format markdown >> $GITHUB_STEP_SUMMARY
           echo  -e "\n\n# Static Code Analysis\n" >> $GITHUB_STEP_SUMMARY
           cat .lint.txt >> $GITHUB_STEP_SUMMARY
+          poetry run tbx security pretty-print .security.json >> $GITHUB_STEP_SUMMARY

exasol/toolbox/tools/security.py

@@ -16,8 +16,8 @@
     Iterable,
     Tuple,
 )
-
 import typer
+from pathlib import Path
 
 stdout = print
 stderr = partial(print, file=sys.stderr)
@@ -101,6 +101,64 @@ def from_maven(report: str) -> Iterable[Issue]:
             )
 
 
+@dataclass(frozen=True)
+class SecurityIssue:
+    file_name: str
+    line: int
+    column: int
+    cwe: str
+    test_id: str
+    description: str
+    references: tuple
+
+
+def from_json(report_str: str, prefix: Path) -> Iterable[SecurityIssue]:
+    report = json.loads(report_str)
+    issues = report.get("results", {})
+    for issue in issues:
+        references = []
+        if issue["more_info"]:
+            references.append(issue["more_info"])
+        if issue.get("issue_cwe", {}).get("link", None):
+            references.append(issue["issue_cwe"]["link"])
+        yield SecurityIssue(
+            file_name=issue["filename"].replace(str(prefix) + "/", ""),
+            line=issue["line_number"],
+            column=issue["col_offset"],
+            cwe=str(issue["issue_cwe"].get("id", "")),
+            test_id=issue["test_id"],
+            description=issue["issue_text"],
+            references=tuple(references)
+        )
+
+
+def issues_to_markdown(issues: Iterable[SecurityIssue]) -> str:
+    template = cleandoc("""
+        {header}{rows}
+    """)
+
+    def _header():
+        header = "# Security\n\n"
+        header += "|File|line/<br>column|Cwe|Test ID|Details|\n"
+        header += "|---|:-:|:-:|:-:|---|\n"
+        return header
+
+    def _row(issue):
+        row = "|" + issue.file_name + "|"
+        row += f"line: {issue.line}<br>column: {issue.column}|"
+        row += issue.cwe + "|"
+        row += issue.test_id + "|"
+        for element in issue.references:
+            row += element + " ,<br>"
+        row = row[:-5] + "|"
+        return row
+
+    return template.format(
+        header=_header(),
+        rows="\n".join(_row(i) for i in issues)
+    )
+
+
 def security_issue_title(issue: Issue) -> str:
     return f"🔐 {issue.cve}: {issue.coordinates}"
 
@@ -159,7 +217,6 @@ def create_security_issue(issue: Issue, project="") -> Tuple[str, str]:
 CVE_CLI = typer.Typer()
 CLI.add_typer(CVE_CLI, name="cve", help="Work with CVE's")
 
-
 class Format(str, Enum):
     Maven = "maven"
 
@@ -257,6 +314,21 @@ def create(
         stdout(format_jsonl(issue_url, issue))
 
 
+class PPrintFormats(str, Enum):
+    markdown = "markdown"
+
+
+@CLI.command(name="pretty-print")
+def json_issue_to_markdown(
+        json_file: typer.FileText = typer.Argument(mode="r", help="json file with issues to convert"),
+        path: Path = typer.Argument(default=Path("."), help="path to project root")
+) -> None:
+    content = json_file.read()
+    issues = from_json(content, path.absolute())
+    issues = sorted(issues, key=lambda i: (i.file_name, i.cwe, i.test_id))
+    print(issues_to_markdown(issues))
+
+
 def format_jsonl(issue_url: str, issue: Issue) -> str:
     issue_json = asdict(issue)
     issue_json["issue_url"] = issue_url.strip()

test/integration/cli/security-pprint-emty.t

@@ -0,0 +1,18 @@
+Create test input
+
+  $ cat > .security.json <<EOF
+  > {
+  >   "result":[
+  >   ]
+  > }
+  > EOF
+
+Run test case
+
+  $ tbx security pretty-print .security.json
+  # Security
+
+  |File|line/<br>column|Cwe|Test ID|Details|
+  |---|:-:|:-:|:-:|---|
+
+

test/integration/cli/security-pprint.t

@@ -0,0 +1,81 @@
+Create test input
+
+  $ cat > .security.json <<EOF
+  > {
+  >   "results":[
+  >     {
+  >       "code": "555                 subprocess.check_call(\n556                     config.smv_postbuild_command, cwd=current_cwd, shell=True\n557                 )\n558                 if config.smv_postbuild_export_pattern != \"\":\n559                     matches = find_matching_files_and_dirs(\n",
+  >       "col_offset": 16,
+  >       "end_col_offset": 17,
+  >       "filename": "exasol/toolbox/sphinx/multiversion/main.py",
+  >       "issue_confidence": "HIGH",
+  >       "issue_cwe": {
+  >         "id": 78,
+  >         "link": "https://cwe.mitre.org/data/definitions/78.html"
+  >       },
+  >       "issue_severity": "HIGH",
+  >       "issue_text": "subprocess call with shell=True identified, security issue.",
+  >       "line_number": 556,
+  >       "line_range": [
+  >         555,
+  >         556,
+  >         557
+  >       ],
+  >       "more_info": "https://bandit.readthedocs.io/en/1.7.10/plugins/b602_subprocess_popen_with_shell_equals_true.html",
+  >       "test_id": "B602",
+  >       "test_name": "subprocess_popen_with_shell_equals_true"
+  >     },
+  >     {
+  >       "code": "156         )\n157         subprocess.check_call(cmd, cwd=gitroot, stdout=fp)\n158         fp.seek(0)\n",
+  >       "col_offset": 8,
+  >       "end_col_offset": 58,
+  >       "filename": "exasol/toolbox/sphinx/multiversion/git.py",
+  >       "issue_confidence": "HIGH",
+  >       "issue_cwe": {
+  >         "id": 78,
+  >         "link": "https://cwe.mitre.org/data/definitions/78.html"
+  >       },
+  >       "issue_severity": "LOW",
+  >       "issue_text": "subprocess call - check for execution of untrusted input.",
+  >       "line_number": 157,
+  >       "line_range": [
+  >         157
+  >       ],
+  >       "more_info": "https://bandit.readthedocs.io/en/1.7.10/plugins/b603_subprocess_without_shell_equals_true.html",
+  >       "test_id": "B603",
+  >       "test_name": "subprocess_without_shell_equals_true"
+  >     },
+  >     {
+  >       "code": "159         with tarfile.TarFile(fileobj=fp) as tarfp:\n160             tarfp.extractall(dst)\n",
+  >       "col_offset": 12,
+  >       "end_col_offset": 33,
+  >       "filename": "exasol/toolbox/sphinx/multiversion/git.py",
+  >       "issue_confidence": "HIGH",
+  >       "issue_cwe": {
+  >         "id": 22,
+  >         "link": "https://cwe.mitre.org/data/definitions/22.html"
+  >       },
+  >       "issue_severity": "HIGH",
+  >       "issue_text": "tarfile.extractall used without any validation. Please check and discard dangerous members.",
+  >       "line_number": 160,
+  >       "line_range": [
+  >         160
+  >       ],
+  >       "more_info": "https://bandit.readthedocs.io/en/1.7.10/plugins/b202_tarfile_unsafe_members.html",
+  >       "test_id": "B202",
+  >       "test_name": "tarfile_unsafe_members"
+  >     }
+  >   ]
+  > }
+  > EOF
+
+Run test case
+
+  $ tbx security pretty-print .security.json
+  # Security
+
+  |File|line/<br>column|Cwe|Test ID|Details|
+  |---|:-:|:-:|:-:|---|
+  |exasol/toolbox/sphinx/multiversion/git.py|line: 160<br>column: 12|22|B202|https://bandit.readthedocs.io/en/1.7.10/plugins/b202_tarfile_unsafe_members.html ,<br>https://cwe.mitre.org/data/definitions/22.html |
+  |exasol/toolbox/sphinx/multiversion/git.py|line: 157<br>column: 8|78|B603|https://bandit.readthedocs.io/en/1.7.10/plugins/b603_subprocess_without_shell_equals_true.html ,<br>https://cwe.mitre.org/data/definitions/78.html |
+  |exasol/toolbox/sphinx/multiversion/main.py|line: 556<br>column: 16|78|B602|https://bandit.readthedocs.io/en/1.7.10/plugins/b602_subprocess_popen_with_shell_equals_true.html ,<br>https://cwe.mitre.org/data/definitions/78.html |