Merge pull request #503 from opencybersecurityalliance/develop

v1.8.3
opencybersecurityalliance · Apr 22, 2024 · ec1e9d3 · ec1e9d3
2 parents 5059b90 + 33955f7
commit ec1e9d3
Show file tree

Hide file tree

Showing 85 changed files with 2,548 additions and 1,107 deletions.
diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml
@@ -22,9 +22,9 @@ jobs:
   codecov:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.10'
       - name: Install Python Tools

diff --git a/.github/workflows/code-style.yml b/.github/workflows/code-style.yml
@@ -22,9 +22,9 @@ jobs:
   codestyle:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
           python-version: '3.10'
       - name: Install Kestrel package

diff --git a/.github/workflows/kaas-docker-image.yml b/.github/workflows/kaas-docker-image.yml
@@ -14,7 +14,7 @@ jobs:
         run: sleep 600s
         shell: bash
       - name: Checkout
-        uses: actions/checkout@v3.5.3
+        uses: actions/checkout@v4
       - name: Info
         run: echo "Parameters. ${{ github.event.base_ref }}, ${{ github.ref_type }}, ${{ github.ref }}"
       - name: Log in to Docker Hub

diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml
@@ -22,8 +22,8 @@ jobs:
         shell: bash
         working-directory: ./packages/${{ matrix.package }}
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.10'
       - name: Install building environment

diff --git a/.github/workflows/stixshifter-module-verification.yml b/.github/workflows/stixshifter-module-verification.yml
@@ -15,9 +15,9 @@ jobs:
         shell: bash
         working-directory: ./packages/kestrel_datasource_stixshifter
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
           python-version: '3.10'
       - name: Install Python Tools

diff --git a/.github/workflows/unit-testing-kestrel2.yml b/.github/workflows/unit-testing-kestrel2.yml
@@ -30,9 +30,9 @@ jobs:
         shell: bash
         working-directory: ./packages-nextgen/kestrel_core
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install Python Tools
@@ -42,3 +42,28 @@ jobs:
       - name: Unit testing
         run: pytest -vv
 
+  test-kestrel-interface-opensearch:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash
+        working-directory: ./packages-nextgen/kestrel_interface_opensearch
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Python Tools
+        run: pip install --upgrade pip setuptools wheel pytest
+      - name: Install kestrel_core
+        working-directory: ./packages-nextgen/kestrel_core
+        run: pip install .
+      - name: Install kestrel_interface_opensearch
+        run: pip install .
+      - name: Unit testing
+        run: pytest -vv
diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml
@@ -23,16 +23,16 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
     runs-on: ${{ matrix.os }}
     defaults:
       run:
         shell: bash
         working-directory: ./packages/kestrel_core
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install Python Tools
@@ -52,16 +52,16 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11.6']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
     runs-on: ${{ matrix.os }}
     defaults:
       run:
         shell: bash
         working-directory: ./packages/kestrel_datasource_stixshifter
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install Python Tools
@@ -78,16 +78,16 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
     runs-on: ${{ matrix.os }}
     defaults:
       run:
         shell: bash
         working-directory: ./packages/kestrel_analytics_python
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install Python Tools
@@ -107,16 +107,16 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
     runs-on: ${{ matrix.os }}
     defaults:
       run:
         shell: bash
         working-directory: ./packages/kestrel_jupyter
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install Python Tools

diff --git a/.github/workflows/unused-import.yml b/.github/workflows/unused-import.yml
@@ -22,9 +22,9 @@ jobs:
   unusedimports:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
           python-version: '3.10'
       - name: Install Kestrel package

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -9,6 +9,22 @@ The format is based on `Keep a Changelog`_.
 Unreleased
 ==========
 
+1.8.3 (2024-04-22)
+==================
+
+Added
+-----
+
+- Support of disabling certificate verification of stix-shifter v7 with config option `verify_cert`
+- Documentation on how to use the `verify_cert` option in the stix-shifter interface
+- Python 3.12 support (multiprocessing library behavior steering to avoid a CPU-blocking issue)
+- More generic HTML parsing of PyPI for stix-shfiter connector verification
+
+Changed
+-------
+
+- stix-shifter upgraded to v7 (v7.0.6), the first version abandoning invalid certificate support
+
 1.8.2 (2024-02-20)
 ==================
 

diff --git a/README.rst b/README.rst
@@ -2,31 +2,11 @@
    :width: 460
    :alt: Kestrel Threat Hunting Language
 
-.. image:: https://readthedocs.org/projects/kestrel/badge/?version=latest
-        :target: https://kestrel.readthedocs.io/en/latest/?badge=latest
-        :alt: Documentation Status
-
-.. image:: https://img.shields.io/pypi/v/kestrel-jupyter
-        :target: https://pypi.python.org/pypi/kestrel-jupyter
-        :alt: Latest Version
-
-.. image:: https://img.shields.io/pypi/dm/kestrel-core
-        :target: https://pypistats.org/packages/kestrel-core
-        :alt: PyPI Downloads
-
-.. image:: https://codecov.io/gh/opencybersecurityalliance/kestrel-lang/branch/develop/graph/badge.svg?token=HM4ax10IW3
-        :target: https://codecov.io/gh/opencybersecurityalliance/kestrel-lang
-        :alt: Code Coverage
-
-.. image:: https://img.shields.io/badge/code%20style-black-000000.svg
-        :target: https://github.com/psf/black
-        :alt: Code Style: Black
-
 |
 
-**[News]** Kestrel session at `Black Hat USA 2023`_
+|readthedocs| |pypi| |downloads| |codecoverage| |black|
 
---------
+|
 
 Kestrel is a threat hunting language aiming to make cyber threat hunting *fast*
 by providing a layer of abstraction to build reusable, composable, and
@@ -215,3 +195,24 @@ Connecting With The Community
 .. _contributing guideline: CONTRIBUTING.rst
 .. _governance documentation: GOVERNANCE.rst
 .. _Apache License 2.0: LICENSE.md
+
+
+.. |readthedocs| image:: https://readthedocs.org/projects/kestrel/badge/?version=latest
+        :target: https://kestrel.readthedocs.io/en/latest/?badge=latest
+        :alt: Documentation Status
+
+.. |pypi| image:: https://img.shields.io/pypi/v/kestrel-jupyter
+        :target: https://pypi.python.org/pypi/kestrel-jupyter
+        :alt: Latest Version
+
+.. |downloads| image:: https://img.shields.io/pypi/dm/kestrel-core
+        :target: https://pypistats.org/packages/kestrel-core
+        :alt: PyPI Downloads
+
+.. |codecoverage| image:: https://codecov.io/gh/opencybersecurityalliance/kestrel-lang/branch/develop/graph/badge.svg?token=HM4ax10IW3
+        :target: https://codecov.io/gh/opencybersecurityalliance/kestrel-lang
+        :alt: Code Coverage
+
+.. |black| image:: https://img.shields.io/badge/code%20style-black-000000.svg
+        :target: https://github.com/psf/black
+        :alt: Code Style: Black
diff --git a/docs/installation/runtime.rst b/docs/installation/runtime.rst
@@ -8,7 +8,11 @@ please use Python inside Windows Subsystem for Linux (WSL).
 General Requirements
 ====================
 
-Python 3.8 is required. Follow the `Python installation guide`_ to install or upgrade Python.
+Python 3 is required.
+
+* End-of-life Python versions are not supported. Check `Python releases`_.
+
+* Follow the `Python installation guide`_ to install or upgrade Python.
 
 OS-specific Requirements
 ========================
@@ -190,6 +194,7 @@ What's to Do Next
 - :doc:`../language/index`
 
 .. _Python installation guide: http://docs.python-guide.org/en/latest/starting/installation/
+.. _Python releases: https://devguide.python.org/versions/
 .. _Python virtual environment: https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/
 .. _Xcode: https://developer.apple.com/xcode/
 .. _kestrel-lang: http://github.com/opencybersecurityalliance/kestrel-lang

diff --git a/packages-nextgen/kestrel_core/pyproject.toml b/packages-nextgen/kestrel_core/pyproject.toml
@@ -38,6 +38,7 @@ dependencies = [
     "mashumaro>=3.10",
     "networkx>=3.1", # networkx==3.2.1 only for Python>=3.9
     "SQLAlchemy>=2.0.23",
+    "dpath>=2.1.6",
 ]
 
 [project.optional-dependencies]

diff --git a/packages-nextgen/kestrel_core/src/kestrel/cache/base.py b/packages-nextgen/kestrel_core/src/kestrel/cache/base.py
@@ -1,23 +1,24 @@
+from __future__ import annotations
 from pandas import DataFrame
 from typing import MutableMapping
 from uuid import UUID
 from abc import abstractmethod
 
 from kestrel.config.internal import CACHE_INTERFACE_IDENTIFIER
-from kestrel.interface.datasource import AbstractDataSourceInterface
+from kestrel.interface import AbstractInterface
 
 
-class AbstractCache(AbstractDataSourceInterface, MutableMapping):
+class AbstractCache(AbstractInterface, MutableMapping):
     """Base class for Kestrel cache
 
-    Additional @abstractmethod from AbstractDataSourceInterface:
+    Additional @abstractmethod from AbstractInterface:
 
         - evaluate_graph()
     """
 
-    @property
-    def name(self):
-        return CACHE_INTERFACE_IDENTIFIER
+    @staticmethod
+    def schemes() -> Iterable[str]:
+        return [CACHE_INTERFACE_IDENTIFIER]
 
     @abstractmethod
     def __del__(self):
@@ -28,6 +29,8 @@ def __del__(self):
     def __getitem__(self, instruction_id: UUID) -> DataFrame:
         """Get the dataframe for the cached instruction
 
+        This method will automatically support `uuid in cache`
+
         Parameters:
             instruction_id: id of the instruction
 
@@ -57,16 +60,32 @@ def __delitem__(self, instruction_id: UUID):
         """
         ...
 
-    def store(self, instruction_id: UUID, data: DataFrame):
-        self[instruction_id] = data
+    @abstractmethod
+    def get_virtual_copy(self) -> AbstractCache:
+        """Create a virtual cache object from this cache
 
-    def __contain__(self, instruction_id: UUID) -> bool:
-        """Whether the evaluated instruction is cached
+        This method needs to reimplement __del__, __getitem__, __setitem__,
+        __delitem__ to not actually hit the store media, e.g., SQLite.
 
-        Parameters:
-            instruction_id: id of the instruction
+        The virtual cache is useful for the implementation of the Explain()
+        instruction, pretending the dependent graphs are evaluated, so the
+        evaluation can continue towards the Return() instruction.
+
+        Because Python invokes special methods from class methods, replacing
+        the __getitem__, __setitem__, and __delitem__ in the object does not
+        help. It is better to derive a subclass and replace __class__ of the
+        object to the subclass to correctly invoke the new set of __xitem___.
+
+        https://docs.python.org/3/reference/datamodel.html#special-lookup
+
+        And Python garbage collector could clean up the virtual cache when
+        not in use, so the __del__ method should be reimplemented to make
+        sure the store media is not closed.
         """
-        return instruction_id in self.cache_catalog
+        ...
+
+    def store(self, instruction_id: UUID, data: DataFrame):
+        self[instruction_id] = data
 
     def __iter__(self) -> UUID:
         """Return UUIDs of instructions cached