Merge pull request #503 from opencybersecurityalliance/develop

v1.8.3

.github/workflows/code-coverage.yml

@@ -22,9 +22,9 @@ jobs:
  codecov:
  runs-on: ubuntu-latest
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
  - name: Set up Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
  with:
  python-version: '3.10'
  - name: Install Python Tools

.github/workflows/code-style.yml

@@ -22,9 +22,9 @@ jobs:
  codestyle:
  runs-on: ubuntu-latest
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
  - name: Set up Python
- uses: actions/setup-python@v3
+ uses: actions/setup-python@v5
  with:
  python-version: '3.10'
  - name: Install Kestrel package

.github/workflows/kaas-docker-image.yml

@@ -14,7 +14,7 @@ jobs:
  run: sleep 600s
  shell: bash
  - name: Checkout
- uses: actions/checkout@v3.5.3
+ uses: actions/checkout@v4
  - name: Info
  run: echo "Parameters. ${{ github.event.base_ref }}, ${{ github.ref_type }}, ${{ github.ref }}"
  - name: Log in to Docker Hub

.github/workflows/publish-to-pypi.yml

@@ -22,8 +22,8 @@ jobs:
  shell: bash
  working-directory: ./packages/${{ matrix.package }}
  steps:
- - uses: actions/checkout@v3
- - uses: actions/setup-python@v3
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
  with:
  python-version: '3.10'
  - name: Install building environment

.github/workflows/stixshifter-module-verification.yml

@@ -15,9 +15,9 @@ jobs:
  shell: bash
  working-directory: ./packages/kestrel_datasource_stixshifter
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
  - name: Set up Python
- uses: actions/setup-python@v3
+ uses: actions/setup-python@v5
  with:
  python-version: '3.10'
  - name: Install Python Tools

.github/workflows/unit-testing-kestrel2.yml

@@ -30,9 +30,9 @@ jobs:
  shell: bash
  working-directory: ./packages-nextgen/kestrel_core
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
  - name: Set up Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
  with:
  python-version: ${{ matrix.python-version }}
  - name: Install Python Tools
@@ -42,3 +42,28 @@ jobs:
  - name: Unit testing
  run: pytest -vv
 
+ test-kestrel-interface-opensearch:
+ strategy:
+ matrix:
+ os: [ubuntu-latest, macos-latest]
+ python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+ runs-on: ${{ matrix.os }}
+ defaults:
+ run:
+ shell: bash
+ working-directory: ./packages-nextgen/kestrel_interface_opensearch
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install Python Tools
+ run: pip install --upgrade pip setuptools wheel pytest
+ - name: Install kestrel_core
+ working-directory: ./packages-nextgen/kestrel_core
+ run: pip install .
+ - name: Install kestrel_interface_opensearch
+ run: pip install .
+ - name: Unit testing
+ run: pytest -vv

.github/workflows/unit-testing.yml

@@ -23,16 +23,16 @@ jobs:
  strategy:
  matrix:
  os: [ubuntu-latest, macos-latest]
- python-version: ['3.8', '3.9', '3.10', '3.11']
+ python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
  runs-on: ${{ matrix.os }}
  defaults:
  run:
  shell: bash
  working-directory: ./packages/kestrel_core
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
  - name: Set up Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
  with:
  python-version: ${{ matrix.python-version }}
  - name: Install Python Tools
@@ -52,16 +52,16 @@ jobs:
  strategy:
  matrix:
  os: [ubuntu-latest, macos-latest]
- python-version: ['3.8', '3.9', '3.10', '3.11.6']
+ python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
  runs-on: ${{ matrix.os }}
  defaults:
  run:
  shell: bash
  working-directory: ./packages/kestrel_datasource_stixshifter
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
  - name: Set up Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
  with:
  python-version: ${{ matrix.python-version }}
  - name: Install Python Tools
@@ -78,16 +78,16 @@ jobs:
  strategy:
  matrix:
  os: [ubuntu-latest, macos-latest]
- python-version: ['3.8', '3.9', '3.10', '3.11']
+ python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
  runs-on: ${{ matrix.os }}
  defaults:
  run:
  shell: bash
  working-directory: ./packages/kestrel_analytics_python
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
  - name: Set up Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
  with:
  python-version: ${{ matrix.python-version }}
  - name: Install Python Tools
@@ -107,16 +107,16 @@ jobs:
  strategy:
  matrix:
  os: [ubuntu-latest, macos-latest]
- python-version: ['3.8', '3.9', '3.10', '3.11']
+ python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
  runs-on: ${{ matrix.os }}
  defaults:
  run:
  shell: bash
  working-directory: ./packages/kestrel_jupyter
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
  - name: Set up Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
  with:
  python-version: ${{ matrix.python-version }}
  - name: Install Python Tools

.github/workflows/unused-import.yml

@@ -22,9 +22,9 @@ jobs:
  unusedimports:
  runs-on: ubuntu-latest
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
  - name: Set up Python
- uses: actions/setup-python@v3
+ uses: actions/setup-python@v5
  with:
  python-version: '3.10'
  - name: Install Kestrel package

CHANGELOG.rst

@@ -9,6 +9,22 @@ The format is based on `Keep a Changelog`_.
 Unreleased
 ==========
 
+1.8.3 (2024-04-22)
+==================
+
+Added
+-----
+
+- Support of disabling certificate verification of stix-shifter v7 with config option `verify_cert`
+- Documentation on how to use the `verify_cert` option in the stix-shifter interface
+- Python 3.12 support (multiprocessing library behavior steering to avoid a CPU-blocking issue)
+- More generic HTML parsing of PyPI for stix-shfiter connector verification
+
+Changed
+-------
+
+- stix-shifter upgraded to v7 (v7.0.6), the first version abandoning invalid certificate support
+
 1.8.2 (2024-02-20)
 ==================
 

README.rst

@@ -2,31 +2,11 @@
  :width: 460
  :alt: Kestrel Threat Hunting Language
 
-.. image:: https://readthedocs.org/projects/kestrel/badge/?version=latest
- :target: https://kestrel.readthedocs.io/en/latest/?badge=latest
- :alt: Documentation Status
-
-.. image:: https://img.shields.io/pypi/v/kestrel-jupyter
- :target: https://pypi.python.org/pypi/kestrel-jupyter
- :alt: Latest Version
-
-.. image:: https://img.shields.io/pypi/dm/kestrel-core
- :target: https://pypistats.org/packages/kestrel-core
- :alt: PyPI Downloads
-
-.. image:: https://codecov.io/gh/opencybersecurityalliance/kestrel-lang/branch/develop/graph/badge.svg?token=HM4ax10IW3
- :target: https://codecov.io/gh/opencybersecurityalliance/kestrel-lang
- :alt: Code Coverage
-
-.. image:: https://img.shields.io/badge/code%20style-black-000000.svg
- :target: https://github.com/psf/black
- :alt: Code Style: Black
-
 |
 
-**[News]** Kestrel session at `Black Hat USA 2023`_
+|readthedocs| |pypi| |downloads| |codecoverage| |black|
 
---------
+|
 
 Kestrel is a threat hunting language aiming to make cyber threat hunting *fast*
 by providing a layer of abstraction to build reusable, composable, and
@@ -215,3 +195,24 @@ Connecting With The Community
 .. _contributing guideline: CONTRIBUTING.rst
 .. _governance documentation: GOVERNANCE.rst
 .. _Apache License 2.0: LICENSE.md
+
+
+.. |readthedocs| image:: https://readthedocs.org/projects/kestrel/badge/?version=latest
+ :target: https://kestrel.readthedocs.io/en/latest/?badge=latest
+ :alt: Documentation Status
+
+.. |pypi| image:: https://img.shields.io/pypi/v/kestrel-jupyter
+ :target: https://pypi.python.org/pypi/kestrel-jupyter
+ :alt: Latest Version
+
+.. |downloads| image:: https://img.shields.io/pypi/dm/kestrel-core
+ :target: https://pypistats.org/packages/kestrel-core
+ :alt: PyPI Downloads
+
+.. |codecoverage| image:: https://codecov.io/gh/opencybersecurityalliance/kestrel-lang/branch/develop/graph/badge.svg?token=HM4ax10IW3
+ :target: https://codecov.io/gh/opencybersecurityalliance/kestrel-lang
+ :alt: Code Coverage
+
+.. |black| image:: https://img.shields.io/badge/code%20style-black-000000.svg
+ :target: https://github.com/psf/black
+ :alt: Code Style: Black

docs/installation/runtime.rst

@@ -8,7 +8,11 @@ please use Python inside Windows Subsystem for Linux (WSL).
 General Requirements
 ====================
 
-Python 3.8 is required. Follow the `Python installation guide`_ to install or upgrade Python.
+Python 3 is required.
+
+* End-of-life Python versions are not supported. Check `Python releases`_.
+
+* Follow the `Python installation guide`_ to install or upgrade Python.
 
 OS-specific Requirements
 ========================
@@ -190,6 +194,7 @@ What's to Do Next
 - :doc:`../language/index`
 
 .. _Python installation guide: http://docs.python-guide.org/en/latest/starting/installation/
+.. _Python releases: https://devguide.python.org/versions/
 .. _Python virtual environment: https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/
 .. _Xcode: https://developer.apple.com/xcode/
 .. _kestrel-lang: http://github.com/opencybersecurityalliance/kestrel-lang

packages-nextgen/kestrel_core/pyproject.toml

@@ -38,6 +38,7 @@ dependencies = [
  "mashumaro>=3.10",
  "networkx>=3.1", # networkx==3.2.1 only for Python>=3.9
  "SQLAlchemy>=2.0.23",
+ "dpath>=2.1.6",
 ]
 
 [project.optional-dependencies]

packages-nextgen/kestrel_core/src/kestrel/cache/base.py

@@ -1,23 +1,24 @@
+from __future__ import annotations
 from pandas import DataFrame
 from typing import MutableMapping
 from uuid import UUID
 from abc import abstractmethod
 
 from kestrel.config.internal import CACHE_INTERFACE_IDENTIFIER
-from kestrel.interface.datasource import AbstractDataSourceInterface
+from kestrel.interface import AbstractInterface
 
 
-class AbstractCache(AbstractDataSourceInterface, MutableMapping):
+class AbstractCache(AbstractInterface, MutableMapping):
  """Base class for Kestrel cache
 
- Additional @abstractmethod from AbstractDataSourceInterface:
+ Additional @abstractmethod from AbstractInterface:
 
  - evaluate_graph()
  """
 
- @property
- def name(self):
- return CACHE_INTERFACE_IDENTIFIER
+ @staticmethod
+ def schemes() -> Iterable[str]:
+ return [CACHE_INTERFACE_IDENTIFIER]
 
  @abstractmethod
  def __del__(self):
@@ -28,6 +29,8 @@ def __del__(self):
  def __getitem__(self, instruction_id: UUID) -> DataFrame:
  """Get the dataframe for the cached instruction
 
+ This method will automatically support `uuid in cache`
+
  Parameters:
  instruction_id: id of the instruction
 
@@ -57,16 +60,32 @@ def __delitem__(self, instruction_id: UUID):
  """
  ...
 
- def store(self, instruction_id: UUID, data: DataFrame):
- self[instruction_id] = data
+ @abstractmethod
+ def get_virtual_copy(self) -> AbstractCache:
+ """Create a virtual cache object from this cache
 
- def __contain__(self, instruction_id: UUID) -> bool:
- """Whether the evaluated instruction is cached
+  This method needs to reimplement __del__, __getitem__, __setitem__,
+ __delitem__ to not actually hit the store media, e.g., SQLite.
 
- Parameters:
- instruction_id: id of the instruction
+ The virtual cache is useful for the implementation of the Explain()
+ instruction, pretending the dependent graphs are evaluated, so the
+ evaluation can continue towards the Return() instruction.
+
+ Because Python invokes special methods from class methods, replacing
+ the __getitem__, __setitem__, and __delitem__ in the object does not
+ help. It is better to derive a subclass and replace __class__ of the
+ object to the subclass to correctly invoke the new set of __xitem___.
+
+ https://docs.python.org/3/reference/datamodel.html#special-lookup
+
+ And Python garbage collector could clean up the virtual cache when
+ not in use, so the __del__ method should be reimplemented to make
+ sure the store media is not closed.
  """
- return instruction_id in self.cache_catalog
+ ...
+
+ def store(self, instruction_id: UUID, data: DataFrame):
+ self[instruction_id] = data
 
  def __iter__(self) -> UUID:
  """Return UUIDs of instructions cached