From 401e1d83c3627e707976726c9f0fe04d956b3feb Mon Sep 17 00:00:00 2001 From: Md Azam Date: Fri, 28 Jul 2023 11:57:53 -0300 Subject: [PATCH 01/55] Add readthedocs configurations --- .readthedocs.yaml | 32 ++++++++++ docs/Makefile | 20 +++++++ docs/README.md | 132 ++++++++++++++++++++++++++++++++++++++++++ docs/conf.py | 28 +++++++++ docs/index.rst | 28 +++++++++ docs/make.bat | 35 +++++++++++ docs/requirements.txt | 1 + 7 files changed, 276 insertions(+) create mode 100644 .readthedocs.yaml create mode 100644 docs/Makefile create mode 100644 docs/README.md create mode 100644 docs/conf.py create mode 100644 docs/index.rst create mode 100644 docs/make.bat create mode 100644 docs/requirements.txt diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..59e22f086 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,32 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + # You can also specify other tool versions: + # nodejs: "19" + # rust: "1.64" + # golang: "1.19" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/conf.py + +# Optionally build your docs in additional formats such as PDF and ePub +formats: + - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: docs/requirements.txt \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 000000000..d4bb2cbb9 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 000000000..bd4c8ef9a --- /dev/null +++ b/docs/README.md @@ -0,0 +1,132 @@ +[![example workflow](https://github.com/opencybersecurityalliance/stix-shifter/actions/workflows/main.yml/badge.svg)](https://github.com/opencybersecurityalliance/stix-shifter/actions) +[![codecov](https://codecov.io/gh/opencybersecurityalliance/stix-shifter/branch/develop/graph/badge.svg?token=gQvl14peRj)](https://codecov.io/gh/opencybersecurityalliance/stix-shifter) + +# Introduction to STIX-Shifter + +STIX-shifter is an open source python library allowing software to connect to products that house data repositories by using STIX Patterning, and return results as STIX Observations. + +This library takes in STIX 2 Patterns as input, and "finds" data that matches the patterns inside various products that house repositories of cybersecurity data. Examples of such products include SIEM systems, endpoint management systems, threat intelligence platforms, orchestration platforms, network control points, data lakes, and more. + +In addition to "finding" the data by using these patterns, STIX-Shifter also _transforms the output_ into STIX 2 Observations. Why would we do that you ask? To put it simply - so that all of the security data, regardless of the source, mostly looks and behaves the same. + +## Project Overview and CLI Commands + +For general information about STIX, this project, and the command line utilities, see the [STIX-shifter Overview](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/OVERVIEW.md) + + +# Installation + +The recommended method for installing stix-shifter is via pip. Two prerequisite packages needs to be installed inlcuding the package of stix-shifter connector module to complete a stix-shifter connector installation. Run the below commands to install all the packages: + +1. Main stix-shifter package: `pip install stix-shifter` + +2. Stix-shifter Utility package: `pip install stix-shifter-utils` + +3. Desired stix-shifter connector module package: `pip install stix-shifter-modules- ` + Example: `pip install stix-shifter-modules-qradar` + +## Dependencies + +STIX-shifter requries Python 3.8 or greater. See the [requirements file](stix_shifter/requirements.txt) for library dependencies. + +# Usage + +STIX-Shifter can use used the following ways: + +## 1. As a command line utility + +The STIX-Shifter comes with a bundled script which you can use to translate STIX Pattern to a native datasource query. It can also be used to translate a JSON data source query result to a STIX bundle of observable objects. You can also send query to a datasource by using a transmission option. + +More details of the command line option can be found [here](https://github.com/opencybersecurityalliance/stix-shifter/blob/master/OVERVIEW.md#how-to-use) + +``` +$ stix-shifter translate query "" "" "" +``` +Example: +``` +$ stix-shifter translate qradar query {} "[ipv4-addr:value = '127.0.0.1']" {} +``` + +In order to build `stix-shifter` packages from source follow the below prerequisite steps: + 1. Go to the stix-shifter parent directory + 2. Optionally, you can create a Python 3 virtual environemnt: + `virtualenv -p python3 virtualenv && source virtualenv/bin/activate` + 3. Run setup: `python3 setup.py install` + + +## 2. Running from the source + +You may also use the `python3 main.py` script. All the options are the same as the command line utility described above. + +Example: + +``` +python3 main.py translate qradar query {} "[ipv4-addr:value = '127.0.0.1']" {} +``` + +In order to run `python3 main.py` from the source follow the below prerequisite steps: + 1. Go to the stix-shifter parent directory + 2. Optionally, you can create a Python 3 virtual environemnt: + `virtualenv -p python3 virtualenv && source virtualenv/bin/activate` + 3. Run setup to install dependancies: `INSTALL_REQUIREMENTS_ONLY=1 python3 setup.py install`. + +**Note:** setup.py only installs dependencies when INSTALL_REQUIREMENTS_ONLY=1 directive is used. This option is similar to `python3 generate_requirements.py && pip install -r requirements.txt` + +## 3. As a library + +You can also use this library to integrate STIX Shifter into your own tools. You can translate a STIX Pattern: + +``` +from stix_shifter.stix_translation import stix_translation + +translation = stix_translation.StixTranslation() +response = translation.translate('', 'query', '{}', '', '') + +print(response) +``` +### Use of custom mappings + +If a connector has been installed using pip, the process for editing the STIX mappings is different than if you have pulled-down the project. When working locally, you can edit the mapping files directly. See the [mapping files for the MySQL connector](stix_shifter_modules/mysql/stix_translation/json) as an example. Editing the mapping files won't work if the connector has been installed with pip; the setup script of the stix-shifter package includes the mapppings inside `config.json`. This allows stix-shifter to injest custom mappings as part of the connector's configuration. + +Refer to [Use of custom mappings](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/adapter-guide/custom_mappings.md) for more details on how to edit the mappings in the configuration. + +# Contributing + +We are thrilled you are considering contributing! We welcome all contributors. +Please read our [guidelines for contributing](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/CONTRIBUTING.md). + +## [Connector Developer Guide](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/adapter-guide/develop-stix-adapter.md) + + + +## [Jupyter Notebook Development Labs](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/lab) + + + +# Licensing + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +# More Resources + +## Join us on Slack! + +[Click here](https://docs.google.com/forms/d/1vEAqg9SKBF3UMtmbJJ9qqLarrXN5zeVG3_obedA3DKs) and fill out the form to receive an invite to the Open Cybersecurity Alliance slack instance, then join the #stix-shifter channel, to meet and discuss usage with the team. + +## Introduction Webinar! + +[Click here](https://ibm.biz/BdzTyA) to view an introduction webinar on STIX Shifter and the use cases it solves for. + +## Changelog + +- [Changelog](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/CHANGELOG.md) \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 000000000..702a661b6 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,28 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'STIX-Shifter' +copyright = '2023, Md Azam, Danny Elliott' +author = 'Md Azam, Danny Elliott' +release = '6.0.3' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [] + +templates_path = ['_templates'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'alabaster' +html_static_path = ['_static'] diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 000000000..a6c4b18d0 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,28 @@ +.. STIX-Shifter documentation master file, created by + sphinx-quickstart on Fri Jul 28 11:31:22 2023. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to STIX-Shifter's documentation! +======================================== + +*Introduction to STIX-Shifter* + +STIX-shifter is an open source python library allowing software to connect to products that house data repositories by using STIX Patterning, and return results as STIX Observations. + +This library takes in STIX 2 Patterns as input, and "finds" data that matches the patterns inside various products that house repositories of cybersecurity data. Examples of such products include SIEM systems, endpoint management systems, threat intelligence platforms, orchestration platforms, network control points, data lakes, and more. + +In addition to "finding" the data by using these patterns, STIX-Shifter also _transforms the output_ into STIX 2 Observations. Why would we do that you ask? To put it simply - so that all of the security data, regardless of the source, + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + README + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 000000000..32bb24529 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 000000000..48631c4f5 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1 @@ +sphinx==5.3.0 \ No newline at end of file From 58ead71a93aedca69724c42cb64d32b0ed1dea24 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Mon, 31 Jul 2023 16:04:39 -0300 Subject: [PATCH 02/55] add extension --- docs/Makefile | 20 -------------------- docs/conf.py | 2 +- docs/index.rst | 4 ++-- docs/make.bat | 35 ----------------------------------- docs/requirements.txt | 3 ++- 5 files changed, 5 insertions(+), 59 deletions(-) delete mode 100644 docs/Makefile delete mode 100644 docs/make.bat diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index d4bb2cbb9..000000000 --- a/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py index 702a661b6..37accea69 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,7 +14,7 @@ # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -extensions = [] +extensions = ["myst_parser"] templates_path = ['_templates'] exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] diff --git a/docs/index.rst b/docs/index.rst index a6c4b18d0..91c784601 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,13 +12,13 @@ STIX-shifter is an open source python library allowing software to connect to pr This library takes in STIX 2 Patterns as input, and "finds" data that matches the patterns inside various products that house repositories of cybersecurity data. Examples of such products include SIEM systems, endpoint management systems, threat intelligence platforms, orchestration platforms, network control points, data lakes, and more. -In addition to "finding" the data by using these patterns, STIX-Shifter also _transforms the output_ into STIX 2 Observations. Why would we do that you ask? To put it simply - so that all of the security data, regardless of the source, +In addition to "finding" the data by using these patterns, STIX-Shifter also transforms the output into STIX 2 Observations. Why would we do that you ask? To put it simply - so that all of the security data, regardless of the source, .. toctree:: :maxdepth: 2 :caption: Contents: - README + readme Indices and tables ================== diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index 32bb24529..000000000 --- a/docs/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=. -set BUILDDIR=_build - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.https://www.sphinx-doc.org/ - exit /b 1 -) - -if "%1" == "" goto help - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd diff --git a/docs/requirements.txt b/docs/requirements.txt index 48631c4f5..f39593742 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1 +1,2 @@ -sphinx==5.3.0 \ No newline at end of file +sphinx==5.3.0 +myst-parser==2.0.0 \ No newline at end of file From c4991c43566d453e109df5b0a9c4546f4ae28850 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Mon, 31 Jul 2023 17:55:59 -0300 Subject: [PATCH 03/55] update Sphinx==7.1.1 --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index f39593742..2b3136e74 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ -sphinx==5.3.0 +Sphinx==7.1.1 myst-parser==2.0.0 \ No newline at end of file From 34f81c0169b368fc24b22a3c55bcc2bf2df245ee Mon Sep 17 00:00:00 2001 From: Md Azam Date: Mon, 31 Jul 2023 18:14:18 -0300 Subject: [PATCH 04/55] update1 --- docs/index.rst | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 docs/index.rst diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index 91c784601..000000000 --- a/docs/index.rst +++ /dev/null @@ -1,28 +0,0 @@ -.. STIX-Shifter documentation master file, created by - sphinx-quickstart on Fri Jul 28 11:31:22 2023. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to STIX-Shifter's documentation! -======================================== - -*Introduction to STIX-Shifter* - -STIX-shifter is an open source python library allowing software to connect to products that house data repositories by using STIX Patterning, and return results as STIX Observations. - -This library takes in STIX 2 Patterns as input, and "finds" data that matches the patterns inside various products that house repositories of cybersecurity data. Examples of such products include SIEM systems, endpoint management systems, threat intelligence platforms, orchestration platforms, network control points, data lakes, and more. - -In addition to "finding" the data by using these patterns, STIX-Shifter also transforms the output into STIX 2 Observations. Why would we do that you ask? To put it simply - so that all of the security data, regardless of the source, - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - readme - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` From 2ddce002efb752d662ebdebad0b37aa4cae3e2d3 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Mon, 31 Jul 2023 18:15:13 -0300 Subject: [PATCH 05/55] update2 --- docs/index.md | 28 ++ docs/overview/OVERVIEW.md | 903 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 931 insertions(+) create mode 100644 docs/index.md create mode 100644 docs/overview/OVERVIEW.md diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 000000000..0a250073a --- /dev/null +++ b/docs/index.md @@ -0,0 +1,28 @@ +.. STIX-Shifter documentation master file, created by + sphinx-quickstart on Fri Jul 28 11:31:22 2023. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to STIX-Shifter's documentation! +======================================== + +*Introduction to STIX-Shifter* + +STIX-shifter is an open source python library allowing software to connect to products that house data repositories by using STIX Patterning, and return results as STIX Observations. + +This library takes in STIX 2 Patterns as input, and "finds" data that matches the patterns inside various products that house repositories of cybersecurity data. Examples of such products include SIEM systems, endpoint management systems, threat intelligence platforms, orchestration platforms, network control points, data lakes, and more. + +In addition to "finding" the data by using these patterns, STIX-Shifter also transforms the output into STIX 2 Observations. Why would we do that you ask? To put it simply - so that all of the security data, regardless of the source, + +.. toctree:: + :maxdepth: 2 + + readme + /overview/OVERVIEW.md + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/overview/OVERVIEW.md b/docs/overview/OVERVIEW.md new file mode 100644 index 000000000..832efc674 --- /dev/null +++ b/docs/overview/OVERVIEW.md @@ -0,0 +1,903 @@ +# STIX-SHIFTER + +**Table of Contents** + +- [Introduction](#introduction) + - [What is STIX?](#what-is-stix) + - [What is STIX-SHIFTER?](#what-is-stix-shifter) + - [What is STIX Patterning? What are STIX Observations?](#what-is-stix-patterning-what-are-stix-observations) + - [This sounds like Sigma, I already have that](#this-sounds-like-sigma-i-already-have-that) + - [What is a STIX-SHIFTER connector?](#what-is-a-stix-shifter-connector) + - [Why would I want to use this?](#why-would-i-want-to-use-this) +- [Available Connectors](#available-connectors) +- [How to use](#How-to-use) + - [Translate](#translate) + - [Transmit](#transmit) + - [Execute](#execute) + - [Debug](#Debug) +- [Limitations](#limitations) +- [Glossary](#glossary) +- [Architecture Context](#architecture-context) +- [Contributing](#contributing) +- [Guide for creating new connectors](adapter-guide/develop-stix-adapter.md) +- [Licensing](#licensing) + +## Introduction + +### What is STIX? + +Structured Threat Information eXpression (STIX™) is a language and serialization format that organizations can use to exchange cyber threat intelligence (CTI). CTI is represented with objects and descriptive relationships and stored as JSON for machine readability. + +STIX delivers a consistent and machine-readable way to enable collaborative threat analysis, automated threat exchange, automated detection and response, and more. + +To learn more about STIX, see the following references: + +- [Introduction to STIX](https://oasis-open.github.io/cti-documentation/stix/intro) +- [STIX and TAXII](https://docs.google.com/document/d/1yvqWaPPnPW-2NiVCLqzRszcx91ffMowfT5MmE9Nsy_w/edit?usp=sharing) + +### What is STIX-SHIFTER? + +STIX-shifter is an open source python library allowing software to connect to products that house data repositories by using `STIX Patterning`, and return results as `STIX Observations`. + +### What is STIX Patterning? What are STIX Observations? + +STIX 2 Patterning is a part of STIX that deals with the "matching things" part of STIX, which is an integral component of STIX Indicators. + +##### An example of a STIX pattern: + +`[url:value = 'http://www.testaddress.com'] OR [ipv4-addr:value = '192.168.122.84']` + +A STIX Observation is the `observed-data` STIX Domain Object (SDO). You can think of this as a row of data that is returned from a search triggered by the STIX pattern, and can represent an indicator of compromise. Each observation contains one or more STIX Cyber observable Objects (SCO) which in turn has one or more properties associated to the data returned from the search. + +##### An example of a STIX Observation: + +```json +{ + "id": "observed-data--cf2c58dc-200e-49e0-b6f7-e1997cccf707", + "type": "observed-data", + "created_by_ref": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d8", + "objects": { + "0": { + "type": "network-traffic", + "src_port": 567, + "dst_port": 102, + "src_ref": "1", + "dst_ref": "2" + }, + "1": { + "type": "ipv4-addr", + "value": "192.168.122.84" + }, + "2": { + "type": "ipv4-addr", + "value": "127.0.0.1" + }, + "3": { + "type": "url", + "value": "www.testaddress.com" + } + } +} +``` + +As anyone with experience in data science will tell you, the cleansing and normalizing of the data across domains, is one of the largest hurdles to overcome with attempting to build cross-platform security analytics. This is one of the barriers we are attempting to break down with STIX Shifter. + +### This sounds like Sigma, I already have that + +[Sigma](https://github.com/Neo23x0/sigma) and STIX Patterning have goals that are related, but at the end of the day has slightly different scopes. While Sigma seeks to be "for log files what Snort is for network traffic and YARA is for files", STIX Patterning's goal is to encompass _all three_ fundamental security data source types - network, file, and log - and do so simultaneously, allowing you to create complex queries and analytics that span domains. As such, so does STIX Shifter. It is critical to be able to create search patterns that span SIEM, Endpoint, Network, and File levels, in order to detect the complex patterns used in modern campaigns. + +### What is a STIX-SHIFTER connector? + +A STIX-shifter connector is a module inside Stix-Shifter library that implements an interface for: + +- data source query and result set translation +- data source communication + +Developing a new connector expands on the data sources that STIX-shifter can support. + +The combination of translation and transmission functions allows for a single STIX pattern to generate a native query for each supported data source. Each query is run, and the results are translated back into STIX objects; allowing for a uniform presentation of data. + +The objective is to have all the security data, regardless of the data source to look and behave the same. + +### Why would I want to use this? + +You might want to use this library and contribute to development, if any of the following are true: + +- You are a vendor or project owner who wants to add some form of query or enrichment functions to your product capabilities +- You are an end user and want to have a way to script searches and/or queries as part of your orchestration flow +- You are a vendor or project owner who has data that can be made available, and you want to contribute a connector +- You just want to help make the world a safer place! + +Take a look at the [**currently available connectors**](CONNECTORS.md). + + +## How to use + +### Prerequisites + +Python 3.8 or greater is required to use stix-shifter. + +Stix-shifter provides several functions: `translate` and `transmit` are the primary functions, `execute` offers a way to test the complete stix-shifter flow. + +1. [**Translate**](#translate) + + The `translate` command converts STIX patterns into data source queries (in whatever query language the data source might use) and translates data source results (in JSON format) into bundled STIX observation objects. + +2. [**Transmit**](#transmit) + + The `transmit` command allows stix-shifter to connect with products that house repositories of cybersecurity data. Connection and authentication credentials are passed to the data source APIs where stix-shifter can make calls to ping the data source, make queries, delete queries, check query status, and fetch query results. + +3. [**Execute**](#execute) + + The translation and transmission functions can work in sequence by using the `execute` command from the CLI. + +## Translate + +### CLI Arguments + +| Argument | Description | Accepted Input | +| -------- | ------------ | --------------- | +| TRANSLATION KEYWORD | The keyword specifiying a function will be used to translate queries and results. | `translate` | +| MODULE NAME | The name of the connector being used. This is the module directory name as it appears in [stix_shifter_modules](stix_shifter_modules/). If the connector supports multiple dialects, then by default a query will be generated for each one. You may specifiy a specific dialect by adding `:` directly after the module name | The connector module name with an optional `:` | +| TRANSLATION DATA TYPE | The type of data you wish to translate. This will be `query` for translating STIX patterns to native queries and `results` for translating data source results to STIX. | `query` or `results` | +| STIX IDENTITY OBJECT | This is an object that represents the data source being queried and is inserted into the results bundle of STIX objects. An empty object `"{}"` may be used for the query translation. This must be wrapped in quotes to use with the CLI. | A stringified STIX identity object | +| TRANSLATION DATA | This is the STIX pattern for query translation and a list of JSON results for results translation. This must be wrapped in quote to use with the CLI. | A stringified STIX pattern or list of JSON data | +| OPTIONS | An object of optional parameters. This must be wrapped in quotes to use with the CLI. | A stringified object of options | + +#### CLI Options + +These are general translation options defined in [`config.json`](stix_shifter_modules/config.json) that can apply to all connectors but may be overwritten by individual modules. + +| Option | Translation Data Type | Description | Accepted Values | +| ------ | --------------------- | ----------- | -------------- | +| result_limit | query | The max number of results that can be returned from a query. This value is generally included in translated queries before getting sent to the data source's API query call. The default is `10000` | A number between `1` and `500000` | +| time_range | query | A default time range, in minutes, applied to the translated query when no `START STOP` qualifier is present in the STIX pattern. As an example, this would be the `last x minutes` in a SQL query. The default is `5` | A number between `1` and `10000` | +| dialects | query | Dialects to be used for pattern translation. This will determine what `from_stix_map.json` files will be used. | A list of one or more dialect strings supported by the connector | +| validate_pattern | query | Specifies if pattern validation is run during the query translation call. This can catch errors in the submitted STIX pattern that would otherwise raise exceptions during translation. | `true` or `false` | +| stix_validator | results | Specifies if validation is run on the bundle of STIX data returned with results translation. This is performance intensive and should be used on a small result set. The default if `false`. | `true` or `false` | +| unmapped_fallback | results | If set to `true`, any results data returned, that is not specifired in the to-STIX mapping, will be included in the results in the following STIX object:property format `x-:`. The default is `false` | `true` or `false` | +| stix_2.1 | results | Results are returned as STIX 2.0 objects by default. Setting this option will return results in STIX 2.1 format. The default is `false` | `true` or `false` | + + +### 1. Translate a STIX pattern to a native data source query + +#### INPUT: STIX 2 pattern + +``` +# STIX Pattern: +"[url:value = 'http://www.testaddress.com'] OR [ipv4-addr:value = '192.168.122.84']" +``` + +#### OUTPUT: Native data source query + +``` +# Translated Query (using SQL as an example): +"SELECT * FROM tableName WHERE (Url = 'http://www.testaddress.com') +OR +((SourceIpV4 = '192.168.122.84' OR DestinationIpV4 = '192.168.122.84'))" +``` + +### CLI Command + +Open a terminal and navigate to your python 3 environment. Translation of a **query** is called in the format of: + +`stix-shifter translate query "" "" ""` + +Alternatively, you can run the CLI commands from the source. Open a terminal and navigate to the stix-shifter root directory. Translation of a **query** is called in the format of: + +`python main.py translate query "" "" ""` + +The module name refers to the name of the folder in stix-shifter that contains the connector code. The current module names can be found in the [Available Connectors](#available-connectors) table above. The STIX identity object is only used when translating data source results into STIX, so it can be passed in as an empty object for query translation calls. + +Using the Qradar connector as an example: + +`python main.py translate qradar query "{}" "[url:value = 'http://www.testaddress.com'] OR [ipv4-addr:value = '192.168.122.84']"` + +### Pattern translation using an input file + +Create a text file with the pattern you wish to translate. The file can be used in the query translation call using standard input. + +_pattern.txt_ +``` +[network-traffic:src_ref.value = '127.0.0.1'] OR [ipv4-addr:value = '0.0.0.0'] +``` + +`python main.py translate qradar query '{}' '' < /path/to/file/pattern.txt` + +### 2. Translate a JSON data source query result to a STIX 2.0 bundle of observable objects + +#### INPUT: JSON data source query result + +``` +# Datasource results (in JSON format): +[ + { + "SourcePort": 567, + "DestinationPort": 102, + "SourceIpV4": "192.168.122.84", + "DestinationIpV4": "127.0.0.1", + "Url": "www.testaddress.com" + } +] +``` + +#### OUTPUT: STIX 2.0 bundle of observable objects + +``` +# STIX Observables +{ + "type": "bundle", + "id": "bundle--2042a6e9-7f34-4a03-a745-502e358594c3", + "spec_version": "2.0", + "objects": [ + { + "type": "identity", + "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d8", + "name": "YourDataSource", + "identity_class": "events" + }, + { + "id": "observed-data--cf2c58dc-200e-49e0-b6f7-e1997cccf707", + "type": "observed-data", + "created_by_ref": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d8", + "objects": { + "0": { + "type": "network-traffic", + "src_port": 567, + "dst_port": 102, + "src_ref": "1", + "dst_ref": "2" + }, + "1": { + "type": "ipv4-addr", + "value": "192.168.122.84" + }, + "2": { + "type": "ipv4-addr", + "value": "127.0.0.1" + }, + "3": { + "type": "url", + "value": "www.testaddress.com" + } + } + } + ] +} +``` + +### CLI Command + +Open a terminal and navigate to your python 3 environment. Translation of a **results** is called in the format of: + +`stix-shifter translate result '' ''` + +Alternatively, you can run the CLI commands from the source. Open a terminal and navigate to the stix-shifter root directory. Translation of **results** is called in the format of: + +`python main.py translate result '' ''` + +The module name refers to the name of the folder in stix-shifter that contains the connector code. The current module names can be found in the [Available Connectors](#available-connectors) table above. The STIX Identity object represents the data source and is passed in to allow stix-shifter to create a reference between the data source and the generated STIX observed objects. + +Using the QRadar connector as an example: + +``` +python main.py translate qradar results \ +'{"type": "identity", "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", "name": "QRadar", "identity_class": "events"}' \ +'[{"sourceip": "192.0.2.0", "filename": "someFile.exe", "sourceport": "0123", "username": "root"}]' +``` + +### Translating results into STIX 2.1 + +By default, JSON results are translated into STIX 2.0. To return STIX 2.1 results include `'{"stix_2.1": true}'` in the CLI command + +``` +python main.py translate qradar results \ +'{"type": "identity", "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", "name": "QRadar", "identity_class": "events"}' \ +'[{"sourceip": "192.0.2.0", "filename": "someFile.exe", "sourceport": "0123", "username": "root"}]' '{"stix_2.1": true}' +``` + + +### Validating translated STIX 2.1 bundle from the CLI + +You can validate translated STIX results from the CLI provided they conform to the 2.1 standard. The `--stix-validator` flag at the end will run validation on the returned STIX objects to ensure they conform to the STIX 2.1 standard. Alternatively, `'{ "stix_validator": true }'` can be passed in at the end as an options dictionary. + +``` +python main.py translate qradar results \ +'{"type": "identity", "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", "name": "QRadar", "identity_class": "events"}' \ +'[{"sourceip": "192.0.2.0", "filename": "someFile.exe", "sourceport": "0123", "username": "root"}]' '{"stix_2.1": true, "stix_validator: true}' +``` + +### Validating STIX 2.0 and 2.1 bundles with the validator script + +Refer to the [STIX validator](bundle_validator/README.md) + +### Results translation using an input file + +Create a JSON file with the results you wish to translate. The file can be used in the results translation call using standard input. + +_results.json_ +``` +[ + { + "starttime": "1563892019916", + "endtime": "1563892019916", + "sourceip": "9.21.122.127", + "sourceport": "100", + "identityip": "0.0.0.0", + "destinationip": "127.0.0.1", + "destinationport": "800", + "username": "admin", + "protocol": "tcp" + } +] +``` + +`python main.py translate qradar results '{"type": "identity","id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff","name": "QRadar","identity_class": "system"}' '' < /path/to/file/results.json` + +## Transmit + +### Connection and Configuration objects + +STIX-shifter expects connection and configuration objects to be passed in during transmission calls. The connection object contains the host address and port of the data source being connected to, as well as an optional self signed certificate. + +### Connection + +This object contains information needed to connect to a specific data source. The `host` and `port` keys are required. + +``` +{ + "host": , + "port": , + "selfSignedCert": , + "cert": , + "resultSizeLimit": , + "timeRange": , + "options": {} +} +``` + +### Connection Options + +These are general options defined in [`config.json`](stix_shifter_modules/config.json) that can apply to all connectors but may be overwritten by individual modules. + +| Option | Description | Accepted Values | +| ------ | ----------- | -------------- | +| timeout | The max amount of time in seconds before the query times out. The default is `30`. | A number between `1` and `60` | + + +### Configuration + +This object contains an `auth` key who's value stores authentication information for the data source. What keys and values get stored in the auth will depend on the authentication requirements of the data source (username, password, auth token, etc). + +``` +{ + "auth": { + "username": , + "password": + } +} +``` + +``` +{ + "auth": { + "tenant": , + "clientId": , + "clientSecret": + } +} +``` + +``` +{ + "auth": { + "SEC": + } +} +``` + +``` +{ + "auth": { + "token": + } +} +``` + +``` +{ + "auth": { + "accountId": , + "apiKey": + } +} +``` + +### Transmit functions + +Transmit offers several functions: `ping`, `query`, `status` (for asynchronous data sources), `results`, `delete` (if supported by the data source), and `is_async`. + +Each of the transmit functions takes in common arguments: the module name, the connection object, and the configuration object. The module name refers to the name of the folder in stix-shifter that contains the connector code. The current module names can be found in the [Available Connectors](#available-connectors) table above. Information on the [connection and configuration objects](#transmit) can also be found above. Each of the CLI commands can be run from a terminal in the stix-shifter root director. + +Any failed transmission function call will return an error in the format of: + +`{'success': False, 'error': , 'code': }` + + +### CLI Arguments + +| Argument | Description | Accepted Input | +| -------- | ------------ | --------------- | +| TRANSMISSION KEYWORD | The keyword specifiying a function will be used to transmit API calls to the target data source. | `transmit` | +| MODULE NAME | The name of the connector being used. This is the module directory name as it appears in [stix_shifter_modules](stix_shifter_modules/). | The connector module name | +| CONNECTION OBJECT | This contains the information needed to connect to the target data source, such as host and port. This must be wrapped in quotes to use with the CLI. | A stringified connection object | +| CONFIGURATION OBJECT | This contains the information needed to authenticate with the target data source, such as username and password. This must be wrapped in quotes to use with the CLI. | A stringified configuration object | +| TRANSMISSION FUNCTION | The transmission function used to communicate with the target data source. | `is_async`, `ping`, `query`, `status`, `results`, `delete` | + +### Transmission Functions and Arguments + +| Function | Description | Function Argument | Function Returns | +| ------ | --------------------- | ----------- | -------------- | +| is_async | Checks if the connector is asynchronous. | NA | `true` or `false` +| ping | Calls the data source ping API endpont (or equivalent) to see if a connection can be made.| NA | Object containing `success` of `true` or `false` | +| query | Sends a native query string, as translated from the STIX pattern, to target data source API. | Tranlated query string | Query string | Object containing `success` of `true` or `false` and the `search_id`. If the connector is synchronous, the search_id will be the original query string. | +| status | Checks the status of a query. Only used by asynchronous connectors. | The `search_id` returned from the query call | Object continaing `success` of `true` or `false`, `status` of `RUNNING`, `COMPLETED`, `CANCELED`, or `ERROR`, and `progress` with a number indicating the percentage complete. | +| results | Fetches the native results of a completed query. | The `search_id` returned from the query call followed by OFFSET and LENGTH as numbers | A list of JSON results | +| delete | Deletes a query from the target data source | The `search_id` returned from the query call | Object continaing `success` of `true` or `false` | +| results_stix | Fetches the results of a completed query and runs results-to-stix translation. This essentially combines the `results` transmission function with the results translation | The `search_id` returned from the query call, followed by OFFSET and LENGTH as numbers, followed by the stringified STIX identity object. | A bundle of STIX objects. | + +### Ping + +Uses the data source API to ping the connection. + +#### CLI Command + +`stix-shifter transmit '' '' ping` + +#### OUTPUT: + +`{'success': True}` + +### Query + +Uses the data source API to submit a query to the connection. + +#### CLI Command + +`stix-shifter transmit '' '' query ` + +#### OUTPUT: + +`{'success': True, 'search_id': }` + +An asynchronous data source will typically return a search ID supplied by the API response. In the event where the API doesn't return a search id, such as with a synchronous data source, the search id will be defined in the transmission module. + +### Status + +Uses the data source API to look up the query status based on the `search_id` that is returned from the query call. This is only used for asynchronous data sources where the results are not returned right after making a query call. If the connector supports, you can specify `metadata` parameter which may contain extra information to make the status api call. + +#### CLI Command + +`stix-shifter transmit '' '' status ` + +#### OUTPUT: + +`{'success': True, 'status': , 'progress': }` + +The status can be one of: `COMPLETED`, `ERROR`, `CANCELLED`, `TIMEOUT`, or `RUNNING`. Depending on the data source, the progress may return with less than 100 while still showing the status as completed. + +### Results + +Uses the data source API to fetch the query results based on the search ID, offset, and length. + +If the connector supports, you can specify `metadata` parameter which may contain extra information to fetch the next batch of results from the datasource. This is a recomended parameter for the datasource that supports pagination. + +#### CLI Command + +`stix-shifter transmit '' '' results ` + +The `OFFSET` and `LENGTH` control what pages/rows of data are returned in the query results. + +#### OUTPUT: + +`{'success': True, 'data': []}` + +#### OUTPUT(with metadata): + +`{'success': True, 'data': [], 'metadata': }` + +#### Example: +``` +{ + "success": true, + "data": [ + { + "event": { + "securityEvent": { + "eventTimestamp": "2022-06-13T14:36:54.216539700Z", + "eventType": "FILE_CREATION", + "vendorName": "Microsoft", + "productEventType": "DeviceFileEvents", + "ingestedTimestamp": "2022-06-13T15:36:26.275010Z" + }, + "securityResult": [ + { + "summary": "FileCreated", + "category": "alert" + } + ] + } + } + ], + "metadata": { + "result_count": 2, + "next_page_token": "CgwIlqLjoAYQ2NfggwESCwiGl52VBhC0xKB" + } +} +``` + +### Results as STIX + +Uses the data source API to fetch the query results based on the search ID, offset, and length, and transforms into a bundle of STIX objects. + +#### CLI Command + +`stix-shifter transmit '' '' results_stix ''` + +#### OUTPUT: + +STIX bundle of objects. + +The `OFFSET` and `LENGTH` control what pages/rows of data are returned in the query results. + +### Is Async + +Checks if the data source connection is asynchronous. + +#### CLI Command + +`stix-shifter transmit '' '' is_async` + +#### OUTPUT: + +`True` or `False` + +## Execute + +The `execute` command tests all steps of the translation-transmission flow: + +1. A STIX pattern is translated into a list of one or more native data source queries (using a **translate query** call). +2. Each translated query in the list is sent to the data source via a **transmit query** call. +3. If the data source is asynchronous, a **transmit status** call is made for each query. Otherwise the flow moves to the next step. +4. A **transmit results** call is made for each query (using the returned query ID in step 2). If data is returned, the resulting JSON objects get added to a list. +5. The list of JSON results get translated into a bundle of STIX objects with a **translate query** call. This bundle includes the STIX `identity` object and `observed-data` objects. + +### CLI Arguments + +| Argument | Description | Accepted Input | +| -------- | ------------ | --------------- | +| EXECUTE KEYWORD | The keyword specifiying that the execute function will be used. | `execute` | +| TRANSMISSION MODULE NAME | The name of the connector being used for transmission functions. This is the module directory name as it appears in [stix_shifter_modules](stix_shifter_modules/). | The connector module name | +| TRANSLATION MODULE NAME | The name of the connector being used for translation functions. This is the module directory name as it appears in [stix_shifter_modules](stix_shifter_modules/). | The connector module name | +| STIX IDENTITY OBJECT | This is an object that represents the data source being queried and is inserted into the results bundle of STIX objects. An empty object `"{}"` may be used for the query translation. This must be wrapped in quotes to use with the CLI. | A stringified STIX identity object | +| CONNECTION OBJECT | This contains the information needed to connect to the target data source, such as host and port. This must be wrapped in quotes to use with the CLI. | A stringified connection object | +| CONFIGURATION OBJECT | This contains the information needed to authenticate with the target data source, such as username and password. This must be wrapped in quotes to use with the CLI. | A stringified configuration object | +| STIX PATTERN | This is the STIX pattern to be used for the query. This must be wrapped in quote to use with the CLI. | A stringified STIX pattern | + +### Connection Object Options + +These are general options defined in [`config.json`](stix_shifter_modules/config.json) that can apply to all connectors but may be overwritten by individual modules. These should be added as an "options" objects inside the CONNECTION OBJECT. + +| Option | Function Type | Description | Accepted Values | +| ------ | --------------------- | ----------- | -------------- | +| result_limit | query translation | The max number of results that can be returned from a query. This value is generally included in translated queries before getting sent to the data source's API query call. The default is `10000` | A number between `1` and `500000` | +| time_range | query translation | A default time range, in minutes, applied to the translated query when no `START STOP` qualifier is present in the STIX pattern. As an example, this would be the `last x minutes` in a SQL query. The default is `5` | A number between `1` and `10000` | +| dialects | query translation | Dialects to be used for pattern translation. This will determine what `from_stix_map.json` files will be used. | A list of one or more dialect strings supported by the connector | +| validate_pattern | query translation | Specifies if pattern validation is run during the query translation call. This can catch errors in the submitted STIX pattern that would otherwise raise exceptions during translation. | `true` or `false` | +| stix_validator | results translation | Specifies if validation is run on the bundle of STIX data returned with results translation. This is performance intensive and should be used on a small result set. The default if `false`. | `true` or `false` | +| unmapped_fallback | results translation | If set to `true`, any results data returned, that is not specifired in the to-STIX mapping, will be included in the results in the following STIX object:property format `x-:`. The default is `false` | `true` or `false` | +| stix_2.1 | results translation | Results are returned as STIX 2.0 objects by default. Setting this option will return results in STIX 2.1 format. The default is `false` | `true` or `false` | +| timeout | transmission | The max amount of time in seconds before the query times out. The default is `30`. | A number between `1` and `60` | + + +### CLI Command + +`stix-shifter execute '' '' '' ''` + +### CLI Example + +`stix-shifter execute mysql mysql '{"type": "identity","id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff","name": "mysql","identity_class": "system"}' '{"host": "localhost", "database":"demo_db", "options":{"table":"demo_table", "validate_pattern": true}}' '{"auth": {"username":"root", "password":"MyPassword"}}' "[ipv4-addr:value = '213.213.142.5'] START t'2019-01-28T12:24:01.009Z' STOP t'2019-01-28T12:54:01.009Z'"` + +### Debug + +You can add the `--debug` option to your CLI command to see more logs. + +`stix-shifter --debug execute '' '' '' ''` + +### Change max returned results + +You can add the `--results` option with an integer value at the end of your CLI command to limit the maximum number of returned search results (default 10). + +`stix-shifter execute '' '' '' '' --results 50` + +### Save the STIX results to a file + +You can redirect the output of your CLI command to a file to save the STIX results. + +`stix-shifter execute '' '' '' '' > results.json` + +### OUTPUT: + +A bundle of STIX objects + +## Modules + +The `modules` command will return a JSON of the existing connectors along with their dialects and supported languages that are used in query translation. + +### CLI Command + +`python main.py modules` + +#### output +``` +{ + "qradar": { + "flows": { + "language": "stix", + "default": true + }, + "events": { + "language": "stix", + "default": true + }, + "aql": { + "language": "aql", + "default": false + } + }, + "security_advisor": { + "default": { + "language": "stix", + "default": true + } + }, + ... +} +``` + +This command can also be used to get the dialects of a specific connector. + +`python main.py modules ` + +### CLI Command + +`python main.py modules qradar` + +#### output +``` +{ + "qradar": { + "flows": { + "language": "stix", + "default": true + }, + "events": { + "language": "stix", + "default": true + }, + "aql": { + "language": "aql", + "default": true + } + } +} +``` + +In the above example, the QRadar connector can use three dialects: `flows`, `events`, and `aql`. When a connector only has a `default` dialect, such as with Security Advisor, only one dialect is used by the connector. Most dialects will use the `stix` language since they translate STIX patterns into native queries. QRadar's `aql` dialect uses the `aql` language since it is meant to accept an AQL query rather than a STIX pattern. See the [QRadar connector README](stix_shifter_modules/qradar/README.md) for more information on AQL passthrough. + +## configs + +The `configs` command returns the configuration pararmetes of the existing connectors. It basically returns a JSON of the existing connectors along with their connections and configuation objects that are specified in config.json. + +### CLI Command + +`python main.py configs` +#### output +``` +{ + "alertflex": { + "connection": { + "type": { + "type": "connectorType", + "displayName": "Alertflex" + }, + "options": { + "type": "fields", + "async_call": { + "type": "text", + "hidden": true, + "optional": true + }, + "result_limit": { + "default": 10000, + "min": 1, + "max": 500000, + "type": "number", + "previous": "connection.resultSizeLimit" + }, + "time_range": { + "default": 5, + "min": 1, + "max": 10000, + "type": "number", + "previous": "connection.timerange", + "nullable": true + }, + ..... + } + } + } +} +``` + +Specifying the connector module name will return the configuration parameters of a specific connector. + +### CLI Command + +`python main.py configs ` +#### output +``` +{ + "qradar": { + "connection": { + "type": { + "type": "connectorType", + "displayName": "IBM\u00ae QRadar and QRadar On Cloud", + "group": "qradar" + }, + "options": { + "type": "fields", + "async_call": { + "type": "text", + "hidden": true, + "optional": true + }, + "result_limit": { + "default": 10000, + "min": 1, + "max": 500000, + "type": "number", + "previous": "connection.resultSizeLimit" + }, + "time_range": { + "default": 5, + "min": 1, + "max": 10000, + "type": "number", + "previous": "connection.timerange", + "nullable": true + }, + "timeout": { + "default": 30, + "min": 1, + "max": 60, + "hidden": true, + "type": "number", + "previous": "connection.timeoutLimit" + }, + "dialects": { + "type": "array", + "hidden": true, + "optional": true + }, + "language": { + "type": "string", + "default": "stix", + "optional": true, + "hidden": true + }, + "validate_pattern": { + "type": "boolean", + "optional": true, + "hidden": true, + "previous": "connection.validate_pattern", + "default": false + }, + "stix_validator": { + "type": "boolean", + "default": false, + "optional": true, + "hidden": true, + "previous": "connection.stix_validator" + }, + "mapping": { + "type": "json", + "optional": true, + "previous": "connection.mapping" + }, + "unmapped_fallback": { + "type": "boolean", + "default": true, + "optional": true, + "hidden": true + }, + "stix_2.1": { + "type": "boolean", + "default": false, + "optional": true, + "hidden": true + } + }, + "host": { + "type": "text", + "regex": "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9_:/\\-]*[a-zA-Z0-9])\\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9_:/\\-]*[A-Za-z0-9])$" + }, + "port": { + "type": "number", + "default": 443, + "min": 1, + "max": 65535 + }, + "help": { + "default": "data-sources-qradar.html", + "type": "link" + }, + "selfSignedCert": { + "type": "password", + "optional": true + } + }, + "configuration": { + "auth": { + "type": "fields", + "sec": { + "type": "password", + "previous": "configuration.auth.SEC" + } + } + } + } +} +``` +## Limitations + +STIX-Shifter has limitations on the length of a pattern that can be translated into a native query. As the pattern length increases, the translation time increases exponentially due to how ANTLR 4 parses the pattern. See [STIX-Shifter Limitations](adapter-guide/stix-shifter-limitations.md) for more details. + +## Glossary + +| Terms | Definition | +| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Modules | Folders in the stix-shifter project that contains code that is specific to a data source. | +| STIX 2 patterns | STIX patterns are expressions that represent Cyber Observable objects within a STIX Indicator STIX Domain Objects (SDOs).
They are helpful for modeling intelligence that indicates cyber activity. | +| STIX 2 objects | JSON objects that contain CTI data. In STIX, these objects are referred to as Cyber Observable Objects. | +| Data sources | Security products that house data repositories. | +| Data source queries | Queries written in the data source's native query language. | +| Data source query results | Data returned from a data source query. | + +## Architecture Context + +![STIX SHIFTER CLASS DIAGRAM](./adapter-guide/images/architecture.png) + +## Contributing + +We are thrilled you are considering contributing! We welcome all contributors. + +Please read our [guidelines for contributing](CONTRIBUTING.md). + +## Guide for creating new connectors + +If you want to create a new connector for STIX-shifter, see the [developer guide](adapter-guide/develop-stix-adapter.md) + +## Licensing + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. From 286ec24c6b210dcfefd0ba11b885abaa31c20b68 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Mon, 31 Jul 2023 18:28:56 -0300 Subject: [PATCH 06/55] update3 --- docs/README.md | 132 ----------------------------------- docs/{index.md => index.rst} | 2 +- docs/installation.rst | 10 +++ 3 files changed, 11 insertions(+), 133 deletions(-) delete mode 100644 docs/README.md rename docs/{index.md => index.rst} (98%) create mode 100644 docs/installation.rst diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index bd4c8ef9a..000000000 --- a/docs/README.md +++ /dev/null @@ -1,132 +0,0 @@ -[![example workflow](https://github.com/opencybersecurityalliance/stix-shifter/actions/workflows/main.yml/badge.svg)](https://github.com/opencybersecurityalliance/stix-shifter/actions) -[![codecov](https://codecov.io/gh/opencybersecurityalliance/stix-shifter/branch/develop/graph/badge.svg?token=gQvl14peRj)](https://codecov.io/gh/opencybersecurityalliance/stix-shifter) - -# Introduction to STIX-Shifter - -STIX-shifter is an open source python library allowing software to connect to products that house data repositories by using STIX Patterning, and return results as STIX Observations. - -This library takes in STIX 2 Patterns as input, and "finds" data that matches the patterns inside various products that house repositories of cybersecurity data. Examples of such products include SIEM systems, endpoint management systems, threat intelligence platforms, orchestration platforms, network control points, data lakes, and more. - -In addition to "finding" the data by using these patterns, STIX-Shifter also _transforms the output_ into STIX 2 Observations. Why would we do that you ask? To put it simply - so that all of the security data, regardless of the source, mostly looks and behaves the same. - -## Project Overview and CLI Commands - -For general information about STIX, this project, and the command line utilities, see the [STIX-shifter Overview](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/OVERVIEW.md) - - -# Installation - -The recommended method for installing stix-shifter is via pip. Two prerequisite packages needs to be installed inlcuding the package of stix-shifter connector module to complete a stix-shifter connector installation. Run the below commands to install all the packages: - -1. Main stix-shifter package: `pip install stix-shifter` - -2. Stix-shifter Utility package: `pip install stix-shifter-utils` - -3. Desired stix-shifter connector module package: `pip install stix-shifter-modules- ` - Example: `pip install stix-shifter-modules-qradar` - -## Dependencies - -STIX-shifter requries Python 3.8 or greater. See the [requirements file](stix_shifter/requirements.txt) for library dependencies. - -# Usage - -STIX-Shifter can use used the following ways: - -## 1. As a command line utility - -The STIX-Shifter comes with a bundled script which you can use to translate STIX Pattern to a native datasource query. It can also be used to translate a JSON data source query result to a STIX bundle of observable objects. You can also send query to a datasource by using a transmission option. - -More details of the command line option can be found [here](https://github.com/opencybersecurityalliance/stix-shifter/blob/master/OVERVIEW.md#how-to-use) - -``` -$ stix-shifter translate query "" "" "" -``` -Example: -``` -$ stix-shifter translate qradar query {} "[ipv4-addr:value = '127.0.0.1']" {} -``` - -In order to build `stix-shifter` packages from source follow the below prerequisite steps: - 1. Go to the stix-shifter parent directory - 2. Optionally, you can create a Python 3 virtual environemnt: - `virtualenv -p python3 virtualenv && source virtualenv/bin/activate` - 3. Run setup: `python3 setup.py install` - - -## 2. Running from the source - -You may also use the `python3 main.py` script. All the options are the same as the command line utility described above. - -Example: - -``` -python3 main.py translate qradar query {} "[ipv4-addr:value = '127.0.0.1']" {} -``` - -In order to run `python3 main.py` from the source follow the below prerequisite steps: - 1. Go to the stix-shifter parent directory - 2. Optionally, you can create a Python 3 virtual environemnt: - `virtualenv -p python3 virtualenv && source virtualenv/bin/activate` - 3. Run setup to install dependancies: `INSTALL_REQUIREMENTS_ONLY=1 python3 setup.py install`. - -**Note:** setup.py only installs dependencies when INSTALL_REQUIREMENTS_ONLY=1 directive is used. This option is similar to `python3 generate_requirements.py && pip install -r requirements.txt` - -## 3. As a library - -You can also use this library to integrate STIX Shifter into your own tools. You can translate a STIX Pattern: - -``` -from stix_shifter.stix_translation import stix_translation - -translation = stix_translation.StixTranslation() -response = translation.translate('', 'query', '{}', '', '') - -print(response) -``` -### Use of custom mappings - -If a connector has been installed using pip, the process for editing the STIX mappings is different than if you have pulled-down the project. When working locally, you can edit the mapping files directly. See the [mapping files for the MySQL connector](stix_shifter_modules/mysql/stix_translation/json) as an example. Editing the mapping files won't work if the connector has been installed with pip; the setup script of the stix-shifter package includes the mapppings inside `config.json`. This allows stix-shifter to injest custom mappings as part of the connector's configuration. - -Refer to [Use of custom mappings](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/adapter-guide/custom_mappings.md) for more details on how to edit the mappings in the configuration. - -# Contributing - -We are thrilled you are considering contributing! We welcome all contributors. -Please read our [guidelines for contributing](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/CONTRIBUTING.md). - -## [Connector Developer Guide](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/adapter-guide/develop-stix-adapter.md) - - - -## [Jupyter Notebook Development Labs](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/lab) - - - -# Licensing - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -# More Resources - -## Join us on Slack! - -[Click here](https://docs.google.com/forms/d/1vEAqg9SKBF3UMtmbJJ9qqLarrXN5zeVG3_obedA3DKs) and fill out the form to receive an invite to the Open Cybersecurity Alliance slack instance, then join the #stix-shifter channel, to meet and discuss usage with the team. - -## Introduction Webinar! - -[Click here](https://ibm.biz/BdzTyA) to view an introduction webinar on STIX Shifter and the use cases it solves for. - -## Changelog - -- [Changelog](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/CHANGELOG.md) \ No newline at end of file diff --git a/docs/index.md b/docs/index.rst similarity index 98% rename from docs/index.md rename to docs/index.rst index 0a250073a..a1140f71a 100644 --- a/docs/index.md +++ b/docs/index.rst @@ -17,7 +17,7 @@ In addition to "finding" the data by using these patterns, STIX-Shifter also tra .. toctree:: :maxdepth: 2 - readme + installation /overview/OVERVIEW.md Indices and tables diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 000000000..0762b340c --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,10 @@ +**Installation** + +The recommended method for installing stix-shifter is via pip. Two prerequisite packages needs to be installed inlcuding the package of stix-shifter connector module to complete a stix-shifter connector installation. Run the below commands to install all the packages: + +1. Main stix-shifter package: `pip install stix-shifter` + +2. Stix-shifter Utility package: `pip install stix-shifter-utils` + +3. Desired stix-shifter connector module package: `pip install stix-shifter-modules- ` + Example: `pip install stix-shifter-modules-qradar` \ No newline at end of file From eb0954c596cf169019b7cd4178703eaef6c4d2e0 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Mon, 31 Jul 2023 18:36:33 -0300 Subject: [PATCH 07/55] update4 --- docs/conf.py | 1 - docs/index.rst | 2 +- docs/installation.rst | 10 --- docs/overview/README.md | 132 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 133 insertions(+), 12 deletions(-) delete mode 100644 docs/installation.rst create mode 100644 docs/overview/README.md diff --git a/docs/conf.py b/docs/conf.py index 37accea69..954a240cc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,7 +20,6 @@ exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] - # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output diff --git a/docs/index.rst b/docs/index.rst index a1140f71a..5a95adbff 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,7 +17,7 @@ In addition to "finding" the data by using these patterns, STIX-Shifter also tra .. toctree:: :maxdepth: 2 - installation + /overview/README.md /overview/OVERVIEW.md Indices and tables diff --git a/docs/installation.rst b/docs/installation.rst deleted file mode 100644 index 0762b340c..000000000 --- a/docs/installation.rst +++ /dev/null @@ -1,10 +0,0 @@ -**Installation** - -The recommended method for installing stix-shifter is via pip. Two prerequisite packages needs to be installed inlcuding the package of stix-shifter connector module to complete a stix-shifter connector installation. Run the below commands to install all the packages: - -1. Main stix-shifter package: `pip install stix-shifter` - -2. Stix-shifter Utility package: `pip install stix-shifter-utils` - -3. Desired stix-shifter connector module package: `pip install stix-shifter-modules- ` - Example: `pip install stix-shifter-modules-qradar` \ No newline at end of file diff --git a/docs/overview/README.md b/docs/overview/README.md new file mode 100644 index 000000000..bd4c8ef9a --- /dev/null +++ b/docs/overview/README.md @@ -0,0 +1,132 @@ +[![example workflow](https://github.com/opencybersecurityalliance/stix-shifter/actions/workflows/main.yml/badge.svg)](https://github.com/opencybersecurityalliance/stix-shifter/actions) +[![codecov](https://codecov.io/gh/opencybersecurityalliance/stix-shifter/branch/develop/graph/badge.svg?token=gQvl14peRj)](https://codecov.io/gh/opencybersecurityalliance/stix-shifter) + +# Introduction to STIX-Shifter + +STIX-shifter is an open source python library allowing software to connect to products that house data repositories by using STIX Patterning, and return results as STIX Observations. + +This library takes in STIX 2 Patterns as input, and "finds" data that matches the patterns inside various products that house repositories of cybersecurity data. Examples of such products include SIEM systems, endpoint management systems, threat intelligence platforms, orchestration platforms, network control points, data lakes, and more. + +In addition to "finding" the data by using these patterns, STIX-Shifter also _transforms the output_ into STIX 2 Observations. Why would we do that you ask? To put it simply - so that all of the security data, regardless of the source, mostly looks and behaves the same. + +## Project Overview and CLI Commands + +For general information about STIX, this project, and the command line utilities, see the [STIX-shifter Overview](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/OVERVIEW.md) + + +# Installation + +The recommended method for installing stix-shifter is via pip. Two prerequisite packages needs to be installed inlcuding the package of stix-shifter connector module to complete a stix-shifter connector installation. Run the below commands to install all the packages: + +1. Main stix-shifter package: `pip install stix-shifter` + +2. Stix-shifter Utility package: `pip install stix-shifter-utils` + +3. Desired stix-shifter connector module package: `pip install stix-shifter-modules- ` + Example: `pip install stix-shifter-modules-qradar` + +## Dependencies + +STIX-shifter requries Python 3.8 or greater. See the [requirements file](stix_shifter/requirements.txt) for library dependencies. + +# Usage + +STIX-Shifter can use used the following ways: + +## 1. As a command line utility + +The STIX-Shifter comes with a bundled script which you can use to translate STIX Pattern to a native datasource query. It can also be used to translate a JSON data source query result to a STIX bundle of observable objects. You can also send query to a datasource by using a transmission option. + +More details of the command line option can be found [here](https://github.com/opencybersecurityalliance/stix-shifter/blob/master/OVERVIEW.md#how-to-use) + +``` +$ stix-shifter translate query "" "" "" +``` +Example: +``` +$ stix-shifter translate qradar query {} "[ipv4-addr:value = '127.0.0.1']" {} +``` + +In order to build `stix-shifter` packages from source follow the below prerequisite steps: + 1. Go to the stix-shifter parent directory + 2. Optionally, you can create a Python 3 virtual environemnt: + `virtualenv -p python3 virtualenv && source virtualenv/bin/activate` + 3. Run setup: `python3 setup.py install` + + +## 2. Running from the source + +You may also use the `python3 main.py` script. All the options are the same as the command line utility described above. + +Example: + +``` +python3 main.py translate qradar query {} "[ipv4-addr:value = '127.0.0.1']" {} +``` + +In order to run `python3 main.py` from the source follow the below prerequisite steps: + 1. Go to the stix-shifter parent directory + 2. Optionally, you can create a Python 3 virtual environemnt: + `virtualenv -p python3 virtualenv && source virtualenv/bin/activate` + 3. Run setup to install dependancies: `INSTALL_REQUIREMENTS_ONLY=1 python3 setup.py install`. + +**Note:** setup.py only installs dependencies when INSTALL_REQUIREMENTS_ONLY=1 directive is used. This option is similar to `python3 generate_requirements.py && pip install -r requirements.txt` + +## 3. As a library + +You can also use this library to integrate STIX Shifter into your own tools. You can translate a STIX Pattern: + +``` +from stix_shifter.stix_translation import stix_translation + +translation = stix_translation.StixTranslation() +response = translation.translate('', 'query', '{}', '', '') + +print(response) +``` +### Use of custom mappings + +If a connector has been installed using pip, the process for editing the STIX mappings is different than if you have pulled-down the project. When working locally, you can edit the mapping files directly. See the [mapping files for the MySQL connector](stix_shifter_modules/mysql/stix_translation/json) as an example. Editing the mapping files won't work if the connector has been installed with pip; the setup script of the stix-shifter package includes the mapppings inside `config.json`. This allows stix-shifter to injest custom mappings as part of the connector's configuration. + +Refer to [Use of custom mappings](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/adapter-guide/custom_mappings.md) for more details on how to edit the mappings in the configuration. + +# Contributing + +We are thrilled you are considering contributing! We welcome all contributors. +Please read our [guidelines for contributing](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/CONTRIBUTING.md). + +## [Connector Developer Guide](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/adapter-guide/develop-stix-adapter.md) + + + +## [Jupyter Notebook Development Labs](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/lab) + + + +# Licensing + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +# More Resources + +## Join us on Slack! + +[Click here](https://docs.google.com/forms/d/1vEAqg9SKBF3UMtmbJJ9qqLarrXN5zeVG3_obedA3DKs) and fill out the form to receive an invite to the Open Cybersecurity Alliance slack instance, then join the #stix-shifter channel, to meet and discuss usage with the team. + +## Introduction Webinar! + +[Click here](https://ibm.biz/BdzTyA) to view an introduction webinar on STIX Shifter and the use cases it solves for. + +## Changelog + +- [Changelog](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/CHANGELOG.md) \ No newline at end of file From 45708a79c2bccfc665a3e382cf0c66dbce49feba Mon Sep 17 00:00:00 2001 From: Md Azam Date: Tue, 1 Aug 2023 14:04:39 -0300 Subject: [PATCH 08/55] update5 --- docs/{overview => }/README.md | 7 ++++--- docs/index.rst | 4 ++-- docs/overview/index.rst | 4 ++++ docs/overview/introduction.md | 6 ++++++ 4 files changed, 16 insertions(+), 5 deletions(-) rename docs/{overview => }/README.md (94%) create mode 100644 docs/overview/index.rst create mode 100644 docs/overview/introduction.md diff --git a/docs/overview/README.md b/docs/README.md similarity index 94% rename from docs/overview/README.md rename to docs/README.md index bd4c8ef9a..fc18e18ab 100644 --- a/docs/overview/README.md +++ b/docs/README.md @@ -1,6 +1,3 @@ -[![example workflow](https://github.com/opencybersecurityalliance/stix-shifter/actions/workflows/main.yml/badge.svg)](https://github.com/opencybersecurityalliance/stix-shifter/actions) -[![codecov](https://codecov.io/gh/opencybersecurityalliance/stix-shifter/branch/develop/graph/badge.svg?token=gQvl14peRj)](https://codecov.io/gh/opencybersecurityalliance/stix-shifter) - # Introduction to STIX-Shifter STIX-shifter is an open source python library allowing software to connect to products that house data repositories by using STIX Patterning, and return results as STIX Observations. @@ -13,6 +10,10 @@ In addition to "finding" the data by using these patterns, STIX-Shifter also _tr For general information about STIX, this project, and the command line utilities, see the [STIX-shifter Overview](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/OVERVIEW.md) +.. toctree:: + :maxdepth: 2 + +/overview/OVERVIEW.md # Installation diff --git a/docs/index.rst b/docs/index.rst index 5a95adbff..92e4a149f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,8 +17,8 @@ In addition to "finding" the data by using these patterns, STIX-Shifter also tra .. toctree:: :maxdepth: 2 - /overview/README.md - /overview/OVERVIEW.md + README.md + /overview/index.rst Indices and tables ================== diff --git a/docs/overview/index.rst b/docs/overview/index.rst new file mode 100644 index 000000000..4ad2169cb --- /dev/null +++ b/docs/overview/index.rst @@ -0,0 +1,4 @@ +.. toctree:: + :maxdepth: 2 + + introduction.md \ No newline at end of file diff --git a/docs/overview/introduction.md b/docs/overview/introduction.md new file mode 100644 index 000000000..ecfab7b27 --- /dev/null +++ b/docs/overview/introduction.md @@ -0,0 +1,6 @@ +What is STIX? +---------- + +.. include:: + + OVERVIEW.md \ No newline at end of file From 0ae5a74341ada5cbe3c7c56c1d02132e9bb0d4ea Mon Sep 17 00:00:00 2001 From: Md Azam Date: Tue, 1 Aug 2023 15:12:53 -0300 Subject: [PATCH 09/55] mdinclude ext --- docs/README.md | 5 ----- docs/conf.py | 5 ++++- docs/overview/introduction.md | 3 +-- docs/requirements.txt | 3 ++- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/docs/README.md b/docs/README.md index fc18e18ab..7c135b508 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,11 +10,6 @@ In addition to "finding" the data by using these patterns, STIX-Shifter also _tr For general information about STIX, this project, and the command line utilities, see the [STIX-shifter Overview](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/OVERVIEW.md) -.. toctree:: - :maxdepth: 2 - -/overview/OVERVIEW.md - # Installation The recommended method for installing stix-shifter is via pip. Two prerequisite packages needs to be installed inlcuding the package of stix-shifter connector module to complete a stix-shifter connector installation. Run the below commands to install all the packages: diff --git a/docs/conf.py b/docs/conf.py index 954a240cc..08c0c7e63 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,7 +14,10 @@ # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -extensions = ["myst_parser"] +extensions = [ + "myst_parser", + "sphinx_mdinclude" + ] templates_path = ['_templates'] exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] diff --git a/docs/overview/introduction.md b/docs/overview/introduction.md index ecfab7b27..73ae52bb7 100644 --- a/docs/overview/introduction.md +++ b/docs/overview/introduction.md @@ -1,6 +1,5 @@ What is STIX? ---------- -.. include:: - +.. mdinclude:: OVERVIEW.md \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt index 2b3136e74..cf56d2928 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,3 @@ Sphinx==7.1.1 -myst-parser==2.0.0 \ No newline at end of file +myst-parser==2.0.0 +sphinx_mdinclude==0.5.3 \ No newline at end of file From 429ab179f672b157f0c285fd7689c5a8608164bc Mon Sep 17 00:00:00 2001 From: Md Azam Date: Tue, 1 Aug 2023 15:40:06 -0300 Subject: [PATCH 10/55] mdinclude ext2 --- docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 08c0c7e63..f027b9569 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,8 +15,8 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [ - "myst_parser", - "sphinx_mdinclude" + "sphinx_mdinclude", + "myst_parser" ] templates_path = ['_templates'] From 0fb4bf0c5df1fd39e6da546e11d72a7f88b6f1a0 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Tue, 1 Aug 2023 15:46:41 -0300 Subject: [PATCH 11/55] mdinclude ext3 --- docs/conf.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index f027b9569..70cc62638 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,10 +15,14 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [ - "sphinx_mdinclude", - "myst_parser" + "myst_parser", + "sphinx_mdinclude" ] +source_suffix = [ + ".md", + ".rst" +] templates_path = ['_templates'] exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] From d4d594a4866a18764913f407cf64cfea027b3820 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Tue, 1 Aug 2023 15:56:20 -0300 Subject: [PATCH 12/55] mdinclude ext4 --- docs/conf.py | 7 +------ docs/overview/introduction.md | 7 ++----- docs/requirements.txt | 3 +-- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 70cc62638..1fde5bafe 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,14 +15,9 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [ - "myst_parser", - "sphinx_mdinclude" + "myst_parser" ] -source_suffix = [ - ".md", - ".rst" -] templates_path = ['_templates'] exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] diff --git a/docs/overview/introduction.md b/docs/overview/introduction.md index 73ae52bb7..ea5253e92 100644 --- a/docs/overview/introduction.md +++ b/docs/overview/introduction.md @@ -1,5 +1,2 @@ -What is STIX? ----------- - -.. mdinclude:: - OVERVIEW.md \ No newline at end of file +```{include} OVERVIEW.md +``` \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt index cf56d2928..2b3136e74 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,2 @@ Sphinx==7.1.1 -myst-parser==2.0.0 -sphinx_mdinclude==0.5.3 \ No newline at end of file +myst-parser==2.0.0 \ No newline at end of file From d006db77e2fd45b3986f31777ea83d72bed3efbc Mon Sep 17 00:00:00 2001 From: Md Azam Date: Tue, 1 Aug 2023 16:06:06 -0300 Subject: [PATCH 13/55] mdinclude ext5 --- docs/README.md | 5 ++++- docs/index.rst | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/README.md b/docs/README.md index 7c135b508..81729aec6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,7 +8,10 @@ In addition to "finding" the data by using these patterns, STIX-Shifter also _tr ## Project Overview and CLI Commands -For general information about STIX, this project, and the command line utilities, see the [STIX-shifter Overview](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/OVERVIEW.md) +For general information about STIX, this project, and the command line utilities, see the [STIX-shifter Overview] + +```{include} OVERVIEW.md +``` # Installation diff --git a/docs/index.rst b/docs/index.rst index 92e4a149f..2733c952f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,7 +18,6 @@ In addition to "finding" the data by using these patterns, STIX-Shifter also tra :maxdepth: 2 README.md - /overview/index.rst Indices and tables ================== From 0a34260ae67d8571975f74c0654b1193518acc48 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Tue, 1 Aug 2023 16:14:11 -0300 Subject: [PATCH 14/55] myst ext --- docs/{overview => }/OVERVIEW.md | 0 docs/README.md | 2 -- docs/overview/index.rst | 4 ---- docs/overview/introduction.md | 2 -- 4 files changed, 8 deletions(-) rename docs/{overview => }/OVERVIEW.md (100%) delete mode 100644 docs/overview/index.rst delete mode 100644 docs/overview/introduction.md diff --git a/docs/overview/OVERVIEW.md b/docs/OVERVIEW.md similarity index 100% rename from docs/overview/OVERVIEW.md rename to docs/OVERVIEW.md diff --git a/docs/README.md b/docs/README.md index 81729aec6..1c2024e75 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,8 +8,6 @@ In addition to "finding" the data by using these patterns, STIX-Shifter also _tr ## Project Overview and CLI Commands -For general information about STIX, this project, and the command line utilities, see the [STIX-shifter Overview] - ```{include} OVERVIEW.md ``` diff --git a/docs/overview/index.rst b/docs/overview/index.rst deleted file mode 100644 index 4ad2169cb..000000000 --- a/docs/overview/index.rst +++ /dev/null @@ -1,4 +0,0 @@ -.. toctree:: - :maxdepth: 2 - - introduction.md \ No newline at end of file diff --git a/docs/overview/introduction.md b/docs/overview/introduction.md deleted file mode 100644 index ea5253e92..000000000 --- a/docs/overview/introduction.md +++ /dev/null @@ -1,2 +0,0 @@ -```{include} OVERVIEW.md -``` \ No newline at end of file From f036b96847f5aa17868ab1661478695767fc7dcd Mon Sep 17 00:00:00 2001 From: Md Azam Date: Wed, 2 Aug 2023 10:19:32 -0300 Subject: [PATCH 15/55] developer guide --- docs/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/README.md b/docs/README.md index 1c2024e75..b9e799085 100644 --- a/docs/README.md +++ b/docs/README.md @@ -92,7 +92,9 @@ Refer to [Use of custom mappings](https://github.com/opencybersecurityalliance/s We are thrilled you are considering contributing! We welcome all contributors. Please read our [guidelines for contributing](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/CONTRIBUTING.md). -## [Connector Developer Guide](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/adapter-guide/develop-stix-adapter.md) +## Connector Developer Guide +```{include} /adapter-guide/develop-stix-adapter.md +``` From 0f4387dcb578c59257f4e76a896545243e4e256e Mon Sep 17 00:00:00 2001 From: Md Azam Date: Thu, 3 Aug 2023 14:38:54 -0300 Subject: [PATCH 16/55] myst parser update --- docs/{index.rst => bakindex.rst} | 0 docs/index.md | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+) rename docs/{index.rst => bakindex.rst} (100%) create mode 100644 docs/index.md diff --git a/docs/index.rst b/docs/bakindex.rst similarity index 100% rename from docs/index.rst rename to docs/bakindex.rst diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 000000000..f1e889c58 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,23 @@ +```{rubric} STIX-Shifter +``` + +STIX-shifter is an open source python library allowing software to connect to products that house data repositories by using STIX Patterning, and return results as STIX Observations. + +This library takes in STIX 2 Patterns as input, and "finds" data that matches the patterns inside various products that house repositories of cybersecurity data. Examples of such products include SIEM systems, endpoint management systems, threat intelligence platforms, orchestration platforms, network control points, data lakes, and more. + +In addition to "finding" the data by using these patterns, STIX-Shifter also _transforms the output_ into STIX 2 Observations. Why would we do that you ask? To put it simply - so that all of the security data, regardless of the source, mostly looks and behaves the same. + + +````{div} sd-d-flex-row +```{button-ref} OVERVIEW +:ref-type: doc +:color: primary +:class: sd-rounded-pill sd-mr-3 + +Overview +``` + +```{toctree} +:hidden: +OVERVIEW.md +``` \ No newline at end of file From b9290784b50c272d7e79e83b2b40e683d3864ce8 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Thu, 3 Aug 2023 14:50:25 -0300 Subject: [PATCH 17/55] myst parser update2 --- docs/index.md | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/docs/index.md b/docs/index.md index f1e889c58..6feac946c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,16 +8,9 @@ This library takes in STIX 2 Patterns as input, and "finds" data that matches th In addition to "finding" the data by using these patterns, STIX-Shifter also _transforms the output_ into STIX 2 Observations. Why would we do that you ask? To put it simply - so that all of the security data, regardless of the source, mostly looks and behaves the same. -````{div} sd-d-flex-row -```{button-ref} OVERVIEW -:ref-type: doc -:color: primary -:class: sd-rounded-pill sd-mr-3 - -Overview -``` +{doc}`overview` ```{toctree} :hidden: -OVERVIEW.md +overview.md ``` \ No newline at end of file From 44591076d0952b37e8120135941b1996bab8caeb Mon Sep 17 00:00:00 2001 From: Md Azam Date: Thu, 3 Aug 2023 15:09:38 -0300 Subject: [PATCH 18/55] fix conflicts --- docs/OVERVIEW.md | 903 ----------------------------------------------- docs/README.md | 131 ------- 2 files changed, 1034 deletions(-) delete mode 100644 docs/OVERVIEW.md delete mode 100644 docs/README.md diff --git a/docs/OVERVIEW.md b/docs/OVERVIEW.md deleted file mode 100644 index 832efc674..000000000 --- a/docs/OVERVIEW.md +++ /dev/null @@ -1,903 +0,0 @@ -# STIX-SHIFTER - -**Table of Contents** - -- [Introduction](#introduction) - - [What is STIX?](#what-is-stix) - - [What is STIX-SHIFTER?](#what-is-stix-shifter) - - [What is STIX Patterning? What are STIX Observations?](#what-is-stix-patterning-what-are-stix-observations) - - [This sounds like Sigma, I already have that](#this-sounds-like-sigma-i-already-have-that) - - [What is a STIX-SHIFTER connector?](#what-is-a-stix-shifter-connector) - - [Why would I want to use this?](#why-would-i-want-to-use-this) -- [Available Connectors](#available-connectors) -- [How to use](#How-to-use) - - [Translate](#translate) - - [Transmit](#transmit) - - [Execute](#execute) - - [Debug](#Debug) -- [Limitations](#limitations) -- [Glossary](#glossary) -- [Architecture Context](#architecture-context) -- [Contributing](#contributing) -- [Guide for creating new connectors](adapter-guide/develop-stix-adapter.md) -- [Licensing](#licensing) - -## Introduction - -### What is STIX? - -Structured Threat Information eXpression (STIX™) is a language and serialization format that organizations can use to exchange cyber threat intelligence (CTI). CTI is represented with objects and descriptive relationships and stored as JSON for machine readability. - -STIX delivers a consistent and machine-readable way to enable collaborative threat analysis, automated threat exchange, automated detection and response, and more. - -To learn more about STIX, see the following references: - -- [Introduction to STIX](https://oasis-open.github.io/cti-documentation/stix/intro) -- [STIX and TAXII](https://docs.google.com/document/d/1yvqWaPPnPW-2NiVCLqzRszcx91ffMowfT5MmE9Nsy_w/edit?usp=sharing) - -### What is STIX-SHIFTER? - -STIX-shifter is an open source python library allowing software to connect to products that house data repositories by using `STIX Patterning`, and return results as `STIX Observations`. - -### What is STIX Patterning? What are STIX Observations? - -STIX 2 Patterning is a part of STIX that deals with the "matching things" part of STIX, which is an integral component of STIX Indicators. - -##### An example of a STIX pattern: - -`[url:value = 'http://www.testaddress.com'] OR [ipv4-addr:value = '192.168.122.84']` - -A STIX Observation is the `observed-data` STIX Domain Object (SDO). You can think of this as a row of data that is returned from a search triggered by the STIX pattern, and can represent an indicator of compromise. Each observation contains one or more STIX Cyber observable Objects (SCO) which in turn has one or more properties associated to the data returned from the search. - -##### An example of a STIX Observation: - -```json -{ - "id": "observed-data--cf2c58dc-200e-49e0-b6f7-e1997cccf707", - "type": "observed-data", - "created_by_ref": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d8", - "objects": { - "0": { - "type": "network-traffic", - "src_port": 567, - "dst_port": 102, - "src_ref": "1", - "dst_ref": "2" - }, - "1": { - "type": "ipv4-addr", - "value": "192.168.122.84" - }, - "2": { - "type": "ipv4-addr", - "value": "127.0.0.1" - }, - "3": { - "type": "url", - "value": "www.testaddress.com" - } - } -} -``` - -As anyone with experience in data science will tell you, the cleansing and normalizing of the data across domains, is one of the largest hurdles to overcome with attempting to build cross-platform security analytics. This is one of the barriers we are attempting to break down with STIX Shifter. - -### This sounds like Sigma, I already have that - -[Sigma](https://github.com/Neo23x0/sigma) and STIX Patterning have goals that are related, but at the end of the day has slightly different scopes. While Sigma seeks to be "for log files what Snort is for network traffic and YARA is for files", STIX Patterning's goal is to encompass _all three_ fundamental security data source types - network, file, and log - and do so simultaneously, allowing you to create complex queries and analytics that span domains. As such, so does STIX Shifter. It is critical to be able to create search patterns that span SIEM, Endpoint, Network, and File levels, in order to detect the complex patterns used in modern campaigns. - -### What is a STIX-SHIFTER connector? - -A STIX-shifter connector is a module inside Stix-Shifter library that implements an interface for: - -- data source query and result set translation -- data source communication - -Developing a new connector expands on the data sources that STIX-shifter can support. - -The combination of translation and transmission functions allows for a single STIX pattern to generate a native query for each supported data source. Each query is run, and the results are translated back into STIX objects; allowing for a uniform presentation of data. - -The objective is to have all the security data, regardless of the data source to look and behave the same. - -### Why would I want to use this? - -You might want to use this library and contribute to development, if any of the following are true: - -- You are a vendor or project owner who wants to add some form of query or enrichment functions to your product capabilities -- You are an end user and want to have a way to script searches and/or queries as part of your orchestration flow -- You are a vendor or project owner who has data that can be made available, and you want to contribute a connector -- You just want to help make the world a safer place! - -Take a look at the [**currently available connectors**](CONNECTORS.md). - - -## How to use - -### Prerequisites - -Python 3.8 or greater is required to use stix-shifter. - -Stix-shifter provides several functions: `translate` and `transmit` are the primary functions, `execute` offers a way to test the complete stix-shifter flow. - -1. [**Translate**](#translate) - - The `translate` command converts STIX patterns into data source queries (in whatever query language the data source might use) and translates data source results (in JSON format) into bundled STIX observation objects. - -2. [**Transmit**](#transmit) - - The `transmit` command allows stix-shifter to connect with products that house repositories of cybersecurity data. Connection and authentication credentials are passed to the data source APIs where stix-shifter can make calls to ping the data source, make queries, delete queries, check query status, and fetch query results. - -3. [**Execute**](#execute) - - The translation and transmission functions can work in sequence by using the `execute` command from the CLI. - -## Translate - -### CLI Arguments - -| Argument | Description | Accepted Input | -| -------- | ------------ | --------------- | -| TRANSLATION KEYWORD | The keyword specifiying a function will be used to translate queries and results. | `translate` | -| MODULE NAME | The name of the connector being used. This is the module directory name as it appears in [stix_shifter_modules](stix_shifter_modules/). If the connector supports multiple dialects, then by default a query will be generated for each one. You may specifiy a specific dialect by adding `:` directly after the module name | The connector module name with an optional `:` | -| TRANSLATION DATA TYPE | The type of data you wish to translate. This will be `query` for translating STIX patterns to native queries and `results` for translating data source results to STIX. | `query` or `results` | -| STIX IDENTITY OBJECT | This is an object that represents the data source being queried and is inserted into the results bundle of STIX objects. An empty object `"{}"` may be used for the query translation. This must be wrapped in quotes to use with the CLI. | A stringified STIX identity object | -| TRANSLATION DATA | This is the STIX pattern for query translation and a list of JSON results for results translation. This must be wrapped in quote to use with the CLI. | A stringified STIX pattern or list of JSON data | -| OPTIONS | An object of optional parameters. This must be wrapped in quotes to use with the CLI. | A stringified object of options | - -#### CLI Options - -These are general translation options defined in [`config.json`](stix_shifter_modules/config.json) that can apply to all connectors but may be overwritten by individual modules. - -| Option | Translation Data Type | Description | Accepted Values | -| ------ | --------------------- | ----------- | -------------- | -| result_limit | query | The max number of results that can be returned from a query. This value is generally included in translated queries before getting sent to the data source's API query call. The default is `10000` | A number between `1` and `500000` | -| time_range | query | A default time range, in minutes, applied to the translated query when no `START STOP` qualifier is present in the STIX pattern. As an example, this would be the `last x minutes` in a SQL query. The default is `5` | A number between `1` and `10000` | -| dialects | query | Dialects to be used for pattern translation. This will determine what `from_stix_map.json` files will be used. | A list of one or more dialect strings supported by the connector | -| validate_pattern | query | Specifies if pattern validation is run during the query translation call. This can catch errors in the submitted STIX pattern that would otherwise raise exceptions during translation. | `true` or `false` | -| stix_validator | results | Specifies if validation is run on the bundle of STIX data returned with results translation. This is performance intensive and should be used on a small result set. The default if `false`. | `true` or `false` | -| unmapped_fallback | results | If set to `true`, any results data returned, that is not specifired in the to-STIX mapping, will be included in the results in the following STIX object:property format `x-:`. The default is `false` | `true` or `false` | -| stix_2.1 | results | Results are returned as STIX 2.0 objects by default. Setting this option will return results in STIX 2.1 format. The default is `false` | `true` or `false` | - - -### 1. Translate a STIX pattern to a native data source query - -#### INPUT: STIX 2 pattern - -``` -# STIX Pattern: -"[url:value = 'http://www.testaddress.com'] OR [ipv4-addr:value = '192.168.122.84']" -``` - -#### OUTPUT: Native data source query - -``` -# Translated Query (using SQL as an example): -"SELECT * FROM tableName WHERE (Url = 'http://www.testaddress.com') -OR -((SourceIpV4 = '192.168.122.84' OR DestinationIpV4 = '192.168.122.84'))" -``` - -### CLI Command - -Open a terminal and navigate to your python 3 environment. Translation of a **query** is called in the format of: - -`stix-shifter translate query "" "" ""` - -Alternatively, you can run the CLI commands from the source. Open a terminal and navigate to the stix-shifter root directory. Translation of a **query** is called in the format of: - -`python main.py translate query "" "" ""` - -The module name refers to the name of the folder in stix-shifter that contains the connector code. The current module names can be found in the [Available Connectors](#available-connectors) table above. The STIX identity object is only used when translating data source results into STIX, so it can be passed in as an empty object for query translation calls. - -Using the Qradar connector as an example: - -`python main.py translate qradar query "{}" "[url:value = 'http://www.testaddress.com'] OR [ipv4-addr:value = '192.168.122.84']"` - -### Pattern translation using an input file - -Create a text file with the pattern you wish to translate. The file can be used in the query translation call using standard input. - -_pattern.txt_ -``` -[network-traffic:src_ref.value = '127.0.0.1'] OR [ipv4-addr:value = '0.0.0.0'] -``` - -`python main.py translate qradar query '{}' '' < /path/to/file/pattern.txt` - -### 2. Translate a JSON data source query result to a STIX 2.0 bundle of observable objects - -#### INPUT: JSON data source query result - -``` -# Datasource results (in JSON format): -[ - { - "SourcePort": 567, - "DestinationPort": 102, - "SourceIpV4": "192.168.122.84", - "DestinationIpV4": "127.0.0.1", - "Url": "www.testaddress.com" - } -] -``` - -#### OUTPUT: STIX 2.0 bundle of observable objects - -``` -# STIX Observables -{ - "type": "bundle", - "id": "bundle--2042a6e9-7f34-4a03-a745-502e358594c3", - "spec_version": "2.0", - "objects": [ - { - "type": "identity", - "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d8", - "name": "YourDataSource", - "identity_class": "events" - }, - { - "id": "observed-data--cf2c58dc-200e-49e0-b6f7-e1997cccf707", - "type": "observed-data", - "created_by_ref": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d8", - "objects": { - "0": { - "type": "network-traffic", - "src_port": 567, - "dst_port": 102, - "src_ref": "1", - "dst_ref": "2" - }, - "1": { - "type": "ipv4-addr", - "value": "192.168.122.84" - }, - "2": { - "type": "ipv4-addr", - "value": "127.0.0.1" - }, - "3": { - "type": "url", - "value": "www.testaddress.com" - } - } - } - ] -} -``` - -### CLI Command - -Open a terminal and navigate to your python 3 environment. Translation of a **results** is called in the format of: - -`stix-shifter translate result '' ''` - -Alternatively, you can run the CLI commands from the source. Open a terminal and navigate to the stix-shifter root directory. Translation of **results** is called in the format of: - -`python main.py translate result '' ''` - -The module name refers to the name of the folder in stix-shifter that contains the connector code. The current module names can be found in the [Available Connectors](#available-connectors) table above. The STIX Identity object represents the data source and is passed in to allow stix-shifter to create a reference between the data source and the generated STIX observed objects. - -Using the QRadar connector as an example: - -``` -python main.py translate qradar results \ -'{"type": "identity", "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", "name": "QRadar", "identity_class": "events"}' \ -'[{"sourceip": "192.0.2.0", "filename": "someFile.exe", "sourceport": "0123", "username": "root"}]' -``` - -### Translating results into STIX 2.1 - -By default, JSON results are translated into STIX 2.0. To return STIX 2.1 results include `'{"stix_2.1": true}'` in the CLI command - -``` -python main.py translate qradar results \ -'{"type": "identity", "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", "name": "QRadar", "identity_class": "events"}' \ -'[{"sourceip": "192.0.2.0", "filename": "someFile.exe", "sourceport": "0123", "username": "root"}]' '{"stix_2.1": true}' -``` - - -### Validating translated STIX 2.1 bundle from the CLI - -You can validate translated STIX results from the CLI provided they conform to the 2.1 standard. The `--stix-validator` flag at the end will run validation on the returned STIX objects to ensure they conform to the STIX 2.1 standard. Alternatively, `'{ "stix_validator": true }'` can be passed in at the end as an options dictionary. - -``` -python main.py translate qradar results \ -'{"type": "identity", "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", "name": "QRadar", "identity_class": "events"}' \ -'[{"sourceip": "192.0.2.0", "filename": "someFile.exe", "sourceport": "0123", "username": "root"}]' '{"stix_2.1": true, "stix_validator: true}' -``` - -### Validating STIX 2.0 and 2.1 bundles with the validator script - -Refer to the [STIX validator](bundle_validator/README.md) - -### Results translation using an input file - -Create a JSON file with the results you wish to translate. The file can be used in the results translation call using standard input. - -_results.json_ -``` -[ - { - "starttime": "1563892019916", - "endtime": "1563892019916", - "sourceip": "9.21.122.127", - "sourceport": "100", - "identityip": "0.0.0.0", - "destinationip": "127.0.0.1", - "destinationport": "800", - "username": "admin", - "protocol": "tcp" - } -] -``` - -`python main.py translate qradar results '{"type": "identity","id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff","name": "QRadar","identity_class": "system"}' '' < /path/to/file/results.json` - -## Transmit - -### Connection and Configuration objects - -STIX-shifter expects connection and configuration objects to be passed in during transmission calls. The connection object contains the host address and port of the data source being connected to, as well as an optional self signed certificate. - -### Connection - -This object contains information needed to connect to a specific data source. The `host` and `port` keys are required. - -``` -{ - "host": , - "port": , - "selfSignedCert": , - "cert": , - "resultSizeLimit": , - "timeRange": , - "options": {} -} -``` - -### Connection Options - -These are general options defined in [`config.json`](stix_shifter_modules/config.json) that can apply to all connectors but may be overwritten by individual modules. - -| Option | Description | Accepted Values | -| ------ | ----------- | -------------- | -| timeout | The max amount of time in seconds before the query times out. The default is `30`. | A number between `1` and `60` | - - -### Configuration - -This object contains an `auth` key who's value stores authentication information for the data source. What keys and values get stored in the auth will depend on the authentication requirements of the data source (username, password, auth token, etc). - -``` -{ - "auth": { - "username": , - "password": - } -} -``` - -``` -{ - "auth": { - "tenant": , - "clientId": , - "clientSecret": - } -} -``` - -``` -{ - "auth": { - "SEC": - } -} -``` - -``` -{ - "auth": { - "token": - } -} -``` - -``` -{ - "auth": { - "accountId": , - "apiKey": - } -} -``` - -### Transmit functions - -Transmit offers several functions: `ping`, `query`, `status` (for asynchronous data sources), `results`, `delete` (if supported by the data source), and `is_async`. - -Each of the transmit functions takes in common arguments: the module name, the connection object, and the configuration object. The module name refers to the name of the folder in stix-shifter that contains the connector code. The current module names can be found in the [Available Connectors](#available-connectors) table above. Information on the [connection and configuration objects](#transmit) can also be found above. Each of the CLI commands can be run from a terminal in the stix-shifter root director. - -Any failed transmission function call will return an error in the format of: - -`{'success': False, 'error': , 'code': }` - - -### CLI Arguments - -| Argument | Description | Accepted Input | -| -------- | ------------ | --------------- | -| TRANSMISSION KEYWORD | The keyword specifiying a function will be used to transmit API calls to the target data source. | `transmit` | -| MODULE NAME | The name of the connector being used. This is the module directory name as it appears in [stix_shifter_modules](stix_shifter_modules/). | The connector module name | -| CONNECTION OBJECT | This contains the information needed to connect to the target data source, such as host and port. This must be wrapped in quotes to use with the CLI. | A stringified connection object | -| CONFIGURATION OBJECT | This contains the information needed to authenticate with the target data source, such as username and password. This must be wrapped in quotes to use with the CLI. | A stringified configuration object | -| TRANSMISSION FUNCTION | The transmission function used to communicate with the target data source. | `is_async`, `ping`, `query`, `status`, `results`, `delete` | - -### Transmission Functions and Arguments - -| Function | Description | Function Argument | Function Returns | -| ------ | --------------------- | ----------- | -------------- | -| is_async | Checks if the connector is asynchronous. | NA | `true` or `false` -| ping | Calls the data source ping API endpont (or equivalent) to see if a connection can be made.| NA | Object containing `success` of `true` or `false` | -| query | Sends a native query string, as translated from the STIX pattern, to target data source API. | Tranlated query string | Query string | Object containing `success` of `true` or `false` and the `search_id`. If the connector is synchronous, the search_id will be the original query string. | -| status | Checks the status of a query. Only used by asynchronous connectors. | The `search_id` returned from the query call | Object continaing `success` of `true` or `false`, `status` of `RUNNING`, `COMPLETED`, `CANCELED`, or `ERROR`, and `progress` with a number indicating the percentage complete. | -| results | Fetches the native results of a completed query. | The `search_id` returned from the query call followed by OFFSET and LENGTH as numbers | A list of JSON results | -| delete | Deletes a query from the target data source | The `search_id` returned from the query call | Object continaing `success` of `true` or `false` | -| results_stix | Fetches the results of a completed query and runs results-to-stix translation. This essentially combines the `results` transmission function with the results translation | The `search_id` returned from the query call, followed by OFFSET and LENGTH as numbers, followed by the stringified STIX identity object. | A bundle of STIX objects. | - -### Ping - -Uses the data source API to ping the connection. - -#### CLI Command - -`stix-shifter transmit '' '' ping` - -#### OUTPUT: - -`{'success': True}` - -### Query - -Uses the data source API to submit a query to the connection. - -#### CLI Command - -`stix-shifter transmit '' '' query ` - -#### OUTPUT: - -`{'success': True, 'search_id': }` - -An asynchronous data source will typically return a search ID supplied by the API response. In the event where the API doesn't return a search id, such as with a synchronous data source, the search id will be defined in the transmission module. - -### Status - -Uses the data source API to look up the query status based on the `search_id` that is returned from the query call. This is only used for asynchronous data sources where the results are not returned right after making a query call. If the connector supports, you can specify `metadata` parameter which may contain extra information to make the status api call. - -#### CLI Command - -`stix-shifter transmit '' '' status ` - -#### OUTPUT: - -`{'success': True, 'status': , 'progress': }` - -The status can be one of: `COMPLETED`, `ERROR`, `CANCELLED`, `TIMEOUT`, or `RUNNING`. Depending on the data source, the progress may return with less than 100 while still showing the status as completed. - -### Results - -Uses the data source API to fetch the query results based on the search ID, offset, and length. - -If the connector supports, you can specify `metadata` parameter which may contain extra information to fetch the next batch of results from the datasource. This is a recomended parameter for the datasource that supports pagination. - -#### CLI Command - -`stix-shifter transmit '' '' results ` - -The `OFFSET` and `LENGTH` control what pages/rows of data are returned in the query results. - -#### OUTPUT: - -`{'success': True, 'data': []}` - -#### OUTPUT(with metadata): - -`{'success': True, 'data': [], 'metadata': }` - -#### Example: -``` -{ - "success": true, - "data": [ - { - "event": { - "securityEvent": { - "eventTimestamp": "2022-06-13T14:36:54.216539700Z", - "eventType": "FILE_CREATION", - "vendorName": "Microsoft", - "productEventType": "DeviceFileEvents", - "ingestedTimestamp": "2022-06-13T15:36:26.275010Z" - }, - "securityResult": [ - { - "summary": "FileCreated", - "category": "alert" - } - ] - } - } - ], - "metadata": { - "result_count": 2, - "next_page_token": "CgwIlqLjoAYQ2NfggwESCwiGl52VBhC0xKB" - } -} -``` - -### Results as STIX - -Uses the data source API to fetch the query results based on the search ID, offset, and length, and transforms into a bundle of STIX objects. - -#### CLI Command - -`stix-shifter transmit '' '' results_stix ''` - -#### OUTPUT: - -STIX bundle of objects. - -The `OFFSET` and `LENGTH` control what pages/rows of data are returned in the query results. - -### Is Async - -Checks if the data source connection is asynchronous. - -#### CLI Command - -`stix-shifter transmit '' '' is_async` - -#### OUTPUT: - -`True` or `False` - -## Execute - -The `execute` command tests all steps of the translation-transmission flow: - -1. A STIX pattern is translated into a list of one or more native data source queries (using a **translate query** call). -2. Each translated query in the list is sent to the data source via a **transmit query** call. -3. If the data source is asynchronous, a **transmit status** call is made for each query. Otherwise the flow moves to the next step. -4. A **transmit results** call is made for each query (using the returned query ID in step 2). If data is returned, the resulting JSON objects get added to a list. -5. The list of JSON results get translated into a bundle of STIX objects with a **translate query** call. This bundle includes the STIX `identity` object and `observed-data` objects. - -### CLI Arguments - -| Argument | Description | Accepted Input | -| -------- | ------------ | --------------- | -| EXECUTE KEYWORD | The keyword specifiying that the execute function will be used. | `execute` | -| TRANSMISSION MODULE NAME | The name of the connector being used for transmission functions. This is the module directory name as it appears in [stix_shifter_modules](stix_shifter_modules/). | The connector module name | -| TRANSLATION MODULE NAME | The name of the connector being used for translation functions. This is the module directory name as it appears in [stix_shifter_modules](stix_shifter_modules/). | The connector module name | -| STIX IDENTITY OBJECT | This is an object that represents the data source being queried and is inserted into the results bundle of STIX objects. An empty object `"{}"` may be used for the query translation. This must be wrapped in quotes to use with the CLI. | A stringified STIX identity object | -| CONNECTION OBJECT | This contains the information needed to connect to the target data source, such as host and port. This must be wrapped in quotes to use with the CLI. | A stringified connection object | -| CONFIGURATION OBJECT | This contains the information needed to authenticate with the target data source, such as username and password. This must be wrapped in quotes to use with the CLI. | A stringified configuration object | -| STIX PATTERN | This is the STIX pattern to be used for the query. This must be wrapped in quote to use with the CLI. | A stringified STIX pattern | - -### Connection Object Options - -These are general options defined in [`config.json`](stix_shifter_modules/config.json) that can apply to all connectors but may be overwritten by individual modules. These should be added as an "options" objects inside the CONNECTION OBJECT. - -| Option | Function Type | Description | Accepted Values | -| ------ | --------------------- | ----------- | -------------- | -| result_limit | query translation | The max number of results that can be returned from a query. This value is generally included in translated queries before getting sent to the data source's API query call. The default is `10000` | A number between `1` and `500000` | -| time_range | query translation | A default time range, in minutes, applied to the translated query when no `START STOP` qualifier is present in the STIX pattern. As an example, this would be the `last x minutes` in a SQL query. The default is `5` | A number between `1` and `10000` | -| dialects | query translation | Dialects to be used for pattern translation. This will determine what `from_stix_map.json` files will be used. | A list of one or more dialect strings supported by the connector | -| validate_pattern | query translation | Specifies if pattern validation is run during the query translation call. This can catch errors in the submitted STIX pattern that would otherwise raise exceptions during translation. | `true` or `false` | -| stix_validator | results translation | Specifies if validation is run on the bundle of STIX data returned with results translation. This is performance intensive and should be used on a small result set. The default if `false`. | `true` or `false` | -| unmapped_fallback | results translation | If set to `true`, any results data returned, that is not specifired in the to-STIX mapping, will be included in the results in the following STIX object:property format `x-:`. The default is `false` | `true` or `false` | -| stix_2.1 | results translation | Results are returned as STIX 2.0 objects by default. Setting this option will return results in STIX 2.1 format. The default is `false` | `true` or `false` | -| timeout | transmission | The max amount of time in seconds before the query times out. The default is `30`. | A number between `1` and `60` | - - -### CLI Command - -`stix-shifter execute '' '' '' ''` - -### CLI Example - -`stix-shifter execute mysql mysql '{"type": "identity","id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff","name": "mysql","identity_class": "system"}' '{"host": "localhost", "database":"demo_db", "options":{"table":"demo_table", "validate_pattern": true}}' '{"auth": {"username":"root", "password":"MyPassword"}}' "[ipv4-addr:value = '213.213.142.5'] START t'2019-01-28T12:24:01.009Z' STOP t'2019-01-28T12:54:01.009Z'"` - -### Debug - -You can add the `--debug` option to your CLI command to see more logs. - -`stix-shifter --debug execute '' '' '' ''` - -### Change max returned results - -You can add the `--results` option with an integer value at the end of your CLI command to limit the maximum number of returned search results (default 10). - -`stix-shifter execute '' '' '' '' --results 50` - -### Save the STIX results to a file - -You can redirect the output of your CLI command to a file to save the STIX results. - -`stix-shifter execute '' '' '' '' > results.json` - -### OUTPUT: - -A bundle of STIX objects - -## Modules - -The `modules` command will return a JSON of the existing connectors along with their dialects and supported languages that are used in query translation. - -### CLI Command - -`python main.py modules` - -#### output -``` -{ - "qradar": { - "flows": { - "language": "stix", - "default": true - }, - "events": { - "language": "stix", - "default": true - }, - "aql": { - "language": "aql", - "default": false - } - }, - "security_advisor": { - "default": { - "language": "stix", - "default": true - } - }, - ... -} -``` - -This command can also be used to get the dialects of a specific connector. - -`python main.py modules ` - -### CLI Command - -`python main.py modules qradar` - -#### output -``` -{ - "qradar": { - "flows": { - "language": "stix", - "default": true - }, - "events": { - "language": "stix", - "default": true - }, - "aql": { - "language": "aql", - "default": true - } - } -} -``` - -In the above example, the QRadar connector can use three dialects: `flows`, `events`, and `aql`. When a connector only has a `default` dialect, such as with Security Advisor, only one dialect is used by the connector. Most dialects will use the `stix` language since they translate STIX patterns into native queries. QRadar's `aql` dialect uses the `aql` language since it is meant to accept an AQL query rather than a STIX pattern. See the [QRadar connector README](stix_shifter_modules/qradar/README.md) for more information on AQL passthrough. - -## configs - -The `configs` command returns the configuration pararmetes of the existing connectors. It basically returns a JSON of the existing connectors along with their connections and configuation objects that are specified in config.json. - -### CLI Command - -`python main.py configs` -#### output -``` -{ - "alertflex": { - "connection": { - "type": { - "type": "connectorType", - "displayName": "Alertflex" - }, - "options": { - "type": "fields", - "async_call": { - "type": "text", - "hidden": true, - "optional": true - }, - "result_limit": { - "default": 10000, - "min": 1, - "max": 500000, - "type": "number", - "previous": "connection.resultSizeLimit" - }, - "time_range": { - "default": 5, - "min": 1, - "max": 10000, - "type": "number", - "previous": "connection.timerange", - "nullable": true - }, - ..... - } - } - } -} -``` - -Specifying the connector module name will return the configuration parameters of a specific connector. - -### CLI Command - -`python main.py configs ` -#### output -``` -{ - "qradar": { - "connection": { - "type": { - "type": "connectorType", - "displayName": "IBM\u00ae QRadar and QRadar On Cloud", - "group": "qradar" - }, - "options": { - "type": "fields", - "async_call": { - "type": "text", - "hidden": true, - "optional": true - }, - "result_limit": { - "default": 10000, - "min": 1, - "max": 500000, - "type": "number", - "previous": "connection.resultSizeLimit" - }, - "time_range": { - "default": 5, - "min": 1, - "max": 10000, - "type": "number", - "previous": "connection.timerange", - "nullable": true - }, - "timeout": { - "default": 30, - "min": 1, - "max": 60, - "hidden": true, - "type": "number", - "previous": "connection.timeoutLimit" - }, - "dialects": { - "type": "array", - "hidden": true, - "optional": true - }, - "language": { - "type": "string", - "default": "stix", - "optional": true, - "hidden": true - }, - "validate_pattern": { - "type": "boolean", - "optional": true, - "hidden": true, - "previous": "connection.validate_pattern", - "default": false - }, - "stix_validator": { - "type": "boolean", - "default": false, - "optional": true, - "hidden": true, - "previous": "connection.stix_validator" - }, - "mapping": { - "type": "json", - "optional": true, - "previous": "connection.mapping" - }, - "unmapped_fallback": { - "type": "boolean", - "default": true, - "optional": true, - "hidden": true - }, - "stix_2.1": { - "type": "boolean", - "default": false, - "optional": true, - "hidden": true - } - }, - "host": { - "type": "text", - "regex": "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9_:/\\-]*[a-zA-Z0-9])\\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9_:/\\-]*[A-Za-z0-9])$" - }, - "port": { - "type": "number", - "default": 443, - "min": 1, - "max": 65535 - }, - "help": { - "default": "data-sources-qradar.html", - "type": "link" - }, - "selfSignedCert": { - "type": "password", - "optional": true - } - }, - "configuration": { - "auth": { - "type": "fields", - "sec": { - "type": "password", - "previous": "configuration.auth.SEC" - } - } - } - } -} -``` -## Limitations - -STIX-Shifter has limitations on the length of a pattern that can be translated into a native query. As the pattern length increases, the translation time increases exponentially due to how ANTLR 4 parses the pattern. See [STIX-Shifter Limitations](adapter-guide/stix-shifter-limitations.md) for more details. - -## Glossary - -| Terms | Definition | -| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Modules | Folders in the stix-shifter project that contains code that is specific to a data source. | -| STIX 2 patterns | STIX patterns are expressions that represent Cyber Observable objects within a STIX Indicator STIX Domain Objects (SDOs).
They are helpful for modeling intelligence that indicates cyber activity. | -| STIX 2 objects | JSON objects that contain CTI data. In STIX, these objects are referred to as Cyber Observable Objects. | -| Data sources | Security products that house data repositories. | -| Data source queries | Queries written in the data source's native query language. | -| Data source query results | Data returned from a data source query. | - -## Architecture Context - -![STIX SHIFTER CLASS DIAGRAM](./adapter-guide/images/architecture.png) - -## Contributing - -We are thrilled you are considering contributing! We welcome all contributors. - -Please read our [guidelines for contributing](CONTRIBUTING.md). - -## Guide for creating new connectors - -If you want to create a new connector for STIX-shifter, see the [developer guide](adapter-guide/develop-stix-adapter.md) - -## Licensing - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index b9e799085..000000000 --- a/docs/README.md +++ /dev/null @@ -1,131 +0,0 @@ -# Introduction to STIX-Shifter - -STIX-shifter is an open source python library allowing software to connect to products that house data repositories by using STIX Patterning, and return results as STIX Observations. - -This library takes in STIX 2 Patterns as input, and "finds" data that matches the patterns inside various products that house repositories of cybersecurity data. Examples of such products include SIEM systems, endpoint management systems, threat intelligence platforms, orchestration platforms, network control points, data lakes, and more. - -In addition to "finding" the data by using these patterns, STIX-Shifter also _transforms the output_ into STIX 2 Observations. Why would we do that you ask? To put it simply - so that all of the security data, regardless of the source, mostly looks and behaves the same. - -## Project Overview and CLI Commands - -```{include} OVERVIEW.md -``` - -# Installation - -The recommended method for installing stix-shifter is via pip. Two prerequisite packages needs to be installed inlcuding the package of stix-shifter connector module to complete a stix-shifter connector installation. Run the below commands to install all the packages: - -1. Main stix-shifter package: `pip install stix-shifter` - -2. Stix-shifter Utility package: `pip install stix-shifter-utils` - -3. Desired stix-shifter connector module package: `pip install stix-shifter-modules- ` - Example: `pip install stix-shifter-modules-qradar` - -## Dependencies - -STIX-shifter requries Python 3.8 or greater. See the [requirements file](stix_shifter/requirements.txt) for library dependencies. - -# Usage - -STIX-Shifter can use used the following ways: - -## 1. As a command line utility - -The STIX-Shifter comes with a bundled script which you can use to translate STIX Pattern to a native datasource query. It can also be used to translate a JSON data source query result to a STIX bundle of observable objects. You can also send query to a datasource by using a transmission option. - -More details of the command line option can be found [here](https://github.com/opencybersecurityalliance/stix-shifter/blob/master/OVERVIEW.md#how-to-use) - -``` -$ stix-shifter translate query "" "" "" -``` -Example: -``` -$ stix-shifter translate qradar query {} "[ipv4-addr:value = '127.0.0.1']" {} -``` - -In order to build `stix-shifter` packages from source follow the below prerequisite steps: - 1. Go to the stix-shifter parent directory - 2. Optionally, you can create a Python 3 virtual environemnt: - `virtualenv -p python3 virtualenv && source virtualenv/bin/activate` - 3. Run setup: `python3 setup.py install` - - -## 2. Running from the source - -You may also use the `python3 main.py` script. All the options are the same as the command line utility described above. - -Example: - -``` -python3 main.py translate qradar query {} "[ipv4-addr:value = '127.0.0.1']" {} -``` - -In order to run `python3 main.py` from the source follow the below prerequisite steps: - 1. Go to the stix-shifter parent directory - 2. Optionally, you can create a Python 3 virtual environemnt: - `virtualenv -p python3 virtualenv && source virtualenv/bin/activate` - 3. Run setup to install dependancies: `INSTALL_REQUIREMENTS_ONLY=1 python3 setup.py install`. - -**Note:** setup.py only installs dependencies when INSTALL_REQUIREMENTS_ONLY=1 directive is used. This option is similar to `python3 generate_requirements.py && pip install -r requirements.txt` - -## 3. As a library - -You can also use this library to integrate STIX Shifter into your own tools. You can translate a STIX Pattern: - -``` -from stix_shifter.stix_translation import stix_translation - -translation = stix_translation.StixTranslation() -response = translation.translate('', 'query', '{}', '', '') - -print(response) -``` -### Use of custom mappings - -If a connector has been installed using pip, the process for editing the STIX mappings is different than if you have pulled-down the project. When working locally, you can edit the mapping files directly. See the [mapping files for the MySQL connector](stix_shifter_modules/mysql/stix_translation/json) as an example. Editing the mapping files won't work if the connector has been installed with pip; the setup script of the stix-shifter package includes the mapppings inside `config.json`. This allows stix-shifter to injest custom mappings as part of the connector's configuration. - -Refer to [Use of custom mappings](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/adapter-guide/custom_mappings.md) for more details on how to edit the mappings in the configuration. - -# Contributing - -We are thrilled you are considering contributing! We welcome all contributors. -Please read our [guidelines for contributing](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/CONTRIBUTING.md). - -## Connector Developer Guide -```{include} /adapter-guide/develop-stix-adapter.md -``` - - - -## [Jupyter Notebook Development Labs](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/lab) - - - -# Licensing - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -# More Resources - -## Join us on Slack! - -[Click here](https://docs.google.com/forms/d/1vEAqg9SKBF3UMtmbJJ9qqLarrXN5zeVG3_obedA3DKs) and fill out the form to receive an invite to the Open Cybersecurity Alliance slack instance, then join the #stix-shifter channel, to meet and discuss usage with the team. - -## Introduction Webinar! - -[Click here](https://ibm.biz/BdzTyA) to view an introduction webinar on STIX Shifter and the use cases it solves for. - -## Changelog - -- [Changelog](https://github.com/opencybersecurityalliance/stix-shifter/blob/develop/CHANGELOG.md) \ No newline at end of file From 34756f8af841af13c2af632350f4709db7ef29ef Mon Sep 17 00:00:00 2001 From: Md Azam Date: Thu, 3 Aug 2023 15:28:52 -0300 Subject: [PATCH 19/55] myst design --- docs/conf.py | 23 ++++++++++++++++++++++- docs/index.md | 17 ++++++++--------- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 1fde5bafe..ea6a06492 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,9 +15,30 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [ - "myst_parser" + "myst_parser", + "sphinx_design", ] +# -- MyST settings --------------------------------------------------- + +myst_enable_extensions = [ + "dollarmath", + "amsmath", + "deflist", + "fieldlist", + "html_admonition", + "html_image", + "colon_fence", + "smartquotes", + "replacements", + "linkify", + "strikethrough", + "substitution", + "tasklist", + "attrs_inline", + "attrs_block", +] + templates_path = ['_templates'] exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] diff --git a/docs/index.md b/docs/index.md index 6feac946c..7b14a0f97 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,5 +1,4 @@ -```{rubric} STIX-Shifter -``` +# Introduction to STIX-Shifter STIX-shifter is an open source python library allowing software to connect to products that house data repositories by using STIX Patterning, and return results as STIX Observations. @@ -7,10 +6,10 @@ This library takes in STIX 2 Patterns as input, and "finds" data that matches th In addition to "finding" the data by using these patterns, STIX-Shifter also _transforms the output_ into STIX 2 Observations. Why would we do that you ask? To put it simply - so that all of the security data, regardless of the source, mostly looks and behaves the same. - -{doc}`overview` - -```{toctree} -:hidden: -overview.md -``` \ No newline at end of file +:::{card} Overview +Header +^^^ +For general information about STIX, this project, and the command line utilities, see the [STIX-shifter Overview](OVERVIEW.md) ++++ +Footer +::: \ No newline at end of file From 4514f8c229bdceb7d5f928d30f4ec7579c458b1d Mon Sep 17 00:00:00 2001 From: Md Azam Date: Thu, 3 Aug 2023 15:32:34 -0300 Subject: [PATCH 20/55] myst design version added --- docs/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 2b3136e74..6ed439329 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,3 @@ Sphinx==7.1.1 -myst-parser==2.0.0 \ No newline at end of file +myst-parser==2.0.0 +sphinx_design==0.5.0 \ No newline at end of file From 0f2f3f17c5c69b22cab270c6b972df4661a65b94 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Thu, 3 Aug 2023 15:37:11 -0300 Subject: [PATCH 21/55] linkify-it-py==2.0.2 --- docs/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 6ed439329..5eddec8bd 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,4 @@ Sphinx==7.1.1 myst-parser==2.0.0 -sphinx_design==0.5.0 \ No newline at end of file +sphinx_design==0.5.0 +linkify-it-py==2.0.2 \ No newline at end of file From 68d084642101892b5113aa71877d30d623051cb8 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Thu, 3 Aug 2023 15:47:07 -0300 Subject: [PATCH 22/55] card added --- docs/index.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/index.md b/docs/index.md index 7b14a0f97..ea233a1e4 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,10 +6,12 @@ This library takes in STIX 2 Patterns as input, and "finds" data that matches th In addition to "finding" the data by using these patterns, STIX-Shifter also _transforms the output_ into STIX 2 Observations. Why would we do that you ask? To put it simply - so that all of the security data, regardless of the source, mostly looks and behaves the same. -:::{card} Overview -Header -^^^ +::::{grid} 1 2 2 3 +:gutter: 1 1 1 2 + +:::{grid-item-card} {octicon}`markdown;1.5em;sd-mr-1` Overview +:link: OVERVIEW.md +:link-type: ref + For general information about STIX, this project, and the command line utilities, see the [STIX-shifter Overview](OVERVIEW.md) -+++ -Footer ::: \ No newline at end of file From 783284e42dabc76de92759fd814b5a7e2e4b3ab5 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Thu, 3 Aug 2023 15:56:26 -0300 Subject: [PATCH 23/55] card added2 --- docs/index.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/index.md b/docs/index.md index ea233a1e4..8041407ac 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,9 +9,19 @@ In addition to "finding" the data by using these patterns, STIX-Shifter also _tr ::::{grid} 1 2 2 3 :gutter: 1 1 1 2 -:::{grid-item-card} {octicon}`markdown;1.5em;sd-mr-1` Overview +:::{grid-item-card} `markdown;1.5em;sd-mr-1` Overview :link: OVERVIEW.md :link-type: ref -For general information about STIX, this project, and the command line utilities, see the [STIX-shifter Overview](OVERVIEW.md) +For general information about STIX, this project, and the command line utilities, see the STIX-shifter Overview. +::: + +::::{grid} 1 2 2 3 +:gutter: 1 1 1 2 + +:::{grid-item-card} `markdown;1.5em;sd-mr-1` Developer Guide +:link: (adapter-guide/develop-stix-adapter.md) +:link-type: ref + +Follow the developer guide to learn about developing a new STIX-Shifter connector. ::: \ No newline at end of file From ffdf5596233b3c04cf74b8007a0b2d3ca52512c3 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Thu, 3 Aug 2023 16:00:18 -0300 Subject: [PATCH 24/55] card added3 --- docs/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.md b/docs/index.md index 8041407ac..cccbaf3fe 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,7 +9,7 @@ In addition to "finding" the data by using these patterns, STIX-Shifter also _tr ::::{grid} 1 2 2 3 :gutter: 1 1 1 2 -:::{grid-item-card} `markdown;1.5em;sd-mr-1` Overview +:::{grid-item-card}{octicon} `markdown;1.5em;sd-mr-1` Overview :link: OVERVIEW.md :link-type: ref @@ -19,7 +19,7 @@ For general information about STIX, this project, and the command line utilities ::::{grid} 1 2 2 3 :gutter: 1 1 1 2 -:::{grid-item-card} `markdown;1.5em;sd-mr-1` Developer Guide +:::{grid-item-card} {octicon}`tools;1.5em;sd-mr-1` Developer Guide :link: (adapter-guide/develop-stix-adapter.md) :link-type: ref From 3edfc38015424712f0c50789bbcf88b84b0788f4 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Thu, 3 Aug 2023 16:04:08 -0300 Subject: [PATCH 25/55] card fix --- docs/index.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/index.md b/docs/index.md index cccbaf3fe..65f00aecf 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,6 +6,9 @@ This library takes in STIX 2 Patterns as input, and "finds" data that matches th In addition to "finding" the data by using these patterns, STIX-Shifter also _transforms the output_ into STIX 2 Observations. Why would we do that you ask? To put it simply - so that all of the security data, regardless of the source, mostly looks and behaves the same. +:::: + +--- ::::{grid} 1 2 2 3 :gutter: 1 1 1 2 @@ -16,12 +19,13 @@ In addition to "finding" the data by using these patterns, STIX-Shifter also _tr For general information about STIX, this project, and the command line utilities, see the STIX-shifter Overview. ::: -::::{grid} 1 2 2 3 -:gutter: 1 1 1 2 - :::{grid-item-card} {octicon}`tools;1.5em;sd-mr-1` Developer Guide :link: (adapter-guide/develop-stix-adapter.md) :link-type: ref Follow the developer guide to learn about developing a new STIX-Shifter connector. -::: \ No newline at end of file +::: + +:::: + +--- \ No newline at end of file From bf448143ee44cb4f59a0cb45db8732d99b13c7be Mon Sep 17 00:00:00 2001 From: Md Azam Date: Thu, 3 Aug 2023 16:09:50 -0300 Subject: [PATCH 26/55] card fix2 --- docs/index.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/index.md b/docs/index.md index 65f00aecf..d9cae9e43 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,21 +9,27 @@ In addition to "finding" the data by using these patterns, STIX-Shifter also _tr :::: --- + ::::{grid} 1 2 2 3 :gutter: 1 1 1 2 -:::{grid-item-card}{octicon} `markdown;1.5em;sd-mr-1` Overview -:link: OVERVIEW.md +:::{grid-item-card} {octicon}`markdown;1.5em;sd-mr-1` Overview +:link: OVERVIEW.md :link-type: ref For general information about STIX, this project, and the command line utilities, see the STIX-shifter Overview. + ++++ +[Learn more »](OVERVIEW.md) ::: :::{grid-item-card} {octicon}`tools;1.5em;sd-mr-1` Developer Guide -:link: (adapter-guide/develop-stix-adapter.md) -:link-type: ref +:link: adapter-guide/develop-stix-adapter.md +:link-type: doc Follow the developer guide to learn about developing a new STIX-Shifter connector. ++++ +[Learn more »](adapter-guide/develop-stix-adapter.md) ::: :::: From 0e9d3b38c403ecb82b3efcbcf0e0425e57d24a29 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Thu, 3 Aug 2023 16:16:45 -0300 Subject: [PATCH 27/55] card fix3 --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index d9cae9e43..a35c08aac 100644 --- a/docs/index.md +++ b/docs/index.md @@ -25,7 +25,7 @@ For general information about STIX, this project, and the command line utilities :::{grid-item-card} {octicon}`tools;1.5em;sd-mr-1` Developer Guide :link: adapter-guide/develop-stix-adapter.md -:link-type: doc +:link-type: ref Follow the developer guide to learn about developing a new STIX-Shifter connector. +++ From ad5ada0590c43f8299787efba2a43fefac69b6a2 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Thu, 3 Aug 2023 16:18:51 -0300 Subject: [PATCH 28/55] card fix4 --- docs/index.md | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/docs/index.md b/docs/index.md index a35c08aac..da2f6ac31 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,10 +6,6 @@ This library takes in STIX 2 Patterns as input, and "finds" data that matches th In addition to "finding" the data by using these patterns, STIX-Shifter also _transforms the output_ into STIX 2 Observations. Why would we do that you ask? To put it simply - so that all of the security data, regardless of the source, mostly looks and behaves the same. -:::: - ---- - ::::{grid} 1 2 2 3 :gutter: 1 1 1 2 @@ -23,15 +19,7 @@ For general information about STIX, this project, and the command line utilities [Learn more »](OVERVIEW.md) ::: -:::{grid-item-card} {octicon}`tools;1.5em;sd-mr-1` Developer Guide -:link: adapter-guide/develop-stix-adapter.md -:link-type: ref - -Follow the developer guide to learn about developing a new STIX-Shifter connector. -+++ -[Learn more »](adapter-guide/develop-stix-adapter.md) -::: - -:::: - ---- \ No newline at end of file +```{toctree} +:hidden: +README.md +``` \ No newline at end of file From f52e094adbb657aef2bc604a468583cff640ea6b Mon Sep 17 00:00:00 2001 From: Md Azam Date: Fri, 4 Aug 2023 10:21:49 -0300 Subject: [PATCH 29/55] card fix5 --- docs/index.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/index.md b/docs/index.md index da2f6ac31..84544dee1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -19,6 +19,16 @@ For general information about STIX, this project, and the command line utilities [Learn more »](OVERVIEW.md) ::: +:::{grid-item-card} {octicon}`markdown;1.5em;sd-mr-1` Overview +:link: OVERVIEW.md +:link-type: ref + +For general information about STIX, this project, and the command line utilities, see the STIX-shifter Overview. + ++++ +[Learn more »](OVERVIEW.md) +::: + ```{toctree} :hidden: README.md From cc37ad13a326cdf51e71ee246408d129851d7350 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Fri, 4 Aug 2023 10:37:52 -0300 Subject: [PATCH 30/55] card fix6 --- docs/index.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/index.md b/docs/index.md index 84544dee1..76862c83c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,19 +11,19 @@ In addition to "finding" the data by using these patterns, STIX-Shifter also _tr :::{grid-item-card} {octicon}`markdown;1.5em;sd-mr-1` Overview :link: OVERVIEW.md -:link-type: ref +:link-type: doc For general information about STIX, this project, and the command line utilities, see the STIX-shifter Overview. +++ -[Learn more »](OVERVIEW.md) +[Learn more » OVERVIEW](OVERVIEW.md) ::: :::{grid-item-card} {octicon}`markdown;1.5em;sd-mr-1` Overview :link: OVERVIEW.md :link-type: ref -For general information about STIX, this project, and the command line utilities, see the STIX-shifter Overview. +For general information about STIX, this project, and the command line utilities, see the [STIX-shifter Overview](OVERVIEW.md). +++ [Learn more »](OVERVIEW.md) From f6b499eaffc4b8dbae959e1059a625f81fc72996 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Fri, 4 Aug 2023 10:42:38 -0300 Subject: [PATCH 31/55] card fix7 --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 76862c83c..812a17bcc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -21,7 +21,7 @@ For general information about STIX, this project, and the command line utilities :::{grid-item-card} {octicon}`markdown;1.5em;sd-mr-1` Overview :link: OVERVIEW.md -:link-type: ref +:link-type: doc For general information about STIX, this project, and the command line utilities, see the [STIX-shifter Overview](OVERVIEW.md). From 94612c0a94228c526ee29230423fb5959e8078bd Mon Sep 17 00:00:00 2001 From: Md Azam Date: Fri, 4 Aug 2023 11:06:23 -0300 Subject: [PATCH 32/55] card fix8 --- docs/index.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/index.md b/docs/index.md index 812a17bcc..9af4b3b03 100644 --- a/docs/index.md +++ b/docs/index.md @@ -24,9 +24,6 @@ For general information about STIX, this project, and the command line utilities :link-type: doc For general information about STIX, this project, and the command line utilities, see the [STIX-shifter Overview](OVERVIEW.md). - -+++ -[Learn more »](OVERVIEW.md) ::: ```{toctree} From 676b5a183e3f507aa98dc260e2bf7283657f89f6 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Fri, 4 Aug 2023 11:10:24 -0300 Subject: [PATCH 33/55] card fix9 --- docs/index.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/index.md b/docs/index.md index 9af4b3b03..f23ba44d5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -20,9 +20,6 @@ For general information about STIX, this project, and the command line utilities ::: :::{grid-item-card} {octicon}`markdown;1.5em;sd-mr-1` Overview -:link: OVERVIEW.md -:link-type: doc - For general information about STIX, this project, and the command line utilities, see the [STIX-shifter Overview](OVERVIEW.md). ::: From b75d10aeb437a876af6091f65aebec820eccd40e Mon Sep 17 00:00:00 2001 From: Md Azam Date: Fri, 4 Aug 2023 11:40:41 -0300 Subject: [PATCH 34/55] card fix10 --- docs/index.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/index.md b/docs/index.md index f23ba44d5..52b0b4992 100644 --- a/docs/index.md +++ b/docs/index.md @@ -10,17 +10,15 @@ In addition to "finding" the data by using these patterns, STIX-Shifter also _tr :gutter: 1 1 1 2 :::{grid-item-card} {octicon}`markdown;1.5em;sd-mr-1` Overview -:link: OVERVIEW.md -:link-type: doc - -For general information about STIX, this project, and the command line utilities, see the STIX-shifter Overview. +For general information about STIX, this project, and the command line utilities, see the [STIX-shifter Overview](OVERVIEW.md). +::: -+++ -[Learn more » OVERVIEW](OVERVIEW.md) +:::{grid-item-card} {octicon}`plug;1.5em;sd-mr-1` Available Connectors +There are more than 30 connectors. For the list of connectors, see the [Available Connectors](CONNECTORS.md). ::: -:::{grid-item-card} {octicon}`markdown;1.5em;sd-mr-1` Overview -For general information about STIX, this project, and the command line utilities, see the [STIX-shifter Overview](OVERVIEW.md). +:::{grid-item-card} {octicon}`tools;1.5em;sd-mr-1` Developer Guide +Follow the developer guide to learn about developing a new STIX-Shifter connector, see [Connector Developer Guide](adapter-guide/develop-stix-adapter.md). ::: ```{toctree} From beac7bac8b549ad05ad24940aa6c4d601e6aa63d Mon Sep 17 00:00:00 2001 From: Md Azam Date: Fri, 4 Aug 2023 11:48:42 -0300 Subject: [PATCH 35/55] update doctree --- docs/OVERVIEW.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/OVERVIEW.md b/docs/OVERVIEW.md index 89a2cac01..40b0a8a0c 100644 --- a/docs/OVERVIEW.md +++ b/docs/OVERVIEW.md @@ -901,3 +901,8 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +```{toctree} +:hidden: +OVERVIEW.md +``` \ No newline at end of file From d844ec9b6ef999588f0c431db18e948a2b561798 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Fri, 4 Aug 2023 13:32:05 -0300 Subject: [PATCH 36/55] update doctree2 --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 52b0b4992..394fdea0e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -22,6 +22,6 @@ Follow the developer guide to learn about developing a new STIX-Shifter connecto ::: ```{toctree} -:hidden: +:maxdepth: 2 README.md ``` \ No newline at end of file From fd07656e361659c4547223d29001f4149d154e92 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Fri, 4 Aug 2023 13:57:10 -0300 Subject: [PATCH 37/55] update doctree3 --- docs/OVERVIEW.md | 2 +- docs/index.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/OVERVIEW.md b/docs/OVERVIEW.md index 40b0a8a0c..26d56e72d 100644 --- a/docs/OVERVIEW.md +++ b/docs/OVERVIEW.md @@ -903,6 +903,6 @@ See the License for the specific language governing permissions and limitations under the License. ```{toctree} -:hidden: +:maxdepth: 2 OVERVIEW.md ``` \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 394fdea0e..6a5e3a769 100644 --- a/docs/index.md +++ b/docs/index.md @@ -22,6 +22,8 @@ Follow the developer guide to learn about developing a new STIX-Shifter connecto ::: ```{toctree} +:hidden: :maxdepth: 2 README.md +OVERVIEW.md ``` \ No newline at end of file From 0b8b4cb56f90181975379d9e64bd75fb0a28f7ba Mon Sep 17 00:00:00 2001 From: Md Azam Date: Fri, 4 Aug 2023 14:08:15 -0300 Subject: [PATCH 38/55] update doctree4 --- docs/index.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/index.md b/docs/index.md index 6a5e3a769..a3c7dbdf4 100644 --- a/docs/index.md +++ b/docs/index.md @@ -25,5 +25,32 @@ Follow the developer guide to learn about developing a new STIX-Shifter connecto :hidden: :maxdepth: 2 README.md +``` + +```{toctree} +:hidden: +:caption: ✏️ Project Overview + OVERVIEW.md +``` + +```{toctree} +:hidden: +:caption: ✏️ Available Connectors + +CONNECTORS.md +``` + +```{toctree} +:hidden: +:caption: ✏️ Developer Guide + +adapter-guide/develop-stix-adapter.md +adapter-guide/develop-translation-module.md +adapter-guide/custom_mappings.md +adapter-guide/develop-mapping-keywords.md +adapter-guide/custom_mappings.md +adapter-guide/develop-transmission-module.md +adapter-guide/develop-configuration-json.md +adapter-guide/best_practices.md ``` \ No newline at end of file From 7b68119464fb12bd7675f2a70f008635fd7fb8cd Mon Sep 17 00:00:00 2001 From: Md Azam Date: Fri, 4 Aug 2023 14:20:02 -0300 Subject: [PATCH 39/55] html theme added --- docs/OVERVIEW.md | 5 ----- docs/conf.py | 2 ++ 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/OVERVIEW.md b/docs/OVERVIEW.md index 26d56e72d..89a2cac01 100644 --- a/docs/OVERVIEW.md +++ b/docs/OVERVIEW.md @@ -901,8 +901,3 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - -```{toctree} -:maxdepth: 2 -OVERVIEW.md -``` \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index ea6a06492..352f92960 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -39,6 +39,8 @@ "attrs_block", ] +html_theme = "sphinx_book_theme" + templates_path = ['_templates'] exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] From a133b9824eb4d9e577b56ec5e0a85b81e7028cb5 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Fri, 4 Aug 2023 14:22:03 -0300 Subject: [PATCH 40/55] html theme added2 --- docs/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index 5eddec8bd..4ef7807ee 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,5 @@ Sphinx==7.1.1 myst-parser==2.0.0 sphinx_design==0.5.0 +sphinx-book-theme==1.0.1 linkify-it-py==2.0.2 \ No newline at end of file From 171f48efadac23883c5a893d5485335b01e306c0 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Fri, 4 Aug 2023 14:26:54 -0300 Subject: [PATCH 41/55] html theme added3 --- docs/conf.py | 2 +- docs/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 352f92960..6e9cf0b79 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -39,7 +39,7 @@ "attrs_block", ] -html_theme = "sphinx_book_theme" +html_theme = "sphinx_rtd_theme" templates_path = ['_templates'] exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] diff --git a/docs/requirements.txt b/docs/requirements.txt index 4ef7807ee..60ce5e9a1 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ Sphinx==7.1.1 myst-parser==2.0.0 sphinx_design==0.5.0 -sphinx-book-theme==1.0.1 +sphinx-rtd-theme==1.2.2 linkify-it-py==2.0.2 \ No newline at end of file From 7331b2afee7d02ce898a5cfb61c7e688ca42e9b0 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Fri, 4 Aug 2023 14:29:55 -0300 Subject: [PATCH 42/55] html theme added4 --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 60ce5e9a1..fbaaf3879 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -Sphinx==7.1.1 +Sphinx==6.2.1 myst-parser==2.0.0 sphinx_design==0.5.0 sphinx-rtd-theme==1.2.2 From 52e92752dd98fbce12218bbfe6926f07f45d9a8e Mon Sep 17 00:00:00 2001 From: Md Azam Date: Tue, 8 Aug 2023 08:41:45 -0300 Subject: [PATCH 43/55] table of contents --- docs/OVERVIEW.md | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/docs/OVERVIEW.md b/docs/OVERVIEW.md index 89a2cac01..887056cfe 100644 --- a/docs/OVERVIEW.md +++ b/docs/OVERVIEW.md @@ -1,26 +1,7 @@ -# STIX-SHIFTER - -**Table of Contents** - -- [Introduction](#introduction) - - [What is STIX?](#what-is-stix) - - [What is STIX-SHIFTER?](#what-is-stix-shifter) - - [What is STIX Patterning? What are STIX Observations?](#what-is-stix-patterning-what-are-stix-observations) - - [This sounds like Sigma, I already have that](#this-sounds-like-sigma-i-already-have-that) - - [What is a STIX-SHIFTER connector?](#what-is-a-stix-shifter-connector) - - [Why would I want to use this?](#why-would-i-want-to-use-this) -- [Available Connectors](CONNECTORS.md) -- [How to use](#How-to-use) - - [Translate](#translate) - - [Transmit](#transmit) - - [Execute](#execute) - - [Debug](#Debug) -- [Limitations](#limitations) -- [Glossary](#glossary) -- [Architecture Context](#architecture-context) -- [Contributing](#contributing) -- [Guide for creating new connectors](adapter-guide/develop-stix-adapter.md) -- [Licensing](#licensing) + +```{contents} Table of Contents +:depth: 3 +``` ## Introduction From d82660158cd033e1ba326e94f4cbda3a1e2a0ed2 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Tue, 8 Aug 2023 10:42:20 -0300 Subject: [PATCH 44/55] index update --- docs/index.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/docs/index.md b/docs/index.md index a3c7dbdf4..a69e8c762 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,11 +1,7 @@ -# Introduction to STIX-Shifter +# STIX-Shifter STIX-shifter is an open source python library allowing software to connect to products that house data repositories by using STIX Patterning, and return results as STIX Observations. -This library takes in STIX 2 Patterns as input, and "finds" data that matches the patterns inside various products that house repositories of cybersecurity data. Examples of such products include SIEM systems, endpoint management systems, threat intelligence platforms, orchestration platforms, network control points, data lakes, and more. - -In addition to "finding" the data by using these patterns, STIX-Shifter also _transforms the output_ into STIX 2 Observations. Why would we do that you ask? To put it simply - so that all of the security data, regardless of the source, mostly looks and behaves the same. - ::::{grid} 1 2 2 3 :gutter: 1 1 1 2 @@ -22,7 +18,6 @@ Follow the developer guide to learn about developing a new STIX-Shifter connecto ::: ```{toctree} -:hidden: :maxdepth: 2 README.md ``` From 9543f38cbd997d251a7104e15fe782568046164d Mon Sep 17 00:00:00 2001 From: Md Azam Date: Tue, 8 Aug 2023 14:50:51 -0300 Subject: [PATCH 45/55] updates --- docs/README.md | 5 ++- docs/adapter-guide/best_practices.md | 2 +- docs/adapter-guide/custom_mappings.md | 2 +- .../develop-configuration-json.md | 10 +++--- .../adapter-guide/develop-mapping-keywords.md | 20 +++++------ docs/adapter-guide/develop-stix-adapter.md | 33 ++++++++----------- docs/index.md | 2 ++ 7 files changed, 34 insertions(+), 40 deletions(-) diff --git a/docs/README.md b/docs/README.md index 99876f3d3..0dff66a33 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,10 +9,9 @@ This library takes in STIX 2 Patterns as input, and "finds" data that matches th In addition to "finding" the data by using these patterns, STIX-Shifter also _transforms the output_ into STIX 2 Observations. Why would we do that you ask? To put it simply - so that all of the security data, regardless of the source, mostly looks and behaves the same. -## Project Overview and CLI Commands - -For general information about STIX, this project, and the command line utilities, see the [STIX-shifter Overview](OVERVIEW.md) +***Project Documenation*** +For general information about STIX, this project, and the command line utilities, see the [STIX-shifter Documenation](https://kestrel.readthedocs.io/) # Installation diff --git a/docs/adapter-guide/best_practices.md b/docs/adapter-guide/best_practices.md index 1a4be4284..a4f644ba0 100644 --- a/docs/adapter-guide/best_practices.md +++ b/docs/adapter-guide/best_practices.md @@ -1,4 +1,4 @@ -# Best practices when developing a new connector +## Best practices An assessment of the data source APIs should be made before beginning implementation of a connector. The APIs should return a good coverage of [cyber observable](https://docs.oasis-open.org/cti/stix/v2.0/stix-v2.0-part4-cyber-observable-objects.html) data that fits within the standard STIX `observed-data` objects. If most of the data returned is getting mapped to custom STIX objects or properties, it may be an indication that the data source is not a good fit for a connector. The APIs should also allow for robust filtering of the data, or support a query language; this is essential for executing federated searches against multiple connectors using STIX patterning. diff --git a/docs/adapter-guide/custom_mappings.md b/docs/adapter-guide/custom_mappings.md index 9590819f9..b09db76ba 100644 --- a/docs/adapter-guide/custom_mappings.md +++ b/docs/adapter-guide/custom_mappings.md @@ -1,4 +1,4 @@ -# Use of custom mappings +## Use of custom mappings Follow the below steps, if a user or threat hunter wants to use custom mapping after installing stix-shifter libraries: diff --git a/docs/adapter-guide/develop-configuration-json.md b/docs/adapter-guide/develop-configuration-json.md index 486fd6b5a..7ab5fe61e 100644 --- a/docs/adapter-guide/develop-configuration-json.md +++ b/docs/adapter-guide/develop-configuration-json.md @@ -1,10 +1,10 @@ -# Configuration Parameters +## Configuration Parameters A json file needs to be created that contains configuration parameters for each module. The configuration json file is required in order to validate the module specific parameters for a successful translation and transmission call. Please follow this naming convention when you create the file: `config.json` A second json file is required to translate the parameters defined in `lang_en.json` for the UI. This file is necessary in order to help the UI framework show the parameters in human readable format. -## File Location +### File Location Create a directory named `configuration` in your module folder. The json files mentioned above needs to be created inside `configuration`. Make sure json files saved in the following location for your new module- @@ -12,9 +12,9 @@ Create a directory named `configuration` in your module folder. The json files m /stix_shifter_modules//configuration ``` -## JSON File Description +### JSON File Description -### config json file +#### config json file Two top level json objects needs to be preset in the file: `connection` and `configuration`. The child attributes of the `connection` object should be the parameters required for making API calls which can be used by multiple users and role levels. The `configuration` object should contain the parameters that are required for API authentication for individual users and roles. @@ -104,7 +104,7 @@ Configuration object needs to have `auth` child object. `auth` object should con Both connection and configuration object may contain more or different parameters than that are defined in the example above based on the individual module. -### lang json file +#### lang json file The `lang_en.json` file has the similar format like `config.json`. It has different child attributes to translate the files for UI framework. diff --git a/docs/adapter-guide/develop-mapping-keywords.md b/docs/adapter-guide/develop-mapping-keywords.md index 764db51c4..59e7c59f5 100644 --- a/docs/adapter-guide/develop-mapping-keywords.md +++ b/docs/adapter-guide/develop-mapping-keywords.md @@ -1,4 +1,4 @@ -# To STIX mapping Keywords +## Mapping Keywords There are keywords which need to be specified in the `to-stix` mappings in order to perform specific operations on the datasource fields. There are two types of keywords: 1. Required @@ -6,7 +6,7 @@ There are keywords which need to be specified in the `to-stix` mappings in order The below table contains the keywords and their usages: -## Required Keywords +### Required Keywords @@ -41,7 +41,7 @@ The below table contains the keywords and their usages:
-## Optional Keywords +### Optional Keywords @@ -70,9 +70,9 @@ The below table contains the keywords and their usages:
-## Examples of Optional keywords: +### Examples of Optional keywords: -## unwrap +#### unwrap **Mapping:** @@ -140,7 +140,7 @@ This STIX bundle contains two ipv4-addr objects which are created based on `unwr
-## group +#### group **Mapping:** @@ -236,7 +236,7 @@ This STIX bundle contains two ipv4-addr objects which are created based on `unwr
-## group_ref +#### group_ref **Mapping:** @@ -337,7 +337,7 @@ List of nested dictionary in datasource results are referenced in `scanned_refs`
-## value +#### value **Mapping:** @@ -397,7 +397,7 @@ List of nested dictionary in datasource results are referenced in `scanned_refs`
-## references +#### references **Mapping:** @@ -477,7 +477,7 @@ Source `ipv4-addr` object number is referenced in `network-traffic` object:
-## transformer +#### transformer **Mapping:** ``` diff --git a/docs/adapter-guide/develop-stix-adapter.md b/docs/adapter-guide/develop-stix-adapter.md index 7110b5a91..cef437d7a 100644 --- a/docs/adapter-guide/develop-stix-adapter.md +++ b/docs/adapter-guide/develop-stix-adapter.md @@ -1,19 +1,12 @@ -# Developing a new STIX-shifter connector +## Developing a new connector -- [Introduction](../README.md) -- [Overview of the project](../OVERVIEW.md) -- [Scenario](#scenario) -- [Prerequisites](#prerequisites) -- [Best Practices for developing a connector](best_practices.md) -- [Steps](#steps) +### Scenario -## Scenario - -### Participants +***Participants*** This scenario involves a software developer (_Developer A_) and an end user (_User A_). _Developer A_ wants to implement a new connector for the STIX-shifter project that can support a particular security product (_Product A_). _User A_ is another developer that uses the STIX-shifter library. -### Problem to solve +***Problem to solve*** _User A_ performs security monitoring with _Product A_ and several other security products. The other products already have existing STIX-shifter connectors. @@ -24,7 +17,7 @@ _User A_ would like to: By implementing a new connector, _Developer A_ allows _Product A_ to fit into the workflow. -## Prerequisites +### Prerequisites - Your development environment must use Python 3.8 or greater. - You must have access to the target data source. In the sample scenario, you must have access to Product A data source. @@ -33,11 +26,11 @@ By implementing a new connector, _Developer A_ allows _Product A_ to fit into th - Observable objects. See [STIX™ Version 2.0. Part 4: Cyber Observable Objects](http://docs.oasis-open.org/cti/stix/v2.0/stix-v2.0-part4-cyber-observable-objects.html) - Stix patterning. See [STIX™ Version 2.0. Part 5: STIX Patterning](https://docs.oasis-open.org/cti/stix/v2.0/stix-v2.0-part5-stix-patterning.html) -## Best practices +### Best practices Familiarize yourself with some [best practices](best_practices.md) before beginning a new connector. -## Steps +### Steps To develop a STIX-shifter connector for a data source: @@ -49,7 +42,7 @@ To develop a STIX-shifter connector for a data source: 1. [Create the module entry points](#create-module-entry-points). 1. Create a pull request to merge your changes in the `opencybersecurityalliance/stix-shifter` repository. -### Create a module folder +#### Create a module folder Connector modules are stored under the `stix_shifter_modules` directory. To help you get started with creating a new connector, two module templates are available. If your data source executes queries synchronously (there is no API call to check the status of the query), make a copy of the `synchronous_template` folder in the `stix_shifter_modules` directory. If your data source executes queries asynchronously, make a copy of the `async_template` folder. The instructions that follow use the async template as an example. @@ -68,7 +61,7 @@ Each module contains the following directories and files: entry_point.py: Initializes classes and paths used by the connector. -### Create the module entry points +#### Create the module entry points The `EntryPoint` class acts as a gateway to the various methods used by the translation and transmission classes. In most instances, it's fine to use the `setup_transmission_simple` and `setup_translation_simple(dialect_default='default')` methods. In cases where multiple dialects are used by the connector, the `dialect_default` argument is the dialect you wish to use as the default when the entire collection isn't passed in. See [Create a Translation module](develop-translation-module.md) to learn about dialects. @@ -99,7 +92,7 @@ If the data source is synchronous, you must include `set_async(False)` in the co ``` -### Testing a new connector using the proxy host +#### Testing a new connector using the proxy host Work on a new stix-shifter connector occurs after the project has been forked and cloned into a local development environment. Stix-shifter contains a **proxy** connector that facilitates a remote instance of the project calling out to a local instance. While in development, a new connector's working branch can be tested in any project using the stix-shifter library without first merging into the master branch on Github. A host is run on the local instance from the CLI. When a `proxy` data source is passed to the remote instance of stix-shifter, the real connection attributes (data source type, host, and port contained in the options) are passed onto the local instance of stix-shifter running the proxy host. The host will then use the new connector and return results back to the remote stix-shifter instance. @@ -115,7 +108,7 @@ As an example: python main.py host '{"type": "identity","id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff","name": "Bundle","identity_class": "events"}' "192.168.122.83:5000" ``` -#### Calling the proxy host +##### Calling the proxy host Each of the translate and transmit CLI commands outlined in the stix-shifter overview can be used to call the proxy host. @@ -125,7 +118,7 @@ As an example: python main.py transmit proxy '{"options": {"proxy_host": "127.0.0.1", "proxy_port": 5000, "destination": {"connection": {"options": {"result_limit": 10000, "time_range": 5, "timeout": 30}, "host": "", "port": , "type": "qradar"}, "configuration": {"auth": { "SEC": ""} } } }}' '{}' ping ``` -### Packaging individual connectors +#### Packaging individual connectors Stix-shifter can be broken into several python whl packages by using the `setup.py` script found in the root of the project. This packaging script can be called from the CLI: @@ -162,7 +155,7 @@ stix_shifter_modules => entry_point ``` -### Building images of the connectors +#### Building images of the connectors You can build the docker image your developed connector locally and publish it to your desired repository. In order to do that, follow the below steps- diff --git a/docs/index.md b/docs/index.md index a69e8c762..955a6e940 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,6 +2,8 @@ STIX-shifter is an open source python library allowing software to connect to products that house data repositories by using STIX Patterning, and return results as STIX Observations. +[STIX-Shifter github repo](https://github.com/opencybersecurityalliance/stix-shifter) is the official portal of everything STIX-Shifter beyond this documentation: source, connectors, tutorial, community entrances, and more. + ::::{grid} 1 2 2 3 :gutter: 1 1 1 2 From 52443171cb318dedc70c5e19da06b4864f10449a Mon Sep 17 00:00:00 2001 From: Md Azam Date: Tue, 8 Aug 2023 15:09:26 -0300 Subject: [PATCH 46/55] updates2 --- docs/README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/README.md b/docs/README.md index 0dff66a33..5732effdc 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,7 +1,7 @@ [![example workflow](https://github.com/opencybersecurityalliance/stix-shifter/actions/workflows/main.yml/badge.svg)](https://github.com/opencybersecurityalliance/stix-shifter/actions) [![codecov](https://codecov.io/gh/opencybersecurityalliance/stix-shifter/branch/develop/graph/badge.svg?token=gQvl14peRj)](https://codecov.io/gh/opencybersecurityalliance/stix-shifter) -# Introduction to STIX-Shifter +# Introduction STIX-shifter is an open source python library allowing software to connect to products that house data repositories by using STIX Patterning, and return results as STIX Observations. @@ -11,9 +11,9 @@ In addition to "finding" the data by using these patterns, STIX-Shifter also _tr ***Project Documenation*** -For general information about STIX, this project, and the command line utilities, see the [STIX-shifter Documenation](https://kestrel.readthedocs.io/) +For general information about STIX, this project, and the command line utilities, see the [STIX-shifter Documenation](https://stix-shifter.readthedocs.io/) -# Installation +## Installation The recommended method for installing stix-shifter is via pip. Two prerequisite packages needs to be installed inlcuding the package of stix-shifter connector module to complete a stix-shifter connector installation. Run the below commands to install all the packages: @@ -24,15 +24,15 @@ The recommended method for installing stix-shifter is via pip. Two prerequisite 3. Desired stix-shifter connector module package: `pip install stix-shifter-modules- ` Example: `pip install stix-shifter-modules-qradar` -## Dependencies +### Dependencies STIX-shifter requries Python 3.8 or greater. See the [requirements file](../stix_shifter/requirements.txt) for library dependencies. -# Usage +## Usage STIX-Shifter can use used the following ways: -## 1. As a command line utility +### As a command line utility The STIX-Shifter comes with a bundled script which you can use to translate STIX Pattern to a native datasource query. It can also be used to translate a JSON data source query result to a STIX bundle of observable objects. You can also send query to a datasource by using a transmission option. @@ -53,7 +53,7 @@ In order to build `stix-shifter` packages from source follow the below prerequis 3. Run setup: `python3 setup.py install` -## 2. Running from the source +### Running from the source You may also use the `python3 main.py` script. All the options are the same as the command line utility described above. @@ -71,7 +71,7 @@ In order to run `python3 main.py` from the source follow the below prerequisite **Note:** setup.py only installs dependencies when INSTALL_REQUIREMENTS_ONLY=1 directive is used. This option is similar to `python3 generate_requirements.py && pip install -r requirements.txt` -## 3. As a library +### As a library You can also use this library to integrate STIX Shifter into your own tools. You can translate a STIX Pattern: From 917c50c9bc4c741ed061c175bb11e6bb41b0182f Mon Sep 17 00:00:00 2001 From: Md Azam Date: Wed, 9 Aug 2023 15:48:37 -0300 Subject: [PATCH 47/55] tables --- docs/adapter-guide/develop-mapping-keywords.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/adapter-guide/develop-mapping-keywords.md b/docs/adapter-guide/develop-mapping-keywords.md index 59e7c59f5..d9993f740 100644 --- a/docs/adapter-guide/develop-mapping-keywords.md +++ b/docs/adapter-guide/develop-mapping-keywords.md @@ -8,7 +8,7 @@ The below table contains the keywords and their usages: ### Required Keywords - +
@@ -43,7 +43,7 @@ The below table contains the keywords and their usages: ### Optional Keywords -
Keywords Type Descriptions Usage Example
+
From dc699908f41f9621d8f4594840a5484043b10100 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Wed, 9 Aug 2023 16:03:47 -0300 Subject: [PATCH 48/55] tables2 --- .../adapter-guide/develop-mapping-keywords.md | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/docs/adapter-guide/develop-mapping-keywords.md b/docs/adapter-guide/develop-mapping-keywords.md index d9993f740..c3d3a3e11 100644 --- a/docs/adapter-guide/develop-mapping-keywords.md +++ b/docs/adapter-guide/develop-mapping-keywords.md @@ -5,6 +5,52 @@ There are keywords which need to be specified in the `to-stix` mappings in order 2. Optional The below table contains the keywords and their usages: +
Keywords Type Descriptions Usage
+ + + + + + + + + + + + + + + + + + + + + + + + + +
KeywordsTypeDescriptionsUsageExample
keyStringThe STIX object and properties whose path is defined in dot notation."key": "stix-object.stix_object_property.sub_property" + ```json + { + "sha256hash": { + "key": "file.hashes.SHA-256", + "object": "fl" + } + } + ``` +
objectStringThe name specified in the object is used to add properties of same the object. "object": "src_ip" + ```json + { + "sourceip": { + "key": "ipv4-addr.value", + "object": "src_ip" + } + } + ``` +
+ ### Required Keywords From 1aff23d3b29ec2f79774e983282a5cd076002144 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Wed, 9 Aug 2023 16:10:18 -0300 Subject: [PATCH 49/55] tables3 --- .../adapter-guide/develop-mapping-keywords.md | 48 +------------------ 1 file changed, 1 insertion(+), 47 deletions(-) diff --git a/docs/adapter-guide/develop-mapping-keywords.md b/docs/adapter-guide/develop-mapping-keywords.md index c3d3a3e11..331ef7c19 100644 --- a/docs/adapter-guide/develop-mapping-keywords.md +++ b/docs/adapter-guide/develop-mapping-keywords.md @@ -5,58 +5,12 @@ There are keywords which need to be specified in the `to-stix` mappings in order 2. Optional The below table contains the keywords and their usages: - - - - - - - - - - - - - - - - - - - - - - - - - - -
KeywordsTypeDescriptionsUsageExample
keyStringThe STIX object and properties whose path is defined in dot notation."key": "stix-object.stix_object_property.sub_property" - ```json - { - "sha256hash": { - "key": "file.hashes.SHA-256", - "object": "fl" - } - } - ``` -
objectStringThe name specified in the object is used to add properties of same the object. "object": "src_ip" - ```json - { - "sourceip": { - "key": "ipv4-addr.value", - "object": "src_ip" - } - } - ``` -
- ### Required Keywords - +
Keywords Type Descriptions Usage Example Keywords Type Descriptions Usage Example
key String The STIX object and properties whose path is defined in dot notation. "key": "stix-object.stix_object_property.sub_property" From 75b36b11aa328fd9df7ab81e51f29db618ed243a Mon Sep 17 00:00:00 2001 From: Md Azam Date: Wed, 9 Aug 2023 16:39:57 -0300 Subject: [PATCH 50/55] tables4 --- docs/adapter-guide/develop-mapping-keywords.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/adapter-guide/develop-mapping-keywords.md b/docs/adapter-guide/develop-mapping-keywords.md index 331ef7c19..89b417f8d 100644 --- a/docs/adapter-guide/develop-mapping-keywords.md +++ b/docs/adapter-guide/develop-mapping-keywords.md @@ -8,7 +8,7 @@ The below table contains the keywords and their usages: ### Required Keywords - +
From 60d787aed86174a5b52cf0295d453f2de31efb02 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Wed, 9 Aug 2023 16:51:51 -0300 Subject: [PATCH 51/55] tables5 --- docs/adapter-guide/develop-mapping-keywords.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/adapter-guide/develop-mapping-keywords.md b/docs/adapter-guide/develop-mapping-keywords.md index 89b417f8d..78d92ab08 100644 --- a/docs/adapter-guide/develop-mapping-keywords.md +++ b/docs/adapter-guide/develop-mapping-keywords.md @@ -8,8 +8,18 @@ The below table contains the keywords and their usages: ### Required Keywords + +
Keywords Type Descriptions Usage Example
- + From 0bacb8cb07ec147ac1782f0928df29c1be40a020 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Wed, 9 Aug 2023 17:07:22 -0300 Subject: [PATCH 52/55] tables6 --- docs/adapter-guide/develop-mapping-keywords.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/adapter-guide/develop-mapping-keywords.md b/docs/adapter-guide/develop-mapping-keywords.md index 78d92ab08..89148d1fd 100644 --- a/docs/adapter-guide/develop-mapping-keywords.md +++ b/docs/adapter-guide/develop-mapping-keywords.md @@ -22,7 +22,7 @@ th, td { - + - +
Keywords Type Descriptions Usage Example
key String The STIX object and properties whose path is defined in dot notation. "key": "stix-object.stix_object_property.sub_property"
Keywords Type Descriptions Usage Example
key String The STIX object and properties whose path is defined in dot notation. "key": "stix-object.stix_object_property.sub_property" key String The STIX object and properties whose path is defined in dot notation. "key": "stix-object.stix-object-property.sub-property" ```json @@ -75,7 +75,7 @@ th, td {
group_ref Boolean This keyword needs to be used when there is a nested list of dictionaries and each dictionary item creates an object. This keyword groups together references in a list and sets where the object is mapped. "group_ref": true
ds_key String This keyword is used when datasource results are formatted to modify some field names. The value assigned to the keyword determines the mapping of a STIX object. This keyword is only used in the aws_athena and aws_cloud_watch_logs modules to resolve nested dictionary mappings. This keyword has been deprecated since nested dictionary mappings are now handled by the JSON to STIX translation utility. "ds_key": "resource_instancedetails_networkinterfaces_0_networkinterfaceid" ds_key String This keyword is used when datasource results are formatted to modify some field names. The value assigned to the keyword determines the mapping of a STIX object. This keyword is only used in the aws_athena and aws_cloud_watch_logs modules to resolve nested dictionary mappings. This keyword has been deprecated since nested dictionary mappings are now handled by the JSON to STIX translation utility. "ds_key": "resource_instancedetails"
From 9b62bb9d39d0fef5b3d50a2b13bff833e34e8d81 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Wed, 9 Aug 2023 17:13:07 -0300 Subject: [PATCH 53/55] tables7 --- docs/adapter-guide/develop-mapping-keywords.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/adapter-guide/develop-mapping-keywords.md b/docs/adapter-guide/develop-mapping-keywords.md index 89148d1fd..9205473ce 100644 --- a/docs/adapter-guide/develop-mapping-keywords.md +++ b/docs/adapter-guide/develop-mapping-keywords.md @@ -53,9 +53,9 @@ th, td { ### Optional Keywords - +
- + From 84a36c08ca203b250f6c16adb5944a061e2290ef Mon Sep 17 00:00:00 2001 From: Md Azam Date: Tue, 15 Aug 2023 11:34:10 -0300 Subject: [PATCH 54/55] html theme --- docs/conf.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 6e9cf0b79..c1fe3fc3e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -48,5 +48,16 @@ # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -html_theme = 'alabaster' html_static_path = ['_static'] + +html_theme_options = { + "home_page_in_toc": True, + "github_url": "https://github.com/opencybersecurityalliance/stix-shifter", + "repository_url": "https://github.com/opencybersecurityalliance/stix-shifter", + "repository_branch": "master", + "path_to_docs": "docs", + "use_repository_button": True, + "use_edit_page_button": True, + "use_issues_button": True, + "announcement": "6.1.0 is now out! See the Changelog for details", +} From 86b3fd6ee51772cbbe9b0a77d8e5be290bcbf2c7 Mon Sep 17 00:00:00 2001 From: Md Azam Date: Tue, 15 Aug 2023 11:53:04 -0300 Subject: [PATCH 55/55] cleanup --- docs/bakindex.rst | 27 --------------------------- docs/conf.py | 19 +++---------------- docs/index.md | 4 ++++ 3 files changed, 7 insertions(+), 43 deletions(-) delete mode 100644 docs/bakindex.rst diff --git a/docs/bakindex.rst b/docs/bakindex.rst deleted file mode 100644 index 2733c952f..000000000 --- a/docs/bakindex.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. STIX-Shifter documentation master file, created by - sphinx-quickstart on Fri Jul 28 11:31:22 2023. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to STIX-Shifter's documentation! -======================================== - -*Introduction to STIX-Shifter* - -STIX-shifter is an open source python library allowing software to connect to products that house data repositories by using STIX Patterning, and return results as STIX Observations. - -This library takes in STIX 2 Patterns as input, and "finds" data that matches the patterns inside various products that house repositories of cybersecurity data. Examples of such products include SIEM systems, endpoint management systems, threat intelligence platforms, orchestration platforms, network control points, data lakes, and more. - -In addition to "finding" the data by using these patterns, STIX-Shifter also transforms the output into STIX 2 Observations. Why would we do that you ask? To put it simply - so that all of the security data, regardless of the source, - -.. toctree:: - :maxdepth: 2 - - README.md - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/conf.py b/docs/conf.py index c1fe3fc3e..204ee5de0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -9,7 +9,7 @@ project = 'STIX-Shifter' copyright = '2023, Md Azam, Danny Elliott' author = 'Md Azam, Danny Elliott' -release = '6.0.3' +release = '6.1.0' # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration @@ -39,8 +39,6 @@ "attrs_block", ] -html_theme = "sphinx_rtd_theme" - templates_path = ['_templates'] exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] @@ -48,16 +46,5 @@ # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -html_static_path = ['_static'] - -html_theme_options = { - "home_page_in_toc": True, - "github_url": "https://github.com/opencybersecurityalliance/stix-shifter", - "repository_url": "https://github.com/opencybersecurityalliance/stix-shifter", - "repository_branch": "master", - "path_to_docs": "docs", - "use_repository_button": True, - "use_edit_page_button": True, - "use_issues_button": True, - "announcement": "6.1.0 is now out! See the Changelog for details", -} +html_theme = "sphinx_rtd_theme" +html_static_path = ['_static'] \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 955a6e940..8372c10dd 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,6 +4,8 @@ STIX-shifter is an open source python library allowing software to connect to pr [STIX-Shifter github repo](https://github.com/opencybersecurityalliance/stix-shifter) is the official portal of everything STIX-Shifter beyond this documentation: source, connectors, tutorial, community entrances, and more. +:::: + ::::{grid} 1 2 2 3 :gutter: 1 1 1 2 @@ -19,6 +21,8 @@ There are more than 30 connectors. For the list of connectors, see the [Availabl Follow the developer guide to learn about developing a new STIX-Shifter connector, see [Connector Developer Guide](adapter-guide/develop-stix-adapter.md). ::: +:::: + ```{toctree} :maxdepth: 2 README.md
Keywords Type Descriptions Usage Keywords Type Descriptions Usage
references String/List(string) Specifies named objects to reference in another object. "references": "src_ip"
"references": ["dst_mac"]