From 3c725945ee55763ce87e8b0347b9d9ba8b97cc4a Mon Sep 17 00:00:00 2001 From: Jiyong Jang Date: Sat, 29 May 2021 13:46:22 +0000 Subject: [PATCH 01/10] set zip_safe false for native namespace package build --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index 044e726b2..15411d64b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,3 +9,6 @@ description-file = README.md # need to generate separate wheels for each Python version that you # support. universal=1 + +[options] +zip_safe = False From 3d282c2ec66bfa973eebb0d9eb0b31414a7ff58d Mon Sep 17 00:00:00 2001 From: Jiyong Jang Date: Sat, 29 May 2021 13:48:41 +0000 Subject: [PATCH 02/10] fix and change the default build mode for shifter_modules build --- setup.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 9f0c49c11..c57040adc 100644 --- a/setup.py +++ b/setup.py @@ -40,13 +40,20 @@ def fill_connectors(projects, modules_path): if not os.path.isfile(os.path.join(modules_path, module, SKIP_ME)): projects['stix_shifter_modules_' + module] = ['stix_shifter_modules/' + module] +def fill_connectors_mode3(projects, modules_path): + modules = [name for name in os.listdir(modules_path) + if (os.path.isdir(os.path.join(modules_path, name)) and (not name.startswith('__')))] + for module in modules: + if not os.path.isfile(os.path.join(modules_path, module, SKIP_ME)): + projects['stix_shifter_modules'].append('stix_shifter_modules/' + module) + # The mode determines how the stix-shifter is packaged # 1 = Include everything in 1 whl package # 3 - 3 whl packages respectively for stix-shifter, stix-shifter-utils and stix-shifter-modules # N - stix-shifter, stix-shifter-utils, and each connector is packaged separately # - package only the specified connector -mode = 'N' +mode = '3' if 'MODE' in os.environ: mode = os.environ['MODE'] @@ -67,8 +74,9 @@ def fill_connectors(projects, modules_path): projects = { "stix_shifter_utils": ["stix_shifter_utils"], "stix_shifter": ["stix_shifter"], - "stix_shifter_modules": ["stix_shifter_modules"], + "stix_shifter_modules": [], } + fill_connectors_mode3(projects, "stix_shifter_modules") elif mode == 'N': projects = { "stix_shifter_utils": ["stix_shifter_utils"], @@ -88,7 +96,7 @@ def fill_connectors(projects, modules_path): for project_name in projects.keys(): cleanup_file_list = [] - temp_dir = None + temp_dir_list = [] module_dir = None src_folders = projects[project_name] @@ -207,6 +215,7 @@ def fill_connectors(projects, modules_path): with open(os.path.join(conf_path, 'dialects.json'), 'w', encoding="utf-8") as f: f.write(json.dumps(dialects_full, indent=4, sort_keys=False)) temp_dir = tempfile.TemporaryDirectory() + temp_dir_list.append([temp_dir, module_dir]) shutil.move(configuration_path, temp_dir.name) os.rename(conf_path, configuration_path) cleanup_file_list.append(configuration_path) @@ -245,8 +254,9 @@ def fill_connectors(projects, modules_path): shutil.rmtree(cleanup_file) else: os.remove(cleanup_file) - if temp_dir is not None: - shutil.move(os.path.join(temp_dir.name, 'configuration'), module_dir) - temp_dir = None + for temp_dir, module_dir in temp_dir_list: + if temp_dir is not None: + shutil.move(os.path.join(temp_dir.name, 'configuration'), module_dir) + temp_dir.cleanup() print('---------------------------------') shutil.rmtree(TMP_MAPPING_DIR) From 24df168b5a280d0af9684c16b6e168b7f6542652 Mon Sep 17 00:00:00 2001 From: Arthur Muradyan Date: Fri, 21 Jan 2022 09:39:47 -0500 Subject: [PATCH 03/10] Removed setup mode 3, Added pip install in setup --- generate_requirements.py | 49 ++++++++++++++++++++----------------- setup.py | 53 +++++++++++++++++++++++----------------- 2 files changed, 57 insertions(+), 45 deletions(-) diff --git a/generate_requirements.py b/generate_requirements.py index 1a7242f54..88b367183 100644 --- a/generate_requirements.py +++ b/generate_requirements.py @@ -1,25 +1,30 @@ import os -src_folders = ["stix_shifter_utils", "stix_shifter", "stix_shifter_modules"] -install_requires = set() -requirements_files = [] -for src_folder in src_folders: - for r, d, f in os.walk(src_folder): - for file in f: - if 'requirements.txt'==file and not os.path.isfile(os.path.join(r, 'SKIP.ME')): - requirements_files.append(os.path.join(r, file)) -print('requirements_files: %s' % requirements_files) -for requirements_file in requirements_files: - with open(requirements_file) as f: - lines = f.readlines() - lines = [x.strip() for x in lines] - lines = list(filter(lambda s: len(s)>0, lines)) - install_requires.update(lines) -install_requires = list(install_requires) -install_requires.sort() -print('install_requires: %s' % install_requires) +def generate_requirements(): + src_folders = ["stix_shifter_utils", "stix_shifter", "stix_shifter_modules"] + install_requires = set() + requirements_files = [] + for src_folder in src_folders: + for r, d, f in os.walk(src_folder): + for file in f: + if 'requirements.txt'==file and not os.path.isfile(os.path.join(r, 'SKIP.ME')): + requirements_files.append(os.path.join(r, file)) + print('requirements_files: %s' % requirements_files) + for requirements_file in requirements_files: + with open(requirements_file) as f: + lines = f.readlines() + lines = [x.strip() for x in lines] + lines = list(filter(lambda s: len(s)>0, lines)) + install_requires.update(lines) + install_requires = list(install_requires) + install_requires.sort() + print('install_requires: %s' % install_requires) -with open('requirements.txt', 'w') as out_file: - for item in install_requires: - out_file.write(item) - out_file.write('\n') + with open('requirements.txt', 'w') as out_file: + for item in install_requires: + out_file.write(item) + out_file.write('\n') + + +if __name__ == "__main__": + generate_requirements() \ No newline at end of file diff --git a/setup.py b/setup.py index e4dda57ce..4c05d522b 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,32 @@ +import os +import subprocess +import sys + +if sys.version_info.major == 3 and sys.version_info.minor > 5: + # good + print(sys.version) +else: + print("Error: stix-shifter requires python version at least or greater than 3.6") + exit(1) + + +from generate_requirements import generate_requirements +generate_requirements() + +subprocess.check_call([sys.executable, "-m", "pip", "install", "-r", "requirements-dev.txt"]) + + + +if os.getenv('INSTALL_REQUIREMENTS_ONLY', None) == '1': + exit(0) + + from setuptools import find_packages # To use a consistent encoding from codecs import open -import sys import shutil -import subprocess import json import io -import os from jsonmerge import merge import tempfile import importlib @@ -40,20 +60,13 @@ def fill_connectors(projects, modules_path): if not os.path.isfile(os.path.join(modules_path, module, SKIP_ME)): projects['stix_shifter_modules_' + module] = ['stix_shifter_modules/' + module] -def fill_connectors_mode3(projects, modules_path): - modules = [name for name in os.listdir(modules_path) - if (os.path.isdir(os.path.join(modules_path, name)) and (not name.startswith('__')))] - for module in modules: - if not os.path.isfile(os.path.join(modules_path, module, SKIP_ME)): - projects['stix_shifter_modules'].append('stix_shifter_modules/' + module) - # The mode determines how the stix-shifter is packaged # 1 = Include everything in 1 whl package # 3 - 3 whl packages respectively for stix-shifter, stix-shifter-utils and stix-shifter-modules # N - stix-shifter, stix-shifter-utils, and each connector is packaged separately # - package only the specified connector -mode = '3' +mode = 'N' if 'MODE' in os.environ: mode = os.environ['MODE'] @@ -70,13 +83,6 @@ def fill_connectors_mode3(projects, modules_path): 'stix_shifter_modules' ] } -elif mode == '3': - projects = { - "stix_shifter_utils": ["stix_shifter_utils"], - "stix_shifter": ["stix_shifter"], - "stix_shifter_modules": [], - } - fill_connectors_mode3(projects, "stix_shifter_modules") elif mode == 'N': projects = { "stix_shifter_utils": ["stix_shifter_utils"], @@ -224,11 +230,12 @@ def fill_connectors_mode3(projects, modules_path): cleanup_file_list.append(configuration_path) # Inject util files - for util_src, util_dest in utils_include_list.items(): - util_dest = util_dest % module_dir - if not shutil.os.path.exists(util_dest): - shutil.copyfile(util_src, util_dest) - cleanup_file_list.append(util_dest) + if mode != "1": + for util_src, util_dest in utils_include_list.items(): + util_dest = util_dest % module_dir + if not shutil.os.path.exists(util_dest): + shutil.copyfile(util_src, util_dest) + cleanup_file_list.append(util_dest) for r, d, f in os.walk(module_dir): r_split = r.split(os.sep) From 121824ec80a40904a933211902b8a647a5878942 Mon Sep 17 00:00:00 2001 From: Arthur Muradyan Date: Fri, 21 Jan 2022 13:53:02 -0500 Subject: [PATCH 04/10] Added modules folder in the setup projects list --- .gitignore | 2 +- requirements.txt | 18 ++++++ setup.py | 12 ++-- stix_shifter_utils/utils/param_validator.py | 11 +++- test.py | 71 +++++++++++++++++++++ test3.py | 6 ++ test_sync.py | 64 +++++++++++++++++++ 7 files changed, 177 insertions(+), 7 deletions(-) create mode 100644 requirements.txt create mode 100644 test.py create mode 100644 test3.py create mode 100644 test_sync.py diff --git a/.gitignore b/.gitignore index adaf55ca9..02b52b7c9 100644 --- a/.gitignore +++ b/.gitignore @@ -61,7 +61,7 @@ coverage.xml .venv venv/ ENV/ -virtualenv/ +virtualenv*/ # mkdocs documentation /site diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..82bb96df2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,18 @@ +adal==1.2.7 +antlr4-python3-runtime==4.8 +boto3==1.20.33 +colorlog==6.6.0 +datadog_api_client==1.2.0 +flask==2.0.2 +flatten_json==0.1.13 +jsonmerge==1.8.0 +mysql-connector-python==8.0.25 +onelogin==2.0.1 +pyOpenSSL==21.0.0 +python-dateutil==2.8.2 +requests_toolbelt==0.9.1 +stix2-matcher==2.0.1 +stix2-patterns==1.3.2 +stix2-validator==1.1.2 +sumologic-sdk==0.1.13 +xmltodict==0.12.0 diff --git a/setup.py b/setup.py index 4c05d522b..c688c2589 100644 --- a/setup.py +++ b/setup.py @@ -81,12 +81,13 @@ def fill_connectors(projects, modules_path): 'stix_shifter_utils', 'stix_shifter', 'stix_shifter_modules' - ] + ] } elif mode == 'N': projects = { "stix_shifter_utils": ["stix_shifter_utils"], "stix_shifter": ["stix_shifter"], + "stix_shifter_modules": ["stix_shifter_modules"], } fill_connectors(projects, "stix_shifter_modules") else: @@ -233,9 +234,12 @@ def fill_connectors(projects, modules_path): if mode != "1": for util_src, util_dest in utils_include_list.items(): util_dest = util_dest % module_dir - if not shutil.os.path.exists(util_dest): - shutil.copyfile(util_src, util_dest) - cleanup_file_list.append(util_dest) + if shutil.os.path.exists(util_src) and not shutil.os.path.exists(util_dest): + try: + shutil.copyfile(util_src, util_dest) + cleanup_file_list.append(util_dest) + except Exception as e: + pass for r, d, f in os.walk(module_dir): r_split = r.split(os.sep) diff --git a/stix_shifter_utils/utils/param_validator.py b/stix_shifter_utils/utils/param_validator.py index 30ad324b3..2ebb3d810 100644 --- a/stix_shifter_utils/utils/param_validator.py +++ b/stix_shifter_utils/utils/param_validator.py @@ -8,9 +8,9 @@ def get_merged_config(module): ss_modules_path = importlib.import_module('stix_shifter_modules') if isinstance(ss_modules_path.__path__, list): - base_path = ss_modules_path.__path__[0] + base_path = choose_module_path(module, ss_modules_path.__path__) else: - base_path = ss_modules_path.__path__._path[0] + base_path = choose_module_path(module, ss_modules_path.__path__._path) module_config_path = path.join(base_path, module, 'configuration', 'config.json') base_config_path = path.join(base_path, 'config.json') with open(module_config_path) as mapping_file: @@ -21,6 +21,13 @@ def get_merged_config(module): module_configs = merge(base_configs, module_configs) return module_configs +def choose_module_path(module, path_list): + path = path_list[0] + module_name = 'stix_shifter_modules_' + module + for p in path_list: + if module_name in p: + return p + return path def modernize_objects(module, params): expected_configs = get_merged_config(module) diff --git a/test.py b/test.py new file mode 100644 index 000000000..6bc40198c --- /dev/null +++ b/test.py @@ -0,0 +1,71 @@ +import asyncio +import sys + +from stix_shifter.stix_transmission.stix_transmission import StixTransmission +from stix_shifter.stix_translation.stix_translation import StixTranslation +from downloads.datasource_config import configurations, connections + + +def print_log(*msg): + print(*msg) + pass + +async def main(count): + module_name = 'mysql' # stix_bundle mysql aws_athena aws_cloud_watch_logs + query_data = "[ipv4-addr:value = '127.0.0.1']" + # query_data = "[ipv4-addr:value LIKE '%'] START t'2020-05-03T08:43:10.003Z' STOP t'2021-06-29T10:43:10.003Z'" + # query_data = "[ipv4-addr:value LIKE '%']" + + connection = connections[module_name] + configuration = configurations[module_name] + + translation = StixTranslation() + transmission = StixTransmission(module_name, connection, configuration) + + # ping + ping = await transmission.ping_async() + print_log('Ping', ping) + + + result = translation.translate(module=module_name, translate_type='query', options=connection['options'], data_source=module_name, data=query_data) + print_log('Source query', result) + + if 'queries' in result: + + result = await transmission.query_async(result['queries'][0]) + if result.get('success'): + search_id = result.get('search_id') + print_log('SEARCH-ID', search_id) + + complete = False + while not complete: + result = await transmission.status_async(search_id) + if result.get('success') and result.get('status') == 'COMPLETED': + complete = True + if result.get('error'): + print('Status', result) + complete = True + + asyncio.sleep(2) + + result = await transmission.results_async(search_id, 0, 100) + print('Results', result) + + delete = await transmission.delete_async(search_id) + print('Delete', delete) + + print(count, "Result:", result) + + +async def async_call(n): + return await asyncio.wait([main(c) for c in range(n)]) + +if __name__ == "__main__": + import time + s = time.perf_counter() + + loop = asyncio.get_event_loop() + loop.run_until_complete(async_call(1)) + + elapsed = time.perf_counter() - s + print(f"{elapsed:0.2f} seconds.") \ No newline at end of file diff --git a/test3.py b/test3.py new file mode 100644 index 000000000..8abc9a4f9 --- /dev/null +++ b/test3.py @@ -0,0 +1,6 @@ +from stix_shifter.stix_translation import stix_translation + +translation = stix_translation.StixTranslation() +response = translation.translate('qradar', 'query', '{}', "[ipv4-addr:value = '127.0.0.1']", {}) + +print(response) \ No newline at end of file diff --git a/test_sync.py b/test_sync.py new file mode 100644 index 000000000..a3c2e6af5 --- /dev/null +++ b/test_sync.py @@ -0,0 +1,64 @@ +import sys + +from stix_shifter.stix_transmission.stix_transmission import StixTransmission +from stix_shifter.stix_translation.stix_translation import StixTranslation +from downloads.datasource_config import configurations, connections + + +def print_log(*msg): + print(*msg) + print('\n') + pass + +def main(count): + module_name = 'qradar' # stix_bundle, aws_athena, aws_cloud_watch_logs, qradar + query_data = "[ipv4-addr:value = '127.0.0.1']" + # query_data = "[ipv4-addr:value LIKE '%'] START t'2020-05-03T08:43:10.003Z' STOP t'2021-06-29T10:43:10.003Z'" + # query_data = "[ipv4-addr:value LIKE '%']" + + connection = connections[module_name] + configuration = configurations[module_name] + + translation = StixTranslation() + transmission = StixTransmission(module_name, connection, configuration) + + # ping + ping = transmission.ping() + print_log('Ping', ping) + + + result = translation.translate(module=module_name, translate_type='query', options=connection['options'], data_source=module_name, data=query_data) + print_log('Source query', result) + + if 'queries' in result: + + result = transmission.query(result['queries'][0]) + if result.get('success'): + search_id = result.get('search_id') + print_log('SEARCH-ID', search_id) + + complete = False + while not complete: + result = transmission.status(search_id) + print_log('Status', result) + if result.get('success') and result.get('status') == 'COMPLETED': + complete = True + time.sleep(2) + + result = transmission.results(search_id, 0, 100) + print_log('Results', result) + + delete = transmission.delete(search_id) + print_log('Delete', delete) + + print_log(count, "Result:", result) + + +if __name__ == "__main__": + import time + s = time.perf_counter() + + main(1) + + elapsed = time.perf_counter() - s + print(f"{elapsed:0.2f} seconds.") \ No newline at end of file From 3061195568dd995c7a966f4b4a060d22406c5774 Mon Sep 17 00:00:00 2001 From: Arthur Muradyan Date: Fri, 21 Jan 2022 13:56:08 -0500 Subject: [PATCH 05/10] Removed test files --- requirements.txt | 18 ------------ test.py | 71 ------------------------------------------------ test3.py | 6 ---- test_sync.py | 64 ------------------------------------------- 4 files changed, 159 deletions(-) delete mode 100644 requirements.txt delete mode 100644 test.py delete mode 100644 test3.py delete mode 100644 test_sync.py diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 82bb96df2..000000000 --- a/requirements.txt +++ /dev/null @@ -1,18 +0,0 @@ -adal==1.2.7 -antlr4-python3-runtime==4.8 -boto3==1.20.33 -colorlog==6.6.0 -datadog_api_client==1.2.0 -flask==2.0.2 -flatten_json==0.1.13 -jsonmerge==1.8.0 -mysql-connector-python==8.0.25 -onelogin==2.0.1 -pyOpenSSL==21.0.0 -python-dateutil==2.8.2 -requests_toolbelt==0.9.1 -stix2-matcher==2.0.1 -stix2-patterns==1.3.2 -stix2-validator==1.1.2 -sumologic-sdk==0.1.13 -xmltodict==0.12.0 diff --git a/test.py b/test.py deleted file mode 100644 index 6bc40198c..000000000 --- a/test.py +++ /dev/null @@ -1,71 +0,0 @@ -import asyncio -import sys - -from stix_shifter.stix_transmission.stix_transmission import StixTransmission -from stix_shifter.stix_translation.stix_translation import StixTranslation -from downloads.datasource_config import configurations, connections - - -def print_log(*msg): - print(*msg) - pass - -async def main(count): - module_name = 'mysql' # stix_bundle mysql aws_athena aws_cloud_watch_logs - query_data = "[ipv4-addr:value = '127.0.0.1']" - # query_data = "[ipv4-addr:value LIKE '%'] START t'2020-05-03T08:43:10.003Z' STOP t'2021-06-29T10:43:10.003Z'" - # query_data = "[ipv4-addr:value LIKE '%']" - - connection = connections[module_name] - configuration = configurations[module_name] - - translation = StixTranslation() - transmission = StixTransmission(module_name, connection, configuration) - - # ping - ping = await transmission.ping_async() - print_log('Ping', ping) - - - result = translation.translate(module=module_name, translate_type='query', options=connection['options'], data_source=module_name, data=query_data) - print_log('Source query', result) - - if 'queries' in result: - - result = await transmission.query_async(result['queries'][0]) - if result.get('success'): - search_id = result.get('search_id') - print_log('SEARCH-ID', search_id) - - complete = False - while not complete: - result = await transmission.status_async(search_id) - if result.get('success') and result.get('status') == 'COMPLETED': - complete = True - if result.get('error'): - print('Status', result) - complete = True - - asyncio.sleep(2) - - result = await transmission.results_async(search_id, 0, 100) - print('Results', result) - - delete = await transmission.delete_async(search_id) - print('Delete', delete) - - print(count, "Result:", result) - - -async def async_call(n): - return await asyncio.wait([main(c) for c in range(n)]) - -if __name__ == "__main__": - import time - s = time.perf_counter() - - loop = asyncio.get_event_loop() - loop.run_until_complete(async_call(1)) - - elapsed = time.perf_counter() - s - print(f"{elapsed:0.2f} seconds.") \ No newline at end of file diff --git a/test3.py b/test3.py deleted file mode 100644 index 8abc9a4f9..000000000 --- a/test3.py +++ /dev/null @@ -1,6 +0,0 @@ -from stix_shifter.stix_translation import stix_translation - -translation = stix_translation.StixTranslation() -response = translation.translate('qradar', 'query', '{}', "[ipv4-addr:value = '127.0.0.1']", {}) - -print(response) \ No newline at end of file diff --git a/test_sync.py b/test_sync.py deleted file mode 100644 index a3c2e6af5..000000000 --- a/test_sync.py +++ /dev/null @@ -1,64 +0,0 @@ -import sys - -from stix_shifter.stix_transmission.stix_transmission import StixTransmission -from stix_shifter.stix_translation.stix_translation import StixTranslation -from downloads.datasource_config import configurations, connections - - -def print_log(*msg): - print(*msg) - print('\n') - pass - -def main(count): - module_name = 'qradar' # stix_bundle, aws_athena, aws_cloud_watch_logs, qradar - query_data = "[ipv4-addr:value = '127.0.0.1']" - # query_data = "[ipv4-addr:value LIKE '%'] START t'2020-05-03T08:43:10.003Z' STOP t'2021-06-29T10:43:10.003Z'" - # query_data = "[ipv4-addr:value LIKE '%']" - - connection = connections[module_name] - configuration = configurations[module_name] - - translation = StixTranslation() - transmission = StixTransmission(module_name, connection, configuration) - - # ping - ping = transmission.ping() - print_log('Ping', ping) - - - result = translation.translate(module=module_name, translate_type='query', options=connection['options'], data_source=module_name, data=query_data) - print_log('Source query', result) - - if 'queries' in result: - - result = transmission.query(result['queries'][0]) - if result.get('success'): - search_id = result.get('search_id') - print_log('SEARCH-ID', search_id) - - complete = False - while not complete: - result = transmission.status(search_id) - print_log('Status', result) - if result.get('success') and result.get('status') == 'COMPLETED': - complete = True - time.sleep(2) - - result = transmission.results(search_id, 0, 100) - print_log('Results', result) - - delete = transmission.delete(search_id) - print_log('Delete', delete) - - print_log(count, "Result:", result) - - -if __name__ == "__main__": - import time - s = time.perf_counter() - - main(1) - - elapsed = time.perf_counter() - s - print(f"{elapsed:0.2f} seconds.") \ No newline at end of file From b0785905b6bddfbcf4e26048b2308ea3541eb1d1 Mon Sep 17 00:00:00 2001 From: Arthur Muradyan Date: Thu, 27 Jan 2022 12:24:59 -0500 Subject: [PATCH 06/10] Fixed local build pathes --- setup.py | 3 +-- .../stix_translation/stix_translation_error_mapper.py | 2 +- stix_shifter_utils/utils/module_discovery.py | 3 ++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index c688c2589..79bc93a53 100644 --- a/setup.py +++ b/setup.py @@ -86,8 +86,7 @@ def fill_connectors(projects, modules_path): elif mode == 'N': projects = { "stix_shifter_utils": ["stix_shifter_utils"], - "stix_shifter": ["stix_shifter"], - "stix_shifter_modules": ["stix_shifter_modules"], + "stix_shifter": ["stix_shifter"] } fill_connectors(projects, "stix_shifter_modules") else: diff --git a/stix_shifter_utils/stix_translation/stix_translation_error_mapper.py b/stix_shifter_utils/stix_translation/stix_translation_error_mapper.py index b17fd0d5b..ad7a38c1a 100644 --- a/stix_shifter_utils/stix_translation/stix_translation_error_mapper.py +++ b/stix_shifter_utils/stix_translation/stix_translation_error_mapper.py @@ -37,7 +37,7 @@ def set_error_code(data_dict, return_obj): if exception is not None: exception_type = type(exception).__name__ ErrorMapper.logger.error("received exception => {}: {}".format(exception_type, exception)) - ErrorMapper.logger.debug(logger.exception_to_string(exception)) + ErrorMapper.logger.info(logger.exception_to_string(exception)) if exception_type in error_mapping: error_code = error_mapping[exception_type][0] error_message = error_mapping[exception_type][1] diff --git a/stix_shifter_utils/utils/module_discovery.py b/stix_shifter_utils/utils/module_discovery.py index 074d8b8a9..c090df4e9 100644 --- a/stix_shifter_utils/utils/module_discovery.py +++ b/stix_shifter_utils/utils/module_discovery.py @@ -1,6 +1,7 @@ import os from importlib import import_module from pathlib import Path +from .param_validator import choose_module_path def process_dialects(cli_module, options): @@ -43,7 +44,7 @@ def dialect_list(module): if '__file__' in dir(modules) and modules.__file__ is not None: modules_path = Path(modules.__file__).parent else: - modules_path = modules.__path__._path[0] + modules_path = choose_module_path(module, modules.__path__._path) dialects_path = os.path.join(modules_path, f'{module}/stix_translation/json') ENDING = '_from_stix_map.json' dialects = [] From e2d4227f37966daebdccb1479dcb06f4c8ab7d3d Mon Sep 17 00:00:00 2001 From: Arthur Muradyan Date: Thu, 27 Jan 2022 12:31:28 -0500 Subject: [PATCH 07/10] Reverted info to debug in stix_translation_error_mapper.py --- .../stix_translation/stix_translation_error_mapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix_shifter_utils/stix_translation/stix_translation_error_mapper.py b/stix_shifter_utils/stix_translation/stix_translation_error_mapper.py index ad7a38c1a..b17fd0d5b 100644 --- a/stix_shifter_utils/stix_translation/stix_translation_error_mapper.py +++ b/stix_shifter_utils/stix_translation/stix_translation_error_mapper.py @@ -37,7 +37,7 @@ def set_error_code(data_dict, return_obj): if exception is not None: exception_type = type(exception).__name__ ErrorMapper.logger.error("received exception => {}: {}".format(exception_type, exception)) - ErrorMapper.logger.info(logger.exception_to_string(exception)) + ErrorMapper.logger.debug(logger.exception_to_string(exception)) if exception_type in error_mapping: error_code = error_mapping[exception_type][0] error_message = error_mapping[exception_type][1] From fb0a55ced1c217d173584bcbf7647d5a304b3216 Mon Sep 17 00:00:00 2001 From: Arthur Muradyan Date: Mon, 31 Jan 2022 14:16:28 -0500 Subject: [PATCH 08/10] Updated installation instructions --- README.md | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index cd2d783ac..10e77381b 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ The recommended method for installing the STIX-shifter is via pip. Two prerequis 1. Main stix-shifter package: `pip install stix-shifter` -2. stix-shifter-utility package: `pip install stix-shifter-utils` +2. Stix-shifter Utility package: `pip install stix-shifter-utils` 3. Desired stix-shifter connector module package: `pip install stix-shifter-modules- ` Example: `pip install stix-shifter-modules-qradar` @@ -40,7 +40,7 @@ The recommended method for installing the STIX-shifter is via pip. Two prerequis ## Usage -### As A Script +### As A Command Line Utility The STIX-Shifter comes with a bundled script which you can use to translate STIX Pattern to a native datasource query. It can also be used to translate a JSON data source query result to a STIX bundle of observable objects. You can also send query to a datasource by using a transmission option. @@ -56,10 +56,26 @@ $ stix-shifter translate qradar query {} "[ipv4-addr:value = '127.0.0.1']" {} **Note:** In order to build `stix-shifter` packages from source follow the below prerequisite steps: 1. Go to the stix-shifter parent directory - 2. Generate latest requirements.txt: `python3 generate_requirements.py` - 3. Install the dependencies in your python 3 environment: `pip install -r requirements.txt` - 4. Alternatively you can create a Python 3 virtual environemnt: - `virtualenv -p python3 virtualenv && source virtualenv/bin/activate && pip install -r requirements-dev.txt` + 2. Run setup: `python3 setup.py install` + 3. Alternatively you can create a Python 3 virtual environemnt: + `virtualenv -p python3 virtualenv && source virtualenv/bin/activate && python3 setup.py install` + + +### Running From the Source + +You may also use `python3 main.py` local script. All the options are the same as for the *"As A Command Line Utility"* usage above. + +Example: + +``` +python3 main.py translate qradar query {} "[ipv4-addr:value = '127.0.0.1']" {} +``` + +In order to run `python3 main.py` from the source follow the below prerequisite steps: + 1. Go to the stix-shifter parent directory + 2. Run setup to install dependancies: `python3 generate_requirements.py && pip install -r requirements.txt`, or alternatively `INSTALL_REQUIREMENTS_ONLY=1 python3 setup.py install`. + 3. Alternatively you can create a Python 3 virtual environemnt: + `virtualenv -p python3 virtualenv && source virtualenv/bin/activate && INSTALL_REQUIREMENTS_ONLY=1 python3 setup.py install` ### As A Library From 78d3b51bc9523dde4807824036397bbdbf78dfe2 Mon Sep 17 00:00:00 2001 From: Arthur Muradyan Date: Wed, 2 Feb 2022 12:53:28 -0500 Subject: [PATCH 09/10] Improved Readme for stix-shifter setup --- README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 10e77381b..a7c88b44c 100644 --- a/README.md +++ b/README.md @@ -54,11 +54,11 @@ Example: $ stix-shifter translate qradar query {} "[ipv4-addr:value = '127.0.0.1']" {} ``` -**Note:** In order to build `stix-shifter` packages from source follow the below prerequisite steps: +In order to build `stix-shifter` packages from source follow the below prerequisite steps: 1. Go to the stix-shifter parent directory - 2. Run setup: `python3 setup.py install` - 3. Alternatively you can create a Python 3 virtual environemnt: - `virtualenv -p python3 virtualenv && source virtualenv/bin/activate && python3 setup.py install` + 2. Optionally, you can create a Python 3 virtual environemnt: + `virtualenv -p python3 virtualenv && source virtualenv/bin/activate` + 3. Run setup: `python3 setup.py install` ### Running From the Source @@ -73,9 +73,11 @@ python3 main.py translate qradar query {} "[ipv4-addr:value = '127.0.0.1']" {} In order to run `python3 main.py` from the source follow the below prerequisite steps: 1. Go to the stix-shifter parent directory - 2. Run setup to install dependancies: `python3 generate_requirements.py && pip install -r requirements.txt`, or alternatively `INSTALL_REQUIREMENTS_ONLY=1 python3 setup.py install`. - 3. Alternatively you can create a Python 3 virtual environemnt: - `virtualenv -p python3 virtualenv && source virtualenv/bin/activate && INSTALL_REQUIREMENTS_ONLY=1 python3 setup.py install` + 2. Optionally, you can create a Python 3 virtual environemnt: + `virtualenv -p python3 virtualenv && source virtualenv/bin/activate` + 3. Run setup to install dependancies: `INSTALL_REQUIREMENTS_ONLY=1 python3 setup.py install`. + +**Note:** INSTALL_REQUIREMENTS_ONLY=1 is an directive to forse the setup to stop after it installs the pip dependances. This oprtion is similar to `python3 generate_requirements.py && pip install -r requirements.txt` ### As A Library From 663f0b71a2cec99054a722be54c80683204891ad Mon Sep 17 00:00:00 2001 From: Arthur Muradyan Date: Wed, 2 Feb 2022 16:03:36 -0500 Subject: [PATCH 10/10] Readme wording fixes --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a7c88b44c..68169e06f 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ In order to build `stix-shifter` packages from source follow the below prerequis ### Running From the Source -You may also use `python3 main.py` local script. All the options are the same as for the *"As A Command Line Utility"* usage above. +You may also use `python3 main.py` script. All the options are the same as "As a command line utility" usage above. Example: @@ -77,7 +77,7 @@ In order to run `python3 main.py` from the source follow the below prerequisite `virtualenv -p python3 virtualenv && source virtualenv/bin/activate` 3. Run setup to install dependancies: `INSTALL_REQUIREMENTS_ONLY=1 python3 setup.py install`. -**Note:** INSTALL_REQUIREMENTS_ONLY=1 is an directive to forse the setup to stop after it installs the pip dependances. This oprtion is similar to `python3 generate_requirements.py && pip install -r requirements.txt` +**Note:** setup.py only installs dependencies when INSTALL_REQUIREMENTS_ONLY=1 directive is used. This option is similar to `python3 generate_requirements.py && pip install -r requirements.txt` ### As A Library