From 97d9ee135e637507dff1e876d31e36d7375bb82e Mon Sep 17 00:00:00 2001 From: Ryan Deivert Date: Thu, 31 Oct 2019 11:13:39 -0700 Subject: [PATCH 1/8] updating gsuite streamalert apps --- streamalert/apps/_apps/gsuite.py | 33 +++++++++++++++++-- .../apps/test_apps/test_app_base.py | 7 ++-- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/streamalert/apps/_apps/gsuite.py b/streamalert/apps/_apps/gsuite.py index 985b03da7..43ae55cf2 100644 --- a/streamalert/apps/_apps/gsuite.py +++ b/streamalert/apps/_apps/gsuite.py @@ -229,6 +229,15 @@ def _sleep_seconds(self): return 0 +@StreamAlertApp +class GSuiteAccessTransparencyReports(GSuiteReportsApp): + """G Suite Access Transparency Report app integration""" + + @classmethod + def _type(cls): + return 'access_transparency' + + @StreamAlertApp class GSuiteAdminReports(GSuiteReportsApp): """G Suite Admin Activity Report app integration""" @@ -257,12 +266,21 @@ def _type(cls): @StreamAlertApp -class GSuiteGroupsReports(GSuiteReportsApp): - """G Suite Groups Activity Report app integration""" +class GSuiteGroupReports(GSuiteReportsApp): + """G Suite Group Activity Report app integration""" + + @classmethod + def _type(cls): + return 'group' + + +@StreamAlertApp +class GSuiteGroupEnterpriseReports(GSuiteReportsApp): + """G Suite Groups Enterprise Activity Report app integration""" @classmethod def _type(cls): - return 'groups' + return 'groups_enterprise' @StreamAlertApp @@ -317,3 +335,12 @@ class GSuiteTokenReports(GSuiteReportsApp): @classmethod def _type(cls): return 'token' + + +@StreamAlertApp +class GSuiteUserAccountsReports(GSuiteReportsApp): + """G Suite User Accounts Activity Report app integration""" + + @classmethod + def _type(cls): + return 'user_accounts' diff --git a/tests/unit/streamalert/apps/test_apps/test_app_base.py b/tests/unit/streamalert/apps/test_apps/test_app_base.py index a7234eac3..e731894e1 100644 --- a/tests/unit/streamalert/apps/test_apps/test_app_base.py +++ b/tests/unit/streamalert/apps/test_apps/test_app_base.py @@ -49,16 +49,19 @@ def test_get_all_apps(self): 'box_admin_events', 'duo_admin', 'duo_auth', + 'gsuite_access_transparency', 'gsuite_admin', 'gsuite_calendar', 'gsuite_drive', 'gsuite_gplus', - 'gsuite_groups', + 'gsuite_group', + 'gsuite_groups_enterprise', 'gsuite_login', 'gsuite_mobile', 'gsuite_rules', 'gsuite_saml', 'gsuite_token', + 'gsuite_user_accounts', 'intercom_admin_activity_logs', 'onelogin_events', 'salesforce_console', @@ -71,7 +74,7 @@ def test_get_all_apps(self): 'aliyun_actiontrail' } - assert_count_equal(expected_apps, StreamAlertApp.get_all_apps()) + assert_equal(expected_apps, set(StreamAlertApp.get_all_apps())) @patch('streamalert.apps.app_base.Batcher', Mock()) def test_get_app(self): From 766180f8dff61e495e34e05a5c3097837d91c672 Mon Sep 17 00:00:00 2001 From: Ryan Deivert Date: Thu, 31 Oct 2019 11:27:37 -0700 Subject: [PATCH 2/8] removing deprecated saml app --- streamalert/apps/_apps/gsuite.py | 9 --------- tests/unit/streamalert/apps/test_apps/test_app_base.py | 1 - 2 files changed, 10 deletions(-) diff --git a/streamalert/apps/_apps/gsuite.py b/streamalert/apps/_apps/gsuite.py index 43ae55cf2..bca3c3551 100644 --- a/streamalert/apps/_apps/gsuite.py +++ b/streamalert/apps/_apps/gsuite.py @@ -319,15 +319,6 @@ def _type(cls): return 'rules' -@StreamAlertApp -class GSuiteSAMLReports(GSuiteReportsApp): - """G Suite SAML Activity Report app integration""" - - @classmethod - def _type(cls): - return 'saml' - - @StreamAlertApp class GSuiteTokenReports(GSuiteReportsApp): """G Suite Token Activity Report app integration""" diff --git a/tests/unit/streamalert/apps/test_apps/test_app_base.py b/tests/unit/streamalert/apps/test_apps/test_app_base.py index e731894e1..bf083f1c6 100644 --- a/tests/unit/streamalert/apps/test_apps/test_app_base.py +++ b/tests/unit/streamalert/apps/test_apps/test_app_base.py @@ -59,7 +59,6 @@ def test_get_all_apps(self): 'gsuite_login', 'gsuite_mobile', 'gsuite_rules', - 'gsuite_saml', 'gsuite_token', 'gsuite_user_accounts', 'intercom_admin_activity_logs', From 695206f0fe9bc11602e063ea55641205095b52af Mon Sep 17 00:00:00 2001 From: Ryan Deivert Date: Thu, 31 Oct 2019 11:33:06 -0700 Subject: [PATCH 3/8] rm unused import --- tests/unit/streamalert/apps/test_apps/test_app_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/streamalert/apps/test_apps/test_app_base.py b/tests/unit/streamalert/apps/test_apps/test_app_base.py index bf083f1c6..9e6693b2d 100644 --- a/tests/unit/streamalert/apps/test_apps/test_app_base.py +++ b/tests/unit/streamalert/apps/test_apps/test_app_base.py @@ -19,7 +19,6 @@ from mock import call, Mock, patch from moto import mock_ssm from nose.tools import ( - assert_count_equal, assert_equal, assert_false, assert_is_instance, From bf7a315418a6b4f3d70253a2a4580d3af189b98d Mon Sep 17 00:00:00 2001 From: Ryan Deivert Date: Thu, 31 Oct 2019 12:57:34 -0700 Subject: [PATCH 4/8] updating requirements for new google api client, etc --- requirements-top-level.txt | 2 +- requirements.txt | 153 ++++++++++++++++++------------------- 2 files changed, 75 insertions(+), 80 deletions(-) diff --git a/requirements-top-level.txt b/requirements-top-level.txt index 169928c7a..c5b445991 100644 --- a/requirements-top-level.txt +++ b/requirements-top-level.txt @@ -9,7 +9,7 @@ boxsdk[jwt] cbapi coverage coveralls -google-api-python-client==1.6.4 +google-api-python-client jmespath jsonlines mock diff --git a/requirements.txt b/requirements.txt index 1152900dd..35176df4e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,124 +1,119 @@ -alabaster==0.7.12 -aliyun-python-sdk-actiontrail==2.0.0 aliyun-python-sdk-core==2.13.5 -aliyun-python-sdk-core-v3==2.13.3 -asn1crypto==0.24.0 -astroid==2.2.5 -attrdict==2.0.1 -attrs==19.1.0 -autoflake==1.3 +aliyun-python-sdk-actiontrail==2.0.0 +autoflake==1.3.1 autopep8==1.4.4 -aws-sam-translator==1.11.0 +backoff==1.8.1 +bandit==1.6.2 +boto3==1.10.6 +boxsdk==2.6.1 +cbapi==1.5.4 +coverage==4.5.4 +coveralls==1.8.2 +google-api-python-client==1.7.11 +jmespath==0.9.4 +jsonlines==1.2.0 +mock==3.0.5 +moto==1.3.13 +netaddr==0.7.19 +nose==1.3.7 +nose-timer==0.7.5 +pathlib2==2.3.5 +policyuniverse==1.3.2.1 +pyfakefs==3.6.1 +pylint==2.4.3 +requests==2.22.0 +Sphinx==2.2.1 +sphinx-rtd-theme==0.4.3 +yapf==0.28.0 +## The following requirements were added by pip freeze: +alabaster==0.7.12 +aliyun-python-sdk-core-v3==2.13.10 +astroid==2.3.2 +atomicwrites==1.3.0 +attrdict==2.0.1 +attrs==19.3.0 +aws-sam-translator==1.15.1 aws-xray-sdk==2.4.2 Babel==2.7.0 -backoff==1.8.0 -bandit==1.6.1 boto==2.49.0 -boto3==1.9.176 -botocore==1.12.176 -boxsdk[jwt]==2.6.0 +botocore==1.13.6 cachetools==3.1.1 -cbapi==1.5.1 -certifi==2019.6.16 -cffi==1.12.3 -cfn-lint==0.21.6 -chainmap==1.0.2 +certifi==2019.9.11 +cffi==1.13.1 +cfn-lint==0.24.6 chardet==3.0.4 -configparser==3.5.0 -cookies==2.2.1 -coverage==4.5.3 -coveralls==1.8.1 -cryptography==2.7 +cryptography==2.8 DateTime==4.3 -decorator==4.4.0 -docker==4.0.2 -docker-pycreds==0.3.0 +decorator==4.4.1 +docker==4.1.0 docopt==0.6.2 -docutils==0.14 -ecdsa==0.13.2 -enum34==1.1.6 -funcsigs==1.0.2 -gitdb2==2.0.5 -GitPython==2.1.11 -google-api-python-client==1.6.4 -httplib2==0.13.0 +docutils==0.15.2 +ecdsa==0.13.3 +future==0.18.2 +gitdb2==2.0.6 +GitPython==3.0.4 +google-auth==1.6.3 +google-auth-httplib2==0.0.3 +httplib2==0.14.0 idna==2.8 imagesize==1.1.0 -ipaddress==1.0.22 -isort==4.3.20 -Jinja2==2.10.1 -jmespath==0.9.4 +importlib-metadata==0.23 +isort==4.3.21 +Jinja2==2.10.3 jsondiff==1.1.2 -jsonlines==1.2.0 jsonpatch==1.24 jsonpickle==1.2 jsonpointer==2.0 -jsonschema==2.6.0 -lazy-object-proxy==1.4.1 +jsonschema==3.1.1 +lazy-object-proxy==1.4.3 MarkupSafe==1.1.1 mccabe==0.6.1 -mock==3.0.5 -moto==1.3.13 -netaddr==0.7.19 -nose==1.3.7 -nose-timer==0.7.5 -oauth2client==4.1.3 -packaging==19.0 -pathlib2==2.3.4 -pbr==5.3.1 -pika==1.0.1 -policyuniverse==1.3.2.0 -prompt-toolkit==2.0.9 -protobuf==3.8.0 -pyaml==18.11.0 -pyasn1==0.4.5 -pyasn1-modules==0.2.5 +more-itertools==7.2.0 +packaging==19.2 +pbr==5.4.3 +pika==1.1.0 +pluggy==0.13.0 +prompt-toolkit==2.0.10 +protobuf==3.10.0 +py==1.8.0 +pyasn1==0.4.7 +pyasn1-modules==0.2.7 pycodestyle==2.5.0 pycparser==2.19 -pycryptodome==3.8.2 -pyfakefs==3.5.8 pyflakes==2.1.1 Pygments==2.4.2 PyJWT==1.7.1 -pylint==2.3.1 -pyOpenSSL==18.0.0 -pyparsing==2.4.0 -pyrsistent==0.15.4 +pyparsing==2.4.2 +pyrsistent==0.15.5 +pytest==5.0.0 python-dateutil==2.8.0 python-jose==3.0.1 -pytz==2019.1 -PyYAML==5.1.1 -requests==2.22.0 +pytz==2019.3 +PyYAML==5.1.2 requests-toolbelt==0.9.1 responses==0.10.6 rsa==4.0 s3transfer==0.2.1 -scandir==1.9.0 -singledispatch==3.4.0.3 six==1.12.0 smmap2==2.0.5 -snowballstemmer==1.2.1 +snowballstemmer==2.0.0 solrq==1.1.1 -Sphinx==2.1.2 -sphinx-rtd-theme==0.4.3 sphinxcontrib-applehelp==1.0.1 sphinxcontrib-devhelp==1.0.1 sphinxcontrib-htmlhelp==1.0.2 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==1.0.2 sphinxcontrib-serializinghtml==1.1.3 -sphinxcontrib-websupport==1.1.0 sshpubkeys==3.1.0 -stevedore==1.30.1 +stevedore==1.31.0 typed-ast==1.4.0 -typing==3.6.6 uritemplate==3.0.0 -urllib3==1.25.3 -validators==0.13.0 +urllib3==1.25.6 +validators==0.14.0 wcwidth==0.1.7 websocket-client==0.56.0 -Werkzeug==0.15.4 +Werkzeug==0.16.0 wrapt==1.11.2 xmltodict==0.12.0 -yapf==0.27.0 +zipp==0.6.0 zope.interface==4.6.0 From 0d98b006a118eaa4942d54ce3090fefa066950d9 Mon Sep 17 00:00:00 2001 From: Ryan Deivert Date: Thu, 31 Oct 2019 12:57:55 -0700 Subject: [PATCH 5/8] updating packaging for new pinned versions --- streamalert_cli/manage_lambda/package.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/streamalert_cli/manage_lambda/package.py b/streamalert_cli/manage_lambda/package.py index 0af662768..8ed370506 100644 --- a/streamalert_cli/manage_lambda/package.py +++ b/streamalert_cli/manage_lambda/package.py @@ -43,16 +43,15 @@ class LambdaPackage: # Define a package dict to support pinning versions across all subclasses PACKAGE_LIBS = { 'aliyun-python-sdk-actiontrail': 'aliyun-python-sdk-actiontrail==2.0.0', - 'backoff': 'backoff==1.8.0', - 'boto3': 'boto3==1.9.208', - 'boxsdk[jwt]': 'boxsdk[jwt]==2.6.0', - 'cbapi': 'cbapi==1.5.1', - 'google-api-python-client': 'google-api-python-client==1.6.4', + 'backoff': 'backoff==1.8.1', + 'boto3': 'boto3==1.10.6', + 'boxsdk[jwt]': 'boxsdk[jwt]==2.6.1', + 'cbapi': 'cbapi==1.5.4', + 'google-api-python-client': 'google-api-python-client==1.7.11', 'jmespath': 'jmespath==0.9.4', 'jsonlines': 'jsonlines==1.2.0', 'netaddr': 'netaddr==0.7.19', - 'oauth2client': 'oauth2client==4.1.3', - 'policyuniverse': 'policyuniverse==1.3.2.0', + 'policyuniverse': 'policyuniverse==1.3.2.1', 'requests': 'requests==2.22.0', } @@ -258,7 +257,6 @@ class AppPackage(LambdaPackage): 'backoff', 'boxsdk[jwt]', 'google-api-python-client', - 'oauth2client', 'requests' } From 64510c685ed75b98baecb2ff225003a3cf8bf17e Mon Sep 17 00:00:00 2001 From: Ryan Deivert Date: Thu, 31 Oct 2019 12:58:12 -0700 Subject: [PATCH 6/8] updating gsuite apps for new python sdk library --- streamalert/apps/_apps/gsuite.py | 30 +++++++++++++------ .../streamalert/apps/test_apps/test_gsuite.py | 29 +++++++++--------- 2 files changed, 36 insertions(+), 23 deletions(-) diff --git a/streamalert/apps/_apps/gsuite.py b/streamalert/apps/_apps/gsuite.py index bca3c3551..0ec60b035 100644 --- a/streamalert/apps/_apps/gsuite.py +++ b/streamalert/apps/_apps/gsuite.py @@ -19,8 +19,10 @@ import socket import ssl -import apiclient -from oauth2client import client, service_account +import googleapiclient.discovery +import googleapiclient.errors +from google.oauth2 import service_account +from google.auth.exceptions import GoogleAuthError from . import AppIntegration, StreamAlertApp, get_logger @@ -36,8 +38,12 @@ class GSuiteReportsApp(AppIntegration): """G Suite Reports base app integration. This is subclassed for various endpoints""" _SCOPES = ['https://www.googleapis.com/auth/admin.reports.audit.readonly'] # A tuple of uncaught exceptions that the googleapiclient can raise - _GOOGLE_API_EXCEPTIONS = (apiclient.errors.Error, client.Error, socket.timeout, - ssl.SSLError) + _GOOGLE_API_EXCEPTIONS = ( + googleapiclient.errors.Error, + GoogleAuthError, + socket.timeout, + ssl.SSLError, + ) # The maximum number of unique event ids to store that occur on the most # recent timestamp. These are used to de-duplicate events in the next poll. @@ -73,12 +79,14 @@ def _load_credentials(cls, keydata): JSON file Returns: - oauth2client.service_account.ServiceAccountCredentials: Instance of + google.oauth2.service_account.ServiceAccountCredentials: Instance of service account credentials for this discovery service """ try: - creds = service_account.ServiceAccountCredentials.from_json_keyfile_dict( - keydata, scopes=cls._SCOPES) + creds = service_account.Credentials.from_service_account_info( + keydata, + scopes=cls._SCOPES, + ) except (ValueError, KeyError): # This has the potential to raise errors. See: https://tinyurl.com/y8q5e9rm LOGGER.exception('[%s] Could not generate credentials from keyfile', cls.type()) @@ -103,9 +111,13 @@ def _create_service(self): if not creds: return False - delegation = creds.create_delegated(self._config.auth['delegation_email']) + delegation = creds.with_subject(self._config.auth['delegation_email']) try: - resource = apiclient.discovery.build('admin', 'reports_v1', credentials=delegation) + resource = googleapiclient.discovery.build( + 'admin', + 'reports_v1', + credentials=delegation + ) except self._GOOGLE_API_EXCEPTIONS: LOGGER.exception('[%s] Failed to build discovery service', self) return False diff --git a/tests/unit/streamalert/apps/test_apps/test_gsuite.py b/tests/unit/streamalert/apps/test_apps/test_gsuite.py index 8040c41bf..298863d77 100644 --- a/tests/unit/streamalert/apps/test_apps/test_gsuite.py +++ b/tests/unit/streamalert/apps/test_apps/test_gsuite.py @@ -19,8 +19,9 @@ import ssl from datetime import datetime, timedelta -import apiclient -import oauth2client +import googleapiclient +from google.auth import exceptions +from google.oauth2 import service_account from mock import Mock, mock_open, patch from moto import mock_ssm from nose.tools import assert_equal, assert_false, assert_count_equal, assert_true, raises @@ -57,7 +58,7 @@ def test_required_auth_info(self): assert_count_equal(list(self._app.required_auth_info().keys()), {'delegation_email', 'keyfile'}) - @patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_dict', + @patch('google.oauth2.service_account.Credentials.from_service_account_info', Mock(return_value=True)) def test_keyfile_validator(self): """GSuiteReportsApp - Keyfile Validation, Success""" @@ -68,7 +69,7 @@ def test_keyfile_validator(self): loaded_keydata = validation_function('fakepath') assert_equal(loaded_keydata, data) - @patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_dict') + @patch('google.oauth2.service_account.Credentials.from_service_account_info') def test_keyfile_validator_failure(self, cred_mock): """GSuiteReportsApp - Keyfile Validation, Failure""" validation_function = self._app.required_auth_info()['keyfile']['format'] @@ -78,7 +79,7 @@ def test_keyfile_validator_failure(self, cred_mock): assert_false(validation_function('fakepath')) cred_mock.assert_called() - @patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_dict') + @patch('google.oauth2.service_account.Credentials.from_service_account_info') def test_keyfile_validator_bad_json(self, cred_mock): """GSuiteReportsApp - Keyfile Validation, Bad JSON""" validation_function = self._app.required_auth_info()['keyfile']['format'] @@ -87,20 +88,20 @@ def test_keyfile_validator_bad_json(self, cred_mock): assert_false(validation_function('fakepath')) cred_mock.assert_not_called() - @patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_dict', + @patch('google.oauth2.service_account.Credentials.from_service_account_info', Mock(return_value=True)) def test_load_credentials(self): """GSuiteReportsApp - Load Credentials, Success""" assert_true(self._app._load_credentials('fakedata')) - @patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_dict') + @patch('google.oauth2.service_account.Credentials.from_service_account_info') def test_load_credentials_bad(self, cred_mock): """GSuiteReportsApp - Load Credentials, ValueError""" cred_mock.side_effect = ValueError('Bad things happened') assert_false(self._app._load_credentials('fakedata')) @patch('streamalert.apps._apps.gsuite.GSuiteReportsApp._load_credentials', Mock()) - @patch('streamalert.apps._apps.gsuite.apiclient.discovery.build') + @patch('streamalert.apps._apps.gsuite.googleapiclient.discovery.build') def test_create_service(self, build_mock): """GSuiteReportsApp - Create Service, Success""" build_mock.return_value.activities.return_value = True @@ -121,16 +122,16 @@ def test_create_service_fail_creds(self): @patch('streamalert.apps._apps.gsuite.GSuiteReportsApp._load_credentials', Mock()) @patch('logging.Logger.exception') - @patch('streamalert.apps._apps.gsuite.apiclient.discovery.build') + @patch('streamalert.apps._apps.gsuite.googleapiclient.discovery.build') def test_create_service_api_error(self, build_mock, log_mock): """GSuiteReportsApp - Create Service, Google API Error""" - build_mock.side_effect = apiclient.errors.Error('This is bad') + build_mock.side_effect = googleapiclient.errors.Error('This is bad') assert_false(self._app._create_service()) log_mock.assert_called_with('[%s] Failed to build discovery service', self._app) @patch('streamalert.apps._apps.gsuite.GSuiteReportsApp._load_credentials', Mock()) @patch('logging.Logger.exception') - @patch('streamalert.apps._apps.gsuite.apiclient.discovery.build') + @patch('streamalert.apps._apps.gsuite.googleapiclient.discovery.build') def test_create_service_ssl_error(self, build_mock, log_mock): """GSuiteReportsApp - Create Service, SSL Handshake Error""" build_mock.side_effect = ssl.SSLError('_ssl.c:574: The handshake operation timed out') @@ -139,7 +140,7 @@ def test_create_service_ssl_error(self, build_mock, log_mock): @patch('streamalert.apps._apps.gsuite.GSuiteReportsApp._load_credentials', Mock()) @patch('logging.Logger.exception') - @patch('streamalert.apps._apps.gsuite.apiclient.discovery.build') + @patch('streamalert.apps._apps.gsuite.googleapiclient.discovery.build') def test_create_service_socket_error(self, build_mock, log_mock): """GSuiteReportsApp - Create Service, Socket Timeout""" build_mock.side_effect = socket.timeout('timeout: timed out') @@ -166,7 +167,7 @@ def test_gather_logs(self): def test_gather_logs_http_error(self, log_mock): """GSuiteReportsApp - Gather Logs, Google API HTTP Error""" with patch.object(self._app, '_activities_service') as service_mock: - error = apiclient.errors.HttpError('response', 'bad'.encode()) + error = googleapiclient.errors.HttpError('response', 'bad'.encode()) service_mock.list.return_value.execute.side_effect = error assert_false(self._app._gather_logs()) log_mock.assert_called_with('[%s] Failed to execute activities listing', self._app) @@ -177,7 +178,7 @@ def test_gather_logs_http_error(self, log_mock): def test_gather_logs_token_error(self, log_mock): """GSuiteReportsApp - Gather Logs, Google API Token Error""" with patch.object(self._app, '_activities_service') as service_mock: - error = oauth2client.client.HttpAccessTokenRefreshError('bad', status=502) + error = exceptions.RefreshError('bad') service_mock.list.return_value.execute.side_effect = error assert_false(self._app._gather_logs()) log_mock.assert_called_with('[%s] Failed to execute activities listing', self._app) From e679420da52e87ce447926c19f189a1947f376f7 Mon Sep 17 00:00:00 2001 From: Ryan Deivert Date: Thu, 31 Oct 2019 14:33:02 -0700 Subject: [PATCH 7/8] misc pylint fixup --- streamalert/classifier/parsers.py | 4 ++-- streamalert/rules_engine/threat_intel.py | 4 +--- streamalert_cli/manage_lambda/package.py | 5 +++-- tests/unit/streamalert/apps/test_apps/test_gsuite.py | 1 - tests/unit/streamalert/classifier/test_parsers_json.py | 5 ++--- tests/unit/streamalert/rules_engine/test_threat_intel.py | 8 +++----- 6 files changed, 11 insertions(+), 16 deletions(-) diff --git a/streamalert/classifier/parsers.py b/streamalert/classifier/parsers.py index 6935253bd..47cf31825 100644 --- a/streamalert/classifier/parsers.py +++ b/streamalert/classifier/parsers.py @@ -732,8 +732,8 @@ def _parse_row(self, row, schema): if parsed_data and parsed_data[0][1] is True: parsed_payload[key] = parsed_data[0][0] continue - else: - return False # break the loop if the nested data was invalid + + return False # break the loop if the nested data was invalid # extract the keys from the row via the index parsed_payload[key] = row[index] diff --git a/streamalert/rules_engine/threat_intel.py b/streamalert/rules_engine/threat_intel.py index 498e76373..845b6a593 100644 --- a/streamalert/rules_engine/threat_intel.py +++ b/streamalert/rules_engine/threat_intel.py @@ -207,9 +207,7 @@ def _run_query(query_values, results): } ) - results.extend([ - result for result in self._deserialize(response['Responses'].get(self._table)) - ]) + results.extend(self._deserialize(response['Responses'].get(self._table))) # Log this as an error for now so it can be picked up in logs if response['UnprocessedKeys']: diff --git a/streamalert_cli/manage_lambda/package.py b/streamalert_cli/manage_lambda/package.py index 8ed370506..72b379745 100644 --- a/streamalert_cli/manage_lambda/package.py +++ b/streamalert_cli/manage_lambda/package.py @@ -15,6 +15,7 @@ """ import os import shutil +import sys import tempfile import zipfile @@ -74,12 +75,12 @@ def create(self): if not self._resolve_libraries(temp_package_path): LOGGER.exception('Failed to install necessary libraries') - exit(1) + sys.exit(1) # Extract any precompiled libs for this package if self.precompiled_libs and not self._extract_precompiled_libs(temp_package_path): LOGGER.exception('Failed to extract precompiled libraries') - exit(1) + sys.exit(1) # Zip up files result = shutil.make_archive( diff --git a/tests/unit/streamalert/apps/test_apps/test_gsuite.py b/tests/unit/streamalert/apps/test_apps/test_gsuite.py index 298863d77..6b2a3135e 100644 --- a/tests/unit/streamalert/apps/test_apps/test_gsuite.py +++ b/tests/unit/streamalert/apps/test_apps/test_gsuite.py @@ -21,7 +21,6 @@ import googleapiclient from google.auth import exceptions -from google.oauth2 import service_account from mock import Mock, mock_open, patch from moto import mock_ssm from nose.tools import assert_equal, assert_false, assert_count_equal, assert_true, raises diff --git a/tests/unit/streamalert/classifier/test_parsers_json.py b/tests/unit/streamalert/classifier/test_parsers_json.py index f9b35a2f6..b4e49d696 100644 --- a/tests/unit/streamalert/classifier/test_parsers_json.py +++ b/tests/unit/streamalert/classifier/test_parsers_json.py @@ -770,9 +770,8 @@ def test_cloudtrail(self): parser = JSONParser(options) assert_equal(parser.parse(record_data), True) - expected_result = [ - rec for rec in data['Records'] - ] + expected_result = data['Records'] + assert_equal(parser.parsed_records, expected_result) def test_cloudwatch(self): diff --git a/tests/unit/streamalert/rules_engine/test_threat_intel.py b/tests/unit/streamalert/rules_engine/test_threat_intel.py index aa34cb322..f2c2501ad 100644 --- a/tests/unit/streamalert/rules_engine/test_threat_intel.py +++ b/tests/unit/streamalert/rules_engine/test_threat_intel.py @@ -241,14 +241,12 @@ def test_process_ioc_values_parameter_error(self, log_mock): def test_segment(self): """ThreatIntel - Segment""" - values = [i for i in range(120)] - expected_result = [ - {i for i in range(100)}, - {i for i in range(100, 120)} + set(range(100)), + set(range(100, 120)) ] - result = list(ThreatIntel._segment(values)) + result = list(ThreatIntel._segment(list(range(120)))) assert_equal(result, expected_result) def test_query_client_error(self): From 4dd002b68f9d08d82c9b97980e054787e3e1b886 Mon Sep 17 00:00:00 2001 From: Ryan Deivert Date: Thu, 31 Oct 2019 15:03:15 -0700 Subject: [PATCH 8/8] pinning pylint version because annoyed --- .pylintrc | 2 +- requirements-top-level.txt | 2 +- requirements.txt | 6 +++--- streamalert/alert_processor/outputs/aws.py | 2 +- streamalert/shared/alert_table.py | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.pylintrc b/.pylintrc index 08807f55f..5fd054f13 100644 --- a/.pylintrc +++ b/.pylintrc @@ -271,7 +271,7 @@ contextmanager-decorators=contextlib.contextmanager # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular # expressions are accepted. -generated-members= +generated-members=.*boto3\.resource.*|.*Table.* # Tells whether missing members accessed in mixin class should be ignored. A # mixin class is detected if its name ends with "mixin" (case insensitive). diff --git a/requirements-top-level.txt b/requirements-top-level.txt index c5b445991..8082d8970 100644 --- a/requirements-top-level.txt +++ b/requirements-top-level.txt @@ -20,7 +20,7 @@ nose-timer pathlib2 policyuniverse pyfakefs -pylint +pylint==2.3.1 requests Sphinx sphinx-rtd-theme diff --git a/requirements.txt b/requirements.txt index 35176df4e..90824c2ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ autoflake==1.3.1 autopep8==1.4.4 backoff==1.8.1 bandit==1.6.2 -boto3==1.10.6 +boto3==1.10.7 boxsdk==2.6.1 cbapi==1.5.4 coverage==4.5.4 @@ -20,7 +20,7 @@ nose-timer==0.7.5 pathlib2==2.3.5 policyuniverse==1.3.2.1 pyfakefs==3.6.1 -pylint==2.4.3 +pylint==2.3.1 requests==2.22.0 Sphinx==2.2.1 sphinx-rtd-theme==0.4.3 @@ -36,7 +36,7 @@ aws-sam-translator==1.15.1 aws-xray-sdk==2.4.2 Babel==2.7.0 boto==2.49.0 -botocore==1.13.6 +botocore==1.13.7 cachetools==3.1.1 certifi==2019.9.11 cffi==1.13.1 diff --git a/streamalert/alert_processor/outputs/aws.py b/streamalert/alert_processor/outputs/aws.py index be6c5c166..05ea73a44 100644 --- a/streamalert/alert_processor/outputs/aws.py +++ b/streamalert/alert_processor/outputs/aws.py @@ -428,7 +428,7 @@ def _dispatch(self, alert, descriptor): """ queue_name = self.config[self.__service__][descriptor] sqs = boto3.resource('sqs', region_name=self.region) - queue = sqs.get_queue_by_name(QueueName=queue_name) + queue = sqs.get_queue_by_name(QueueName=queue_name) # pylint: disable=no-member publication = compose_alert(alert, self, descriptor) diff --git a/streamalert/shared/alert_table.py b/streamalert/shared/alert_table.py index 5b12bd3c2..4ef3dac6e 100644 --- a/streamalert/shared/alert_table.py +++ b/streamalert/shared/alert_table.py @@ -131,7 +131,7 @@ def get_alert_record(self, rule_name, alert_id): 'KeyConditionExpression': Key('RuleName').eq(rule_name) & Key('AlertID').eq(alert_id) } items = list(self._paginate(self._table.query, kwargs)) - return items[0] if items else None + return items[0] if items else {} # ---------- Add/Delete/Update Operations ----------