[apps] update gsuite apps for changes to google-api-python-client

.pylintrc

@@ -271,7 +271,7 @@ contextmanager-decorators=contextlib.contextmanager
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
 # expressions are accepted.
-generated-members=
+generated-members=.*boto3\.resource.*|.*Table.*
 
 # Tells whether missing members accessed in mixin class should be ignored. A
 # mixin class is detected if its name ends with "mixin" (case insensitive).

requirements-top-level.txt

@@ -9,7 +9,7 @@ boxsdk[jwt]
 cbapi
 coverage
 coveralls
-google-api-python-client==1.6.4
+google-api-python-client
 jmespath
 jsonlines
 mock
@@ -20,7 +20,7 @@ nose-timer
 pathlib2
 policyuniverse
 pyfakefs
-pylint
+pylint==2.3.1
 requests
 Sphinx
 sphinx-rtd-theme

requirements.txt

@@ -1,124 +1,119 @@
-alabaster==0.7.12
-aliyun-python-sdk-actiontrail==2.0.0
 aliyun-python-sdk-core==2.13.5
-aliyun-python-sdk-core-v3==2.13.3
-asn1crypto==0.24.0
-astroid==2.2.5
-attrdict==2.0.1
-attrs==19.1.0
-autoflake==1.3
+aliyun-python-sdk-actiontrail==2.0.0
+autoflake==1.3.1
 autopep8==1.4.4
-aws-sam-translator==1.11.0
+backoff==1.8.1
+bandit==1.6.2
+boto3==1.10.7
+boxsdk==2.6.1
+cbapi==1.5.4
+coverage==4.5.4
+coveralls==1.8.2
+google-api-python-client==1.7.11
+jmespath==0.9.4
+jsonlines==1.2.0
+mock==3.0.5
+moto==1.3.13
+netaddr==0.7.19
+nose==1.3.7
+nose-timer==0.7.5
+pathlib2==2.3.5
+policyuniverse==1.3.2.1
+pyfakefs==3.6.1
+pylint==2.3.1
+requests==2.22.0
+Sphinx==2.2.1
+sphinx-rtd-theme==0.4.3
+yapf==0.28.0
+## The following requirements were added by pip freeze:
+alabaster==0.7.12
+aliyun-python-sdk-core-v3==2.13.10
+astroid==2.3.2
+atomicwrites==1.3.0
+attrdict==2.0.1
+attrs==19.3.0
+aws-sam-translator==1.15.1
 aws-xray-sdk==2.4.2
 Babel==2.7.0
-backoff==1.8.0
-bandit==1.6.1
 boto==2.49.0
-boto3==1.9.176
-botocore==1.12.176
-boxsdk[jwt]==2.6.0
+botocore==1.13.7
 cachetools==3.1.1
-cbapi==1.5.1
-certifi==2019.6.16
-cffi==1.12.3
-cfn-lint==0.21.6
-chainmap==1.0.2
+certifi==2019.9.11
+cffi==1.13.1
+cfn-lint==0.24.6
 chardet==3.0.4
-configparser==3.5.0
-cookies==2.2.1
-coverage==4.5.3
-coveralls==1.8.1
-cryptography==2.7
+cryptography==2.8
 DateTime==4.3
-decorator==4.4.0
-docker==4.0.2
-docker-pycreds==0.3.0
+decorator==4.4.1
+docker==4.1.0
 docopt==0.6.2
-docutils==0.14
-ecdsa==0.13.2
-enum34==1.1.6
-funcsigs==1.0.2
-gitdb2==2.0.5
-GitPython==2.1.11
-google-api-python-client==1.6.4
-httplib2==0.13.0
+docutils==0.15.2
+ecdsa==0.13.3
+future==0.18.2
+gitdb2==2.0.6
+GitPython==3.0.4
+google-auth==1.6.3
+google-auth-httplib2==0.0.3
+httplib2==0.14.0
 idna==2.8
 imagesize==1.1.0
-ipaddress==1.0.22
-isort==4.3.20
-Jinja2==2.10.1
-jmespath==0.9.4
+importlib-metadata==0.23
+isort==4.3.21
+Jinja2==2.10.3
 jsondiff==1.1.2
-jsonlines==1.2.0
 jsonpatch==1.24
 jsonpickle==1.2
 jsonpointer==2.0
-jsonschema==2.6.0
-lazy-object-proxy==1.4.1
+jsonschema==3.1.1
+lazy-object-proxy==1.4.3
 MarkupSafe==1.1.1
 mccabe==0.6.1
-mock==3.0.5
-moto==1.3.13
-netaddr==0.7.19
-nose==1.3.7
-nose-timer==0.7.5
-oauth2client==4.1.3
-packaging==19.0
-pathlib2==2.3.4
-pbr==5.3.1
-pika==1.0.1
-policyuniverse==1.3.2.0
-prompt-toolkit==2.0.9
-protobuf==3.8.0
-pyaml==18.11.0
-pyasn1==0.4.5
-pyasn1-modules==0.2.5
+more-itertools==7.2.0
+packaging==19.2
+pbr==5.4.3
+pika==1.1.0
+pluggy==0.13.0
+prompt-toolkit==2.0.10
+protobuf==3.10.0
+py==1.8.0
+pyasn1==0.4.7
+pyasn1-modules==0.2.7
 pycodestyle==2.5.0
 pycparser==2.19
-pycryptodome==3.8.2
-pyfakefs==3.5.8
 pyflakes==2.1.1
 Pygments==2.4.2
 PyJWT==1.7.1
-pylint==2.3.1
-pyOpenSSL==18.0.0
-pyparsing==2.4.0
-pyrsistent==0.15.4
+pyparsing==2.4.2
+pyrsistent==0.15.5
+pytest==5.0.0
 python-dateutil==2.8.0
 python-jose==3.0.1
-pytz==2019.1
-PyYAML==5.1.1
-requests==2.22.0
+pytz==2019.3
+PyYAML==5.1.2
 requests-toolbelt==0.9.1
 responses==0.10.6
 rsa==4.0
 s3transfer==0.2.1
-scandir==1.9.0
-singledispatch==3.4.0.3
 six==1.12.0
 smmap2==2.0.5
-snowballstemmer==1.2.1
+snowballstemmer==2.0.0
 solrq==1.1.1
-Sphinx==2.1.2
-sphinx-rtd-theme==0.4.3
 sphinxcontrib-applehelp==1.0.1
 sphinxcontrib-devhelp==1.0.1
 sphinxcontrib-htmlhelp==1.0.2
 sphinxcontrib-jsmath==1.0.1
 sphinxcontrib-qthelp==1.0.2
 sphinxcontrib-serializinghtml==1.1.3
-sphinxcontrib-websupport==1.1.0
 sshpubkeys==3.1.0
-stevedore==1.30.1
+stevedore==1.31.0
 typed-ast==1.4.0
-typing==3.6.6
 uritemplate==3.0.0
-urllib3==1.25.3
-validators==0.13.0
+urllib3==1.25.6
+validators==0.14.0
 wcwidth==0.1.7
 websocket-client==0.56.0
-Werkzeug==0.15.4
+Werkzeug==0.16.0
 wrapt==1.11.2
 xmltodict==0.12.0
-yapf==0.27.0
+zipp==0.6.0
 zope.interface==4.6.0

streamalert/alert_processor/outputs/aws.py

@@ -428,7 +428,7 @@ def _dispatch(self, alert, descriptor):
         """
         queue_name = self.config[self.__service__][descriptor]
         sqs = boto3.resource('sqs', region_name=self.region)
-        queue = sqs.get_queue_by_name(QueueName=queue_name)
+        queue = sqs.get_queue_by_name(QueueName=queue_name)  # pylint: disable=no-member
 
         publication = compose_alert(alert, self, descriptor)
 

streamalert/apps/_apps/gsuite.py

@@ -19,8 +19,10 @@
 import socket
 import ssl
 
-import apiclient
-from oauth2client import client, service_account
+import googleapiclient.discovery
+import googleapiclient.errors
+from google.oauth2 import service_account
+from google.auth.exceptions import GoogleAuthError
 
 from . import AppIntegration, StreamAlertApp, get_logger
 
@@ -36,8 +38,12 @@ class GSuiteReportsApp(AppIntegration):
     """G Suite Reports base app integration. This is subclassed for various endpoints"""
     _SCOPES = ['https://www.googleapis.com/auth/admin.reports.audit.readonly']
     # A tuple of uncaught exceptions that the googleapiclient can raise
-    _GOOGLE_API_EXCEPTIONS = (apiclient.errors.Error, client.Error, socket.timeout,
-                              ssl.SSLError)
+    _GOOGLE_API_EXCEPTIONS = (
+        googleapiclient.errors.Error,
+        GoogleAuthError,
+        socket.timeout,
+        ssl.SSLError,
+    )
 
     # The maximum number of unique event ids to store that occur on the most
     # recent timestamp. These are used to de-duplicate events in the next poll.
@@ -73,12 +79,14 @@ def _load_credentials(cls, keydata):
                 JSON file
 
         Returns:
-            oauth2client.service_account.ServiceAccountCredentials: Instance of
+             google.oauth2.service_account.ServiceAccountCredentials: Instance of
                 service account credentials for this discovery service
         """
         try:
-            creds = service_account.ServiceAccountCredentials.from_json_keyfile_dict(
-                keydata, scopes=cls._SCOPES)
+            creds = service_account.Credentials.from_service_account_info(
+                keydata,
+                scopes=cls._SCOPES,
+            )
         except (ValueError, KeyError):
             # This has the potential to raise errors. See: https://tinyurl.com/y8q5e9rm
             LOGGER.exception('[%s] Could not generate credentials from keyfile', cls.type())
@@ -103,9 +111,13 @@ def _create_service(self):
         if not creds:
             return False
 
-        delegation = creds.create_delegated(self._config.auth['delegation_email'])
+        delegation = creds.with_subject(self._config.auth['delegation_email'])
         try:
-            resource = apiclient.discovery.build('admin', 'reports_v1', credentials=delegation)
+            resource = googleapiclient.discovery.build(
+                'admin',
+                'reports_v1',
+                credentials=delegation
+            )
         except self._GOOGLE_API_EXCEPTIONS:
             LOGGER.exception('[%s] Failed to build discovery service', self)
             return False
@@ -229,6 +241,15 @@ def _sleep_seconds(self):
         return 0
 
 
+@StreamAlertApp
+class GSuiteAccessTransparencyReports(GSuiteReportsApp):
+    """G Suite Access Transparency  Report app integration"""
+
+    @classmethod
+    def _type(cls):
+        return 'access_transparency'
+
+
 @StreamAlertApp
 class GSuiteAdminReports(GSuiteReportsApp):
     """G Suite Admin Activity Report app integration"""
@@ -257,12 +278,21 @@ def _type(cls):
 
 
 @StreamAlertApp
-class GSuiteGroupsReports(GSuiteReportsApp):
-    """G Suite Groups Activity Report app integration"""
+class GSuiteGroupReports(GSuiteReportsApp):
+    """G Suite Group Activity Report app integration"""
+
+    @classmethod
+    def _type(cls):
+        return 'group'
+
+
+@StreamAlertApp
+class GSuiteGroupEnterpriseReports(GSuiteReportsApp):
+    """G Suite Groups Enterprise Activity Report app integration"""
 
     @classmethod
     def _type(cls):
-        return 'groups'
+        return 'groups_enterprise'
 
 
 @StreamAlertApp
@@ -302,18 +332,18 @@ def _type(cls):
 
 
 @StreamAlertApp
-class GSuiteSAMLReports(GSuiteReportsApp):
-    """G Suite SAML Activity Report app integration"""
+class GSuiteTokenReports(GSuiteReportsApp):
+    """G Suite Token Activity Report app integration"""
 
     @classmethod
     def _type(cls):
-        return 'saml'
+        return 'token'
 
 
 @StreamAlertApp
-class GSuiteTokenReports(GSuiteReportsApp):
-    """G Suite Token Activity Report app integration"""
+class GSuiteUserAccountsReports(GSuiteReportsApp):
+    """G Suite User Accounts Activity Report app integration"""
 
     @classmethod
     def _type(cls):
-        return 'token'
+        return 'user_accounts'

streamalert/classifier/parsers.py

@@ -732,8 +732,8 @@ def _parse_row(self, row, schema):
                 if parsed_data and parsed_data[0][1] is True:
                     parsed_payload[key] = parsed_data[0][0]
                     continue
-                else:
-                    return False  # break the loop if the nested data was invalid
+
+                return False  # break the loop if the nested data was invalid
 
             # extract the keys from the row via the index
             parsed_payload[key] = row[index]

streamalert/rules_engine/threat_intel.py

@@ -207,9 +207,7 @@ def _run_query(query_values, results):
                 }
             )
 
-            results.extend([
-                result for result in self._deserialize(response['Responses'].get(self._table))
-            ])
+            results.extend(self._deserialize(response['Responses'].get(self._table)))
 
             # Log this as an error for now so it can be picked up in logs
             if response['UnprocessedKeys']:

streamalert/shared/alert_table.py

@@ -131,7 +131,7 @@ def get_alert_record(self, rule_name, alert_id):
             'KeyConditionExpression': Key('RuleName').eq(rule_name) & Key('AlertID').eq(alert_id)
         }
         items = list(self._paginate(self._table.query, kwargs))
-        return items[0] if items else None
+        return items[0] if items else {}
 
     # ---------- Add/Delete/Update Operations ----------