diff --git a/ingestion/monitoring/errorLogsToSlack.py b/ingestion/monitoring/errorLogsToSlack.py index 194d1942c..4ffb4f7a0 100644 --- a/ingestion/monitoring/errorLogsToSlack.py +++ b/ingestion/monitoring/errorLogsToSlack.py @@ -4,6 +4,7 @@ import requests import re import sys +from time import sleep import boto3 @@ -17,25 +18,33 @@ def emit(self, record): message_header = {'Content-Type': 'application/json'} message = {'text': f"[{record.levelname}] {record.message}"} response = requests.post(url=self.slack_url, data=json.dumps(message), headers=message_header) - if response.status_code != 200: + if response.status_code == 429 and response['error'] == 'rate_limited': + sleep(response['retry_after']) + elif response.status_code != 200: raise ValueError( f"Request to slack returned an error {response.status_code}, the response is:\n{response.text}" ) def interpret(message): + graham = "<@U011A0TFM7X>" + abhishek = "<@U01F70FAJ6N>" + jim = "<@U01TAHDR4F7>" + engineers = f"{graham} {abhishek} {jim}" lower = message.lower() if "'dateRange': {'start':".lower() in lower: - return (logger.INFO, f"BACKFILL <@U01F70GPXNW>\n{message}") + return (logging.INFO, f"BACKFILL INITIATED\n{message}") if "error" in lower: - return (logger.ERROR, f"ERROR TYPE: Parser <@U011A0TFM7X> <@U017KLSPEM7>\n{message}") + return (logging.ERROR, f"PARSER ERROR: {engineers}\n{message}") if "timed out" in lower: - return (logger.ERROR, f"ERROR TYPE: Time out\n{message}") - return (logger.WARN, message) + return (logging.ERROR, f"TIME OUT: {engineers}\n{message}") + if lower.startswith('info:'): + return (logging.INFO, message) + return (logging.WARN, message) def setup_logger(): logger = logging.getLogger(__name__) - logger.setLevel(logging.DEBUG) + logger.setLevel(logging.WARN) stdoutHandler = logging.StreamHandler(stream=sys.stdout) stdoutHandler.setLevel(logging.DEBUG) logger.addHandler(stdoutHandler) @@ -43,6 +52,11 @@ def setup_logger(): logger.addHandler(slackHandler) return logger +def log_messages(cloudwatch_response, logger): + for message in [e['message'] for e in cloudwatch_response['events']]: + (severity, output) = interpret(message) + logger.log(severity, output) + if __name__ == '__main__': logger = setup_logger() @@ -52,9 +66,18 @@ def setup_logger(): logger.critical(f"Cannot get messages from log group {logGroup} and stream {logStream}") sys.exit(1) logger.info(f"Output from {logGroup}/{logStream}:") - hasMore = True - oldNext = None + hasMore = False + oldNext = '' logClient = boto3.client('logs') + response = logClient.get_log_events( + logGroupName=logGroup, + logStreamName=logStream, + startFromHead=True + ) + log_messages(response, logger) + oldNext = response['nextForwardToken'] + if oldNext and len(oldNext) > 0: + hasMore = true while hasMore: response = logClient.get_log_events( logGroupName=logGroup, @@ -62,12 +85,10 @@ def setup_logger(): startFromHead=True, nextToken=oldNext ) + log_messages(response, logger) newNext = response['nextForwardToken'] if (not newNext) or (newNext == oldNext): hasMore = False else: oldNext = newNext - for message in [e['message'] for e in response['events']]: - (severity, output) = interpret(message) - logger.log(severity, output) logger.info(f"End of output from {logGroup}/{logStream}") diff --git a/ingestion/monitoring/poetry.lock b/ingestion/monitoring/poetry.lock index 7167cb99c..04f9e67c5 100644 --- a/ingestion/monitoring/poetry.lock +++ b/ingestion/monitoring/poetry.lock @@ -30,6 +30,33 @@ urllib3 = ">=1.25.4,<1.27" [package.extras] crt = ["awscrt (==0.12.5)"] +[[package]] +name = "certifi" +version = "2021.10.8" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "charset-normalizer" +version = "2.0.12" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" +optional = false +python-versions = ">=3.5.0" + +[package.extras] +unicode_backport = ["unicodedata2"] + +[[package]] +name = "idna" +version = "3.3" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" + [[package]] name = "jmespath" version = "0.10.0" @@ -49,6 +76,24 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" [package.dependencies] six = ">=1.5" +[[package]] +name = "requests" +version = "2.27.1" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""} +idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] + [[package]] name = "s3transfer" version = "0.5.2" @@ -87,7 +132,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "f72750d640129cd8b3aba6390ef68790154dad2ab0a38307e1e39ac9f4686b72" +content-hash = "60b861f5ce04f9e11a7be3886ba13e441bd4d943ed0a698db843dffa14d3b27b" [metadata.files] boto3 = [ @@ -98,6 +143,18 @@ botocore = [ {file = "botocore-1.24.8-py3-none-any.whl", hash = "sha256:9fbc5c57b31850c51c87abc3e166ed4e0f343665bec4e1a0ff814fbc9704642c"}, {file = "botocore-1.24.8.tar.gz", hash = "sha256:a5431d806dc75fb1844463d921759fcd8d387674443af8d7fd0867f296b02759"}, ] +certifi = [ + {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"}, + {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"}, +] +charset-normalizer = [ + {file = "charset-normalizer-2.0.12.tar.gz", hash = "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597"}, + {file = "charset_normalizer-2.0.12-py3-none-any.whl", hash = "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"}, +] +idna = [ + {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, + {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, +] jmespath = [ {file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"}, {file = "jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9"}, @@ -106,6 +163,10 @@ python-dateutil = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, ] +requests = [ + {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"}, + {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"}, +] s3transfer = [ {file = "s3transfer-0.5.2-py3-none-any.whl", hash = "sha256:7a6f4c4d1fdb9a2b640244008e142cbc2cd3ae34b386584ef044dd0f27101971"}, {file = "s3transfer-0.5.2.tar.gz", hash = "sha256:95c58c194ce657a5f4fb0b9e60a84968c808888aed628cd98ab8771fe1db98ed"}, diff --git a/ingestion/monitoring/pyproject.toml b/ingestion/monitoring/pyproject.toml index fcc85dbaf..fff3e6081 100644 --- a/ingestion/monitoring/pyproject.toml +++ b/ingestion/monitoring/pyproject.toml @@ -8,6 +8,7 @@ license = "MIT" [tool.poetry.dependencies] python = "^3.9" boto3 = "^1.21.8" +requests = "^2.27.1" [tool.poetry.dev-dependencies]