From 2e6cd0f2141c1b557d8d084dc4051c5f67be47b8 Mon Sep 17 00:00:00 2001 From: Abraham Chavez Date: Thu, 21 Sep 2023 23:10:01 -0700 Subject: [PATCH] [p] Fix: Lambda azul-service-anvilprod-servicecachehealth failed due to a timeout (partial #5467) --- lambdas/indexer/openapi.json | 18 ++++++------- lambdas/service/openapi.json | 34 ++++++++++++------------ src/azul/health.py | 43 ++++++++++++++----------------- test/health_check_test_case.py | 28 ++++++++++---------- test/service/test_health_check.py | 22 +++++----------- 5 files changed, 66 insertions(+), 79 deletions(-) diff --git a/lambdas/indexer/openapi.json b/lambdas/indexer/openapi.json index 6d54aaf8b7..bd0b0117fe 100644 --- a/lambdas/indexer/openapi.json +++ b/lambdas/indexer/openapi.json @@ -147,7 +147,7 @@ "description": "\nHealth check of the indexer REST API and all\nresources it depends on. This may take long time to complete\nand exerts considerable load on the API. For that reason it\nshould not be requested frequently or by automated\nmonitoring facilities that would be better served by the\n[`/health/fast`](#operations-Auxiliary-get_health_fast) or\n[`/health/cached`](#operations-Auxiliary-get_health_cached)\nendpoints.\n", "responses": { "200": { - "description": "\nThe checked resources\nare healthy.\n\nThe response consists of the following keys:\n\n* `other_lambdas` \nIndicates whether the companion REST API responds to HTTP requests.\n* `queues` \nReturns information about the SQS queues used by the indexer.\n* `progress` \nThe number of Data Store bundles pending to be indexed and the number\nof index documents in need of updating.\n* `api_endpoints` \nIndicates whether important service API endpoints are operational.\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`true`.\n\n\nAll of the nested `up` keys\nare `true`.\n", + "description": "\nThe checked resources\nare healthy.\n\nThe response consists of the following keys:\n\n* `other_lambdas` \nIndicates whether the companion REST API responds to HTTP requests.\n* `queues` \nReturns information about the SQS queues used by the indexer.\n* `progress` \nThe number of Data Store bundles pending to be indexed and the number\nof index documents in need of updating.\n* `api` \nIndicates whether an important service API endpoint is operational.\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`true`.\n\n\nAll of the nested `up` keys\nare `true`.\n", "content": { "application/json": { "schema": { @@ -183,7 +183,7 @@ "other_lambdas": {}, "queues": {}, "progress": {}, - "api_endpoints": {}, + "api": {}, "elasticsearch": {}, "up": true } @@ -191,7 +191,7 @@ } }, "503": { - "description": "\nAt least one of the checked resources\nis not healthy.\n\nThe response consists of the following keys:\n\n* `other_lambdas` \nIndicates whether the companion REST API responds to HTTP requests.\n* `queues` \nReturns information about the SQS queues used by the indexer.\n* `progress` \nThe number of Data Store bundles pending to be indexed and the number\nof index documents in need of updating.\n* `api_endpoints` \nIndicates whether important service API endpoints are operational.\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`false`.\n\n\nAt least one of the nested `up` keys\nis `false`.\n", + "description": "\nAt least one of the checked resources\nis not healthy.\n\nThe response consists of the following keys:\n\n* `other_lambdas` \nIndicates whether the companion REST API responds to HTTP requests.\n* `queues` \nReturns information about the SQS queues used by the indexer.\n* `progress` \nThe number of Data Store bundles pending to be indexed and the number\nof index documents in need of updating.\n* `api` \nIndicates whether an important service API endpoint is operational.\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`false`.\n\n\nAt least one of the nested `up` keys\nis `false`.\n", "content": { "application/json": { "schema": { @@ -227,7 +227,7 @@ "other_lambdas": {}, "queues": {}, "progress": {}, - "api_endpoints": {}, + "api": {}, "elasticsearch": {}, "up": false } @@ -530,7 +530,7 @@ "items": { "type": "string", "enum": [ - "api_endpoints", + "api", "elasticsearch", "other_lambdas", "progress", @@ -546,7 +546,7 @@ "description": "\nThis endpoint allows clients to request a health check on a\nspecific set of resources. Each resource is identified by a\n*key*, the same key under which the resource appears in a\n[`/health`](#operations-Auxiliary-get_health) response.\n", "responses": { "200": { - "description": "\nThe checked resources\nare healthy.\n\nThe response consists of the following keys:\n\n* `other_lambdas` \nIndicates whether the companion REST API responds to HTTP requests.\n* `queues` \nReturns information about the SQS queues used by the indexer.\n* `progress` \nThe number of Data Store bundles pending to be indexed and the number\nof index documents in need of updating.\n* `api_endpoints` \nIndicates whether important service API endpoints are operational.\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`true`.\n\n\nAll of the nested `up` keys\nare `true`.\n", + "description": "\nThe checked resources\nare healthy.\n\nThe response consists of the following keys:\n\n* `other_lambdas` \nIndicates whether the companion REST API responds to HTTP requests.\n* `queues` \nReturns information about the SQS queues used by the indexer.\n* `progress` \nThe number of Data Store bundles pending to be indexed and the number\nof index documents in need of updating.\n* `api` \nIndicates whether an important service API endpoint is operational.\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`true`.\n\n\nAll of the nested `up` keys\nare `true`.\n", "content": { "application/json": { "schema": { @@ -582,7 +582,7 @@ "other_lambdas": {}, "queues": {}, "progress": {}, - "api_endpoints": {}, + "api": {}, "elasticsearch": {}, "up": true } @@ -590,7 +590,7 @@ } }, "503": { - "description": "\nAt least one of the checked resources\nis not healthy.\n\nThe response consists of the following keys:\n\n* `other_lambdas` \nIndicates whether the companion REST API responds to HTTP requests.\n* `queues` \nReturns information about the SQS queues used by the indexer.\n* `progress` \nThe number of Data Store bundles pending to be indexed and the number\nof index documents in need of updating.\n* `api_endpoints` \nIndicates whether important service API endpoints are operational.\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`false`.\n\n\nAt least one of the nested `up` keys\nis `false`.\n", + "description": "\nAt least one of the checked resources\nis not healthy.\n\nThe response consists of the following keys:\n\n* `other_lambdas` \nIndicates whether the companion REST API responds to HTTP requests.\n* `queues` \nReturns information about the SQS queues used by the indexer.\n* `progress` \nThe number of Data Store bundles pending to be indexed and the number\nof index documents in need of updating.\n* `api` \nIndicates whether an important service API endpoint is operational.\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`false`.\n\n\nAt least one of the nested `up` keys\nis `false`.\n", "content": { "application/json": { "schema": { @@ -626,7 +626,7 @@ "other_lambdas": {}, "queues": {}, "progress": {}, - "api_endpoints": {}, + "api": {}, "elasticsearch": {}, "up": false } diff --git a/lambdas/service/openapi.json b/lambdas/service/openapi.json index b67cd3d9fb..e1f3609cc5 100644 --- a/lambdas/service/openapi.json +++ b/lambdas/service/openapi.json @@ -90,7 +90,7 @@ "description": "\nHealth check of the service REST API and all\nresources it depends on. This may take long time to complete\nand exerts considerable load on the API. For that reason it\nshould not be requested frequently or by automated\nmonitoring facilities that would be better served by the\n[`/health/fast`](#operations-Auxiliary-get_health_fast) or\n[`/health/cached`](#operations-Auxiliary-get_health_cached)\nendpoints.\n", "responses": { "200": { - "description": "\nThe checked resources\nare healthy.\n\nThe response consists of the following keys:\n\n* `other_lambdas` \nIndicates whether the companion REST API responds to HTTP requests.\n* `queues` \nReturns information about the SQS queues used by the indexer.\n* `progress` \nThe number of Data Store bundles pending to be indexed and the number\nof index documents in need of updating.\n* `api_endpoints` \nIndicates whether important service API endpoints are operational.\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`true`.\n\n\nAll of the nested `up` keys\nare `true`.\n", + "description": "\nThe checked resources\nare healthy.\n\nThe response consists of the following keys:\n\n* `other_lambdas` \nIndicates whether the companion REST API responds to HTTP requests.\n* `queues` \nReturns information about the SQS queues used by the indexer.\n* `progress` \nThe number of Data Store bundles pending to be indexed and the number\nof index documents in need of updating.\n* `api` \nIndicates whether an important service API endpoint is operational.\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`true`.\n\n\nAll of the nested `up` keys\nare `true`.\n", "content": { "application/json": { "schema": { @@ -126,7 +126,7 @@ "other_lambdas": {}, "queues": {}, "progress": {}, - "api_endpoints": {}, + "api": {}, "elasticsearch": {}, "up": true } @@ -134,7 +134,7 @@ } }, "503": { - "description": "\nAt least one of the checked resources\nis not healthy.\n\nThe response consists of the following keys:\n\n* `other_lambdas` \nIndicates whether the companion REST API responds to HTTP requests.\n* `queues` \nReturns information about the SQS queues used by the indexer.\n* `progress` \nThe number of Data Store bundles pending to be indexed and the number\nof index documents in need of updating.\n* `api_endpoints` \nIndicates whether important service API endpoints are operational.\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`false`.\n\n\nAt least one of the nested `up` keys\nis `false`.\n", + "description": "\nAt least one of the checked resources\nis not healthy.\n\nThe response consists of the following keys:\n\n* `other_lambdas` \nIndicates whether the companion REST API responds to HTTP requests.\n* `queues` \nReturns information about the SQS queues used by the indexer.\n* `progress` \nThe number of Data Store bundles pending to be indexed and the number\nof index documents in need of updating.\n* `api` \nIndicates whether an important service API endpoint is operational.\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`false`.\n\n\nAt least one of the nested `up` keys\nis `false`.\n", "content": { "application/json": { "schema": { @@ -170,7 +170,7 @@ "other_lambdas": {}, "queues": {}, "progress": {}, - "api_endpoints": {}, + "api": {}, "elasticsearch": {}, "up": false } @@ -278,7 +278,7 @@ "description": "\nReturn a cached copy of the\n[`/health/fast`](#operations-Auxiliary-get_health_fast)\nresponse. This endpoint is optimized for continuously\nrunning, distributed health monitors such as Route 53 health\nchecks. The cache ensures that the service is not\noverloaded by these types of health monitors. The cache is\nupdated every minute.\n", "responses": { "200": { - "description": "\nThe checked resources\nare healthy.\n\nThe response consists of the following keys:\n\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `api_endpoints` \nIndicates whether important service API endpoints are operational.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`true`.\n\n\nAll of the nested `up` keys\nare `true`.\n", + "description": "\nThe checked resources\nare healthy.\n\nThe response consists of the following keys:\n\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `api` \nIndicates whether an important service API endpoint is operational.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`true`.\n\n\nAll of the nested `up` keys\nare `true`.\n", "content": { "application/json": { "schema": { @@ -312,14 +312,14 @@ }, "example": { "elasticsearch": {}, - "api_endpoints": {}, + "api": {}, "up": true } } } }, "503": { - "description": "\nAt least one of the checked resources\nis not healthy.\n\nThe response consists of the following keys:\n\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `api_endpoints` \nIndicates whether important service API endpoints are operational.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`false`.\n\n\nAt least one of the nested `up` keys\nis `false`.\n", + "description": "\nAt least one of the checked resources\nis not healthy.\n\nThe response consists of the following keys:\n\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `api` \nIndicates whether an important service API endpoint is operational.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`false`.\n\n\nAt least one of the nested `up` keys\nis `false`.\n", "content": { "application/json": { "schema": { @@ -353,7 +353,7 @@ }, "example": { "elasticsearch": {}, - "api_endpoints": {}, + "api": {}, "up": false } } @@ -371,7 +371,7 @@ "description": "\nPerformance-optimized health check of the REST API and other\ncritical resources tht it depends on. This endpoint can be\nrequested more frequently than\n[`/health`](#operations-Auxiliary-get_health) but\nperiodically scheduled, automated requests should be made to\n[`/health/cached`](#operations-Auxiliary-get_health_cached).\n", "responses": { "200": { - "description": "\nThe checked resources\nare healthy.\n\nThe response consists of the following keys:\n\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `api_endpoints` \nIndicates whether important service API endpoints are operational.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`true`.\n\n\nAll of the nested `up` keys\nare `true`.\n", + "description": "\nThe checked resources\nare healthy.\n\nThe response consists of the following keys:\n\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `api` \nIndicates whether an important service API endpoint is operational.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`true`.\n\n\nAll of the nested `up` keys\nare `true`.\n", "content": { "application/json": { "schema": { @@ -405,14 +405,14 @@ }, "example": { "elasticsearch": {}, - "api_endpoints": {}, + "api": {}, "up": true } } } }, "503": { - "description": "\nAt least one of the checked resources\nis not healthy.\n\nThe response consists of the following keys:\n\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `api_endpoints` \nIndicates whether important service API endpoints are operational.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`false`.\n\n\nAt least one of the nested `up` keys\nis `false`.\n", + "description": "\nAt least one of the checked resources\nis not healthy.\n\nThe response consists of the following keys:\n\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `api` \nIndicates whether an important service API endpoint is operational.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`false`.\n\n\nAt least one of the nested `up` keys\nis `false`.\n", "content": { "application/json": { "schema": { @@ -446,7 +446,7 @@ }, "example": { "elasticsearch": {}, - "api_endpoints": {}, + "api": {}, "up": false } } @@ -469,7 +469,7 @@ "items": { "type": "string", "enum": [ - "api_endpoints", + "api", "elasticsearch", "other_lambdas", "progress", @@ -485,7 +485,7 @@ "description": "\nThis endpoint allows clients to request a health check on a\nspecific set of resources. Each resource is identified by a\n*key*, the same key under which the resource appears in a\n[`/health`](#operations-Auxiliary-get_health) response.\n", "responses": { "200": { - "description": "\nThe checked resources\nare healthy.\n\nThe response consists of the following keys:\n\n* `other_lambdas` \nIndicates whether the companion REST API responds to HTTP requests.\n* `queues` \nReturns information about the SQS queues used by the indexer.\n* `progress` \nThe number of Data Store bundles pending to be indexed and the number\nof index documents in need of updating.\n* `api_endpoints` \nIndicates whether important service API endpoints are operational.\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`true`.\n\n\nAll of the nested `up` keys\nare `true`.\n", + "description": "\nThe checked resources\nare healthy.\n\nThe response consists of the following keys:\n\n* `other_lambdas` \nIndicates whether the companion REST API responds to HTTP requests.\n* `queues` \nReturns information about the SQS queues used by the indexer.\n* `progress` \nThe number of Data Store bundles pending to be indexed and the number\nof index documents in need of updating.\n* `api` \nIndicates whether an important service API endpoint is operational.\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`true`.\n\n\nAll of the nested `up` keys\nare `true`.\n", "content": { "application/json": { "schema": { @@ -521,7 +521,7 @@ "other_lambdas": {}, "queues": {}, "progress": {}, - "api_endpoints": {}, + "api": {}, "elasticsearch": {}, "up": true } @@ -529,7 +529,7 @@ } }, "503": { - "description": "\nAt least one of the checked resources\nis not healthy.\n\nThe response consists of the following keys:\n\n* `other_lambdas` \nIndicates whether the companion REST API responds to HTTP requests.\n* `queues` \nReturns information about the SQS queues used by the indexer.\n* `progress` \nThe number of Data Store bundles pending to be indexed and the number\nof index documents in need of updating.\n* `api_endpoints` \nIndicates whether important service API endpoints are operational.\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`false`.\n\n\nAt least one of the nested `up` keys\nis `false`.\n", + "description": "\nAt least one of the checked resources\nis not healthy.\n\nThe response consists of the following keys:\n\n* `other_lambdas` \nIndicates whether the companion REST API responds to HTTP requests.\n* `queues` \nReturns information about the SQS queues used by the indexer.\n* `progress` \nThe number of Data Store bundles pending to be indexed and the number\nof index documents in need of updating.\n* `api` \nIndicates whether an important service API endpoint is operational.\n* `elasticsearch` \nIndicates whether the Elasticsearch cluster is responsive.\n* `up` \nindicates the overall result of the health check\n\n\nThe top-level `up` key of the response is\n`false`.\n\n\nAt least one of the nested `up` keys\nis `false`.\n", "content": { "application/json": { "schema": { @@ -565,7 +565,7 @@ "other_lambdas": {}, "queues": {}, "progress": {}, - "api_endpoints": {}, + "api": {}, "elasticsearch": {}, "up": false } diff --git a/src/azul/health.py b/src/azul/health.py index a9cdc1d07d..85943da1a7 100644 --- a/src/azul/health.py +++ b/src/azul/health.py @@ -3,14 +3,12 @@ Mapping, Set, ) -from concurrent.futures import ( - ThreadPoolExecutor, -) from itertools import ( chain, ) import json import logging +import random import time from typing import ( ClassVar, @@ -238,34 +236,33 @@ def progress(self) -> JSON: )) } - def _api_endpoint(self, relative_url: furl) -> tuple[str, JSON]: - url = str(config.service_endpoint.join(relative_url)) - log.info('Requesting %r', url) - response = requests.head(url) - try: - response.raise_for_status() - except requests.exceptions.HTTPError as e: - return url, {'up': False, 'error': repr(e)} - else: - return url, {'up': True} - @cached_property def entity_types(self): return self.controller.metadata_plugin(self.catalog).exposed_indices.keys() @health_property - def api_endpoints(self): + def api(self) -> JSON: """ - Indicates whether important service API endpoints are operational. + Indicates whether an important service API endpoint is operational. """ - endpoints = [ + # Select any single entity index endpoint from the ones available + random.seed(a=1695859751) + endpoint = random.choice([ furl(path=('index', entity_type), args={'size': '1'}) for entity_type in self.entity_types - ] - with ThreadPoolExecutor(len(endpoints)) as tpe: - status = dict(tpe.map(self._api_endpoint, endpoints)) - status['up'] = all(v['up'] for v in status.values()) - return status + ]) + endpoint = str(config.service_endpoint.join(endpoint)) + log.debug('_request(HEAD, %s)', endpoint) + start = time.time() + response = requests.head(endpoint) + log.debug('_request(…) -> %.3fs %r', + time.time() - start, response.status_code) + try: + response.raise_for_status() + except requests.exceptions.HTTPError as e: + return {'up': False, 'error': repr(e)} + else: + return {'up': True} @health_property def elasticsearch(self): @@ -303,7 +300,7 @@ def _lambda(self, lambda_name) -> JSON: ), 'service': ( elasticsearch, - api_endpoints, + api, ) } diff --git a/test/health_check_test_case.py b/test/health_check_test_case.py index 726af18024..449ce5bebc 100644 --- a/test/health_check_test_case.py +++ b/test/health_check_test_case.py @@ -6,6 +6,7 @@ Mapping, ) import os +import random import time from unittest import ( TestSuite, @@ -98,7 +99,7 @@ def test_health_all_ok(self): **self._expected_elasticsearch(True), **self._expected_queues(True), **self._expected_other_lambdas(True), - **self._expected_api_endpoints(endpoint_states), + **self._expected_api(endpoint_states), **self._expected_progress() }, response.json()) @@ -114,7 +115,7 @@ def test_health_endpoint_keys(self): ('elasticsearch', self._expected_elasticsearch(True)), ('queues', self._expected_queues(True)), ('other_lambdas', self._expected_other_lambdas(True)), - ('api_endpoints', self._expected_api_endpoints(endpoint_states)), + ('api', self._expected_api(endpoint_states)), ('progress', self._expected_progress()), ('progress,queues', {**self._expected_progress(), **self._expected_queues(True)}), ] @@ -219,20 +220,19 @@ def _expected_queues(self, up: bool) -> JSON: } } - def _expected_api_endpoints(self, endpoint_states: Mapping[str, bool]) -> JSON: + def _expected_api(self, endpoint_states: Mapping[str, bool]) -> JSON: + random.seed(a=1695859751) + endpoint = random.choice([*endpoint_states]) return { - 'api_endpoints': { - 'up': all(up for endpoint, up in endpoint_states.items()), + 'api': { **({ - self._endpoint(endpoint): { - 'up': up - } if up else { - 'up': up, - 'error': ( - "HTTPError('503 Server Error: " - "Service Unavailable for url: " - f"{self._endpoint(endpoint)}')") - } for endpoint, up in endpoint_states.items() + 'up': True + } if endpoint_states[endpoint] else { + 'up': False, + 'error': ( + "HTTPError('503 Server Error: " + "Service Unavailable for url: " + f"{self._endpoint(endpoint)}')") }) } } diff --git a/test/service/test_health_check.py b/test/service/test_health_check.py index 9f4dad9b77..37d1812f56 100644 --- a/test/service/test_health_check.py +++ b/test/service/test_health_check.py @@ -37,28 +37,18 @@ def _expected_health(self, return { 'up': False, **self._expected_elasticsearch(es_up), - **self._expected_api_endpoints(endpoint_states), + **self._expected_api(endpoint_states), } @mock_sts @mock_sqs - def test_all_api_endpoints_down(self): + def test_api_down(self): self._create_mock_queues() - endpoint_states = self._endpoint_states(up_endpoints=(), - down_endpoints=self.endpoints) - response = self._test(endpoint_states, lambdas_up=True) + endpoint_state = self._endpoint_states(up_endpoints=(), + down_endpoints=self.endpoints) + response = self._test(endpoint_state, lambdas_up=True) self.assertEqual(503, response.status_code) - self.assertEqual(self._expected_health(endpoint_states), response.json()) - - @mock_sts - @mock_sqs - def test_one_api_endpoint_down(self): - self._create_mock_queues() - endpoint_states = self._endpoint_states(up_endpoints=self.endpoints[1:], - down_endpoints=self.endpoints[:1]) - response = self._test(endpoint_states, lambdas_up=True) - self.assertEqual(503, response.status_code) - self.assertEqual(self._expected_health(endpoint_states), response.json()) + self.assertEqual('Service Unavailable', response.reason) del HealthCheckTestCase