From c11d5c04a97e696c4994d660fe6e528396c7f2bf Mon Sep 17 00:00:00 2001 From: Bremer Jonathan Date: Fri, 28 Dec 2018 14:47:13 -0500 Subject: [PATCH] Handled deprecated ES stats (#61) * fixed logic for removing deprecated metrics * string to int for version numbers * added unit test * add dependency * added dependency * added dependency * fixed test errors * mocked module collectd * fixed docker build error * edited README file * halting execution on unit test failure * halting execution on unit test failure * halting execution on unit test failure * halting execution on unit test failure * halting execution on unit test failure * halting execution on unit test failure * halting execution on unit test failure * halting execution on unit test failure --- .circleci/config.yml | 6 ++++-- .gitignore | 7 ++++++- .travis.yml | 1 + elasticsearch_collectd.py | 8 ++++---- elasticsearch_collectd_test.py | 30 ++++++++++++++++++++++++++++++ tests/integration/test.py | 1 - tests/run_tests.sh | 7 +++++++ 7 files changed, 52 insertions(+), 8 deletions(-) create mode 100644 elasticsearch_collectd_test.py diff --git a/.circleci/config.yml b/.circleci/config.yml index b63cb1e..f46b125 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ jobs: set -x VER="17.03.0-ce" apt-get update -q - apt-get install -yq curl python + apt-get install -yq curl python-pip curl -L -o /tmp/docker-$VER.tgz https://get.docker.com/builds/Linux/x86_64/docker-$VER.tgz tar -xz -C /tmp -f /tmp/docker-$VER.tgz mv /tmp/docker/* /usr/bin @@ -27,8 +27,10 @@ jobs: name: Run basic tests working_directory: ~/code/tests command: | + pip install pytest + pip install mock bash run_tests.sh | tee /tmp/test.log || true # The test command always exits non-zero - grep -v -E 'FAILED|ERROR' /tmp/test.log || exit 1 + ! grep -E 'FAILED|ERROR' /tmp/test.log > /dev/null || exit 1 - run: name: Run integration tests working_directory: ~/code/tests/integration diff --git a/.gitignore b/.gitignore index b8de51b..3bc12fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,7 @@ .idea/ -*.tmp \ No newline at end of file +*.tmp +.Python +bin/ +include/ +lib/ +__pycache__/ \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index f2358c8..30e8f55 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,5 +4,6 @@ python: - "2.7" install: pip install flake8 + pip install pytest before_script: flake8 *.py diff --git a/elasticsearch_collectd.py b/elasticsearch_collectd.py index d547287..203bc47 100755 --- a/elasticsearch_collectd.py +++ b/elasticsearch_collectd.py @@ -755,14 +755,14 @@ def remove_deprecated_elements(deprecated, elements, version): # Attempt to parse the major, minor, and revision (major, minor, revision) = version.split('.') - # Sanitize alphas and betas from revision number - revision = revision.split('-')[0] + # Strings to int and sanitize alphas and betas from revision number + (major, minor, revision) = (int(major), int(minor), int(revision.split('-')[0])) # Iterate over deprecation lists and remove any keys that were deprecated # prior to the current version for dep in deprecated: - if (major >= dep['major']) \ - or (major == dep['major'] and minor >= dep['minor']) \ + if (major > dep['major']) \ + or (major == dep['major'] and minor > dep['minor']) \ or (major == dep['major'] and minor == dep['minor'] and revision >= dep['revision']): if type(elements) is list: diff --git a/elasticsearch_collectd_test.py b/elasticsearch_collectd_test.py new file mode 100644 index 0000000..e7c4a2b --- /dev/null +++ b/elasticsearch_collectd_test.py @@ -0,0 +1,30 @@ +import sys + +import mock +import pytest + + +class MockCollectd(mock.MagicMock): + pass + + +sys.modules['collectd'] = MockCollectd() + + +from elasticsearch_collectd import remove_deprecated_elements + + +@pytest.mark.parametrize("deprecated_elements, input_elements, version, expected_elements", [ + ( + [{'major': 1, 'minor': 3, 'revision': 100, 'keys': ['element1', 'element3']}], + ['element2', 'element3'], + '1.03.00104', + ['element2'] + ) +]) +def test_remove_deprecated_elements(deprecated_elements, input_elements, version, expected_elements): + print deprecated_elements, input_elements, version, expected_elements + elements = remove_deprecated_elements(deprecated_elements, input_elements, version) + assert len(elements) == len(expected_elements) + for i in range(len(elements)): + assert elements[i] == expected_elements[i] diff --git a/tests/integration/test.py b/tests/integration/test.py index a2aa13a..df445d4 100644 --- a/tests/integration/test.py +++ b/tests/integration/test.py @@ -42,7 +42,6 @@ def wait_for_metrics_from_each_cluster(): print 'Waiting for metric: %s from cluster %s...' % (metric, c) eventually_true(lambda: any([metric in str(m.get('type_instance')) for m in get_metric_data()]), TIMEOUT_SECS - (time() - start)) - print 'metric: %s Found! from cluster: %s' % (metric, c) diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 1bb70ab..e28a099 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -8,6 +8,13 @@ PYTHON_VERSION=$(${PYTHON} -V 2>&1) echo "Interpreter version: ${PYTHON_VERSION}" +echo "Running unit tests" +pytest ../elasticsearch_collectd_test.py +status=$? +if [[ "$status" != 0 ]]; then + exit ${status} +fi + tmpfile=$(mktemp /tmp/run_tests.sh.XXXXXX) trap 'rm -f $tmpfile' 1 2 3 15 for scenario in `ls data`; do