Merge branch 'develop'

.github/workflows/check-publish.yml

@@ -0,0 +1,112 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Check and publish
+
+on:
+
+  workflow_dispatch: {}
+
+  pull_request:
+    branches: [master]
+
+  release:
+    types: [created]
+
+jobs:
+
+  quality:
+    name: Code QA
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: pip install black flake8 isort
+      - run: black --version
+      - run: isort --check .
+      - run: black --check .
+      - run: flake8 .
+
+  checks:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: ["ubuntu-latest"]
+        python-version: ["3.6", "3.7", "3.8", "3.9"]
+
+    name: Python ${{ matrix.python-version }} on ${{ matrix.platform }}
+    runs-on: ${{ matrix.platform }}
+    needs: quality
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install etcd
+        run: |
+          ETCD_VER=v3.4.14
+          DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download
+          curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
+          mkdir /tmp/etcd-download-test
+          tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1
+          /tmp/etcd-download-test/etcd &
+
+      - name: Install packages
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel
+          pip install -e .
+          pip install -e aviso-server/monitoring
+          pip install -e aviso-server/rest
+          pip install -e aviso-server/auth
+          pip install -e aviso-server/admin
+          pip install -U -r tests/requirements-dev.txt
+          pip install -r docs/requirements.txt
+
+      - name: Tests
+        run: |
+          pip freeze
+          pytest
+        env:
+          AVISO_DEBUG: True
+
+      - name: Documentation
+        run: |
+          make clean
+          make html
+        working-directory: docs
+
+  deploy:
+    if: ${{ github.event_name == 'release' }}
+
+    name: Upload to Pypi
+    needs: checks
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.8'
+
+    - name: Check version
+      run: |
+        release=${GITHUB_REF##*/}
+        version=$(python setup.py --version)
+        test "$release" == "$version"
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine
+
+    - name: Build and publish
+      env:
+        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      run: |
+        python setup.py sdist bdist_wheel
+        twine upload dist/*

.github/workflows/check.yaml

@@ -0,0 +1,69 @@
+name: Check
+
+on:
+  workflow_dispatch:
+
+  push:
+    branches: [develop]
+
+jobs:
+
+  quality:
+    name: Code QA
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: pip install black flake8 isort
+      - run: black --version
+      - run: isort --check .
+      - run: black --check .
+      - run: flake8 .
+
+  checks:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: ["ubuntu-latest"]
+        python-version: ["3.6", "3.7", "3.8", "3.9"]
+
+    name: Python ${{ matrix.python-version }} on ${{ matrix.platform }}
+    runs-on: ${{ matrix.platform }}
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install etcd
+        run: |
+          ETCD_VER=v3.4.14
+          DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download
+          curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
+          mkdir /tmp/etcd-download-test
+          tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1
+          /tmp/etcd-download-test/etcd &
+
+      - name: Install packages
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel
+          pip install -e .
+          pip install -e aviso-server/monitoring
+          pip install -e aviso-server/rest
+          pip install -e aviso-server/auth
+          pip install -e aviso-server/admin
+          pip install -U -r tests/requirements-dev.txt
+          pip install -r docs/requirements.txt
+
+      - name: Tests
+        run: |
+          pytest -v --cov=pyaviso --cache-clear
+        env:
+          AVISO_DEBUG: True
+      - name: Documentation
+        run: |
+          make clean
+          make html
+        working-directory: docs

.gitignore

@@ -1,7 +1,6 @@
 .*
 !.gitignore
-!.dockerignore
-venvDev*/*
+!.github
 __pycache*
 htmlcov
 pyaviso.egg-info/*

MANIFEST.in

@@ -1 +1,2 @@
-include pyaviso/event_listeners/default_listener_schema.json
+include pyaviso/event_listeners/default_listener_schema.json
+include requirements.txt

aviso-server/admin/aviso_admin/__init__.py

@@ -1,5 +1,5 @@
 # (C) Copyright 1996- ECMWF.
-# 
+#
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
 # In applying this licence, ECMWF does not waive the privileges and immunities
@@ -9,7 +9,7 @@
 import logging
 
 # version number for the application.
-__version__ = '0.3.0'
+__version__ = "0.4.0"
 
 # setting application logger
 logger = logging.getLogger("aviso-admin")

aviso-server/admin/aviso_admin/admin.py

@@ -9,21 +9,24 @@
 import time
 
 import schedule
-from aviso_admin import logger, __version__
+from aviso_admin import __version__, logger
 from aviso_admin.cleaner import Cleaner
 from aviso_admin.compactor import Compactor
 from aviso_admin.config import Config
-from aviso_monitoring.reporter.aviso_rest_reporter import AvisoRestReporter
+from aviso_monitoring import __version__ as monitoring_version
+from aviso_monitoring.receiver import Receiver
 from aviso_monitoring.reporter.aviso_auth_reporter import AvisoAuthReporter
+from aviso_monitoring.reporter.aviso_rest_reporter import AvisoRestReporter
 from aviso_monitoring.reporter.etcd_reporter import EtcdReporter
-from aviso_monitoring.receiver import Receiver
+from aviso_monitoring.reporter.prometheus_reporter import PrometheusReporter
 from aviso_monitoring.udp_server import UdpServer
 
 
 def main():
     # load the configuration
     config = Config()
     logger.info(f"Running Aviso-admin v.{__version__}")
+    logger.info(f"aviso_monitoring module v.{monitoring_version}")
     logger.info(f"Configuration loaded: {config}")
 
     # instantiate the compactor and cleaner
@@ -50,10 +53,15 @@ def main():
     auth_reporter = AvisoAuthReporter(config.monitoring, receiver)
     if auth_reporter.enabled:
         schedule.every(auth_reporter.frequency).minutes.do(auth_reporter.run)
-    etcd_reporter = EtcdReporter(config.monitoring)
+    etcd_reporter = EtcdReporter(config.monitoring, receiver)
     if etcd_reporter.enabled:
         schedule.every(etcd_reporter.frequency).minutes.do(etcd_reporter.run)
 
+    # launch the prometheus reporter, this expose some tlms to /metrics
+    prometheus_reporter = PrometheusReporter(config.monitoring, receiver)
+    if prometheus_reporter.enabled:
+        prometheus_reporter.start()
+
     # Loop so that the scheduling task keeps on running all time.
     while True:
         # Checks whether a scheduled task is pending to run or not
@@ -64,4 +72,3 @@ def main():
 # when running directly from this file
 if __name__ == "__main__":
     main()
-