From 2c200363c85c8c3988bdc8c576faf90c85419b6e Mon Sep 17 00:00:00 2001 From: Kyle Gabriel Date: Sun, 20 Nov 2022 18:15:02 -0500 Subject: [PATCH] Docker fixes: improper influxdb host & empty logs --- .dockerignore | 3 ++ .gitignore | 2 ++ CHANGELOG.md | 2 ++ DOCKER.md | 2 ++ docker-compose.yml | 13 ++++---- docker/influxdb/Dockerfile | 6 ++-- install/setup_unattended.sh | 32 ++++++++++---------- mycodo/databases/models/misc.py | 28 ++++++++++++----- mycodo/mycodo_daemon.py | 3 ++ mycodo/mycodo_flask/routes_page.py | 34 +++++++++++---------- mycodo/scripts/measurement_db.py | 48 ++++++++++++++++++++++++++---- mycodo/scripts/upgrade_commands.sh | 6 ++-- 12 files changed, 122 insertions(+), 57 deletions(-) diff --git a/.dockerignore b/.dockerignore index 8366b89e5..43c7358fe 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,6 @@ databases env +.venv .upgrade +.github +.git diff --git a/.gitignore b/.gitignore index 81d1e0ae1..c286ec9f3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,9 @@ # Mycodo files test/ +tests/ site/ env/ +.venv /databases/flask_secret_key *.db *.pem diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f5c88c39..1d87aa191 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,8 @@ New Behavior: InfluxDB 1.x returned data as an epoch float value (e.g. 166722857 - Fix MCP23017 Pump Output initialization - Fix race condition with Kasa RGB Bulb that could sometimes turn it back on after turning off - Fix digits not properly displaying on PID Controller Widget + - Fix determining improper influxdb host in Docker + - Fix flask and nginx logs appearing empty in Docker ### Miscellaneous diff --git a/DOCKER.md b/DOCKER.md index a41523712..c364f355b 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -23,6 +23,8 @@ sudo service mycodoflask stop sudo service nginx stop ``` +If you are building for a Pi Zero, you will need to change `FROM influxdb:1.8.10` to `FROM mendhak/arm32v6-influxdb` in docker/influxdb/Dockerfile prior to building. + ### Install Prerequisites ```shell script diff --git a/docker-compose.yml b/docker-compose.yml index 7c433714a..5d06775ec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,9 +34,6 @@ services: container_name: mycodo_daemon image: app restart: always - build: - context: ./ - dockerfile: docker/Dockerfile volumes: - mycodo:/usr/local/mycodo - mycodo_env:/home/mycodo/env @@ -54,14 +51,18 @@ services: - /opt:/opt - /etc/localtime:/etc/localtime:ro # Use timezone of the host (read-only) privileged: true - command: bash -c "PYTHONPATH=/home/mycodo /home/mycodo/env/bin/python mycodo_daemon.py" + command: bash -c "wget --quiet --no-check-certificate -p http://mycodo_nginx/ -O /dev/null && + PYTHONPATH=/home/mycodo /home/mycodo/env/bin/python mycodo_daemon.py" depends_on: - - mycodo_influxdb + - mycodo_flask mycodo_flask: container_name: mycodo_flask image: app hostname: Mycodo + build: + context: ./ + dockerfile: docker/Dockerfile restart: always volumes: - mycodo:/usr/local/mycodo @@ -81,7 +82,7 @@ services: privileged: true command: /home/mycodo/env/bin/python -m gunicorn --workers 1 --bind unix:/usr/local/mycodo/mycodoflask.sock start_flask_ui:app depends_on: - - mycodo_daemon + - mycodo_influxdb # mycodo_telegraf: # image: telegraf:latest diff --git a/docker/influxdb/Dockerfile b/docker/influxdb/Dockerfile index 887007fb2..59ea89e0f 100644 --- a/docker/influxdb/Dockerfile +++ b/docker/influxdb/Dockerfile @@ -4,8 +4,8 @@ ENV DOCKER_CONTAINER TRUE RUN mkdir -pv /var/influxdb/data -COPY docker/influxdb/run.sh /run.sh -RUN chmod +x /run.sh +COPY ./docker/influxdb/run.sh /home/mycodo/docker/influxdb/run.sh +RUN chmod +x /home/mycodo/docker/influxdb/run.sh ENV PRE_CREATE_DB mycodo_db ENV ADMIN_USER mycodo @@ -13,4 +13,4 @@ ENV PASS mmdu77sj3nIoiajjs EXPOSE 8086 -CMD /run.sh +CMD /home/mycodo/docker/influxdb/run.sh diff --git a/install/setup_unattended.sh b/install/setup_unattended.sh index 8888485f1..26ce7fd5f 100644 --- a/install/setup_unattended.sh +++ b/install/setup_unattended.sh @@ -19,6 +19,22 @@ if [ "$EUID" -ne 0 ]; then exit 1 fi +case "${1:-''}" in + '1') + printf "#### Installing Mycodo with Influxdb 1.x\n" + ;; + '2') + printf "#### Installing Mycodo with Influxdb 2.x\n" + ;; + '0') + printf "#### Installing Mycodo without installing Influxdb\n" + ;; + *) + printf "Error: Unrecognized command: sudo setup_unattended.sh [influxdb version] ('1' for version 1.x and '2' for 2.x)\n" + exit 1 + ;; +esac + printf "Checking Python version...\n" if hash python3 2>/dev/null; then if ! python3 "${INSTALL_DIRECTORY}"/mycodo/scripts/upgrade_check.py --min_python_version "3.6"; then @@ -36,22 +52,6 @@ fi NOW=$(date) printf "### Mycodo installation initiated %s\n\n" "${NOW}" 2>&1 | tee -a "${LOG_LOCATION}" -case "${1:-''}" in - '1') - printf "#### Installing Mycodo with Influxdb 1.x\n" - ;; - '2') - printf "#### Installing Mycodo with Influxdb 2.x\n" - ;; - '0') - printf "#### Installing Mycodo without installing Influxdb\n" - ;; - *) - printf "Error: Unrecognized command: %s\n" "${1}" - exit 1 - ;; -esac - abort() { printf " diff --git a/mycodo/databases/models/misc.py b/mycodo/databases/models/misc.py index b5ea14ae5..5036547f2 100644 --- a/mycodo/databases/models/misc.py +++ b/mycodo/databases/models/misc.py @@ -1,8 +1,9 @@ # coding=utf-8 import logging -from mycodo.databases import CRUDMixin -from mycodo.databases import set_uuid +import requests + +from mycodo.databases import CRUDMixin, set_uuid from mycodo.mycodo_flask.extensions import db logger = logging.getLogger("mycodo.misc") @@ -48,21 +49,32 @@ class Misc(CRUDMixin, db.Model): default_login_page = db.Column(db.String, default='password') # Measurement database - measurement_db_name = db.Column(db.String, default='influxdb') - measurement_db_version = db.Column(db.String, default='1') + influx_db_version = '1' + influxdb_host = 'localhost' + + try: + from mycodo.scripts.measurement_db import get_influxdb_host + tmp_host = get_influxdb_host() + if tmp_host: + influxdb_host = tmp_host + except: + logger.exception("Determining influxdb host") + try: from mycodo.scripts.measurement_db import get_influxdb_info influx_info = get_influxdb_info() if influx_info['influxdb_installed']: if influx_info['influxdb_version'].startswith('1'): - measurement_db_version = db.Column(db.String, default='1') + influx_db_version = '1' elif influx_info['influxdb_version'].startswith('2'): - measurement_db_version = db.Column(db.String, default='2') + influx_db_version = '2' except: - logger.exception("creating influxdb options") + logger.exception("Determining influxdb version") - measurement_db_host = db.Column(db.String, default='localhost') + measurement_db_version = db.Column(db.String, default=influx_db_version) + measurement_db_host = db.Column(db.String, default=influxdb_host) measurement_db_port = db.Column(db.String, default=8086) + measurement_db_name = db.Column(db.String, default='influxdb') measurement_db_user = db.Column(db.String, default='mycodo') measurement_db_password = db.Column(db.String, default='mmdu77sj3nIoiajjs') measurement_db_dbname = db.Column(db.String, default='mycodo_db') diff --git a/mycodo/mycodo_daemon.py b/mycodo/mycodo_daemon.py index 803278a6c..55ef5276c 100755 --- a/mycodo/mycodo_daemon.py +++ b/mycodo/mycodo_daemon.py @@ -90,6 +90,9 @@ def __init__(self, debug): self.logger.info(f"Mycodo daemon v{MYCODO_VERSION} starting") + if DOCKER_CONTAINER: + self.logger.info("Detected running inside a Docker continaer") + self.log_level_debug = debug self.startup_timer = timeit.default_timer() self.startup_time = None diff --git a/mycodo/mycodo_flask/routes_page.py b/mycodo/mycodo_flask/routes_page.py index a51f6f71b..659489b32 100644 --- a/mycodo/mycodo_flask/routes_page.py +++ b/mycodo/mycodo_flask/routes_page.py @@ -26,6 +26,7 @@ from flask_babel import gettext from sqlalchemy import and_ +from mycodo.config import DOCKER_CONTAINER from mycodo.config import ALEMBIC_VERSION from mycodo.config import BACKUP_LOG_FILE from mycodo.config import CAMERA_INFO @@ -890,22 +891,24 @@ def page_logview(): log_field = form_log_view.log.data # Find which log file was requested, generate command to execute - if form_log_view.log.data == 'log_pid_settings': + if form_log_view.log.data == 'log_nginx': + if DOCKER_CONTAINER: + command = f'docker logs -n {lines} mycodo_nginx' + else: + command = f'journalctl -u nginx -n {lines} --no-pager' + elif form_log_view.log.data == 'log_flask': + if DOCKER_CONTAINER: + command = f'docker logs -n {lines} mycodo_flask' + else: + command = f'journalctl -u mycodoflask -n {lines} --no-pager' + elif form_log_view.log.data == 'log_pid_settings': logfile = DAEMON_LOG_FILE logrotate_file = logfile + '.1' if (logrotate_file and os.path.exists(logrotate_file) and logfile and os.path.isfile(logfile)): - command = 'cat {lrlog} {log} | grep -a "PID Settings" | tail -n {lines}'.format( - lrlog=logrotate_file, log=logfile, lines=lines) + command = f'cat {logrotate_file} {logfile} | grep -a "PID Settings" | tail -n {lines}' else: - command = 'grep -a "PID Settings" {log} | tail -n {lines}'.format( - lines=lines, log=logfile) - elif form_log_view.log.data == 'log_nginx': - command = 'journalctl -u nginx -n {lines} --no-pager'.format( - lines=lines) - elif form_log_view.log.data == 'log_flask': - command = 'journalctl -u mycodoflask -n {lines} --no-pager'.format( - lines=lines) + command = f'grep -a "PID Settings" {logfile} | tail -n {lines}' else: if form_log_view.log.data == 'log_login': logfile = LOGIN_LOG_FILE @@ -929,16 +932,15 @@ def page_logview(): logrotate_file = logfile + '.1' if (logrotate_file and os.path.exists(logrotate_file) and logfile and os.path.isfile(logfile)): - command = 'cat {lrlog} {log} | tail -n {lines}'.format( - lrlog=logrotate_file, log=logfile, lines=lines) + command = f'cat {logrotate_file} {logfile} | tail -n {lines}' elif os.path.isfile(logfile): - command = 'tail -n {lines} {log}'.format(lines=lines, - log=logfile) + command = f'tail -n {lines} {logfile}' # Execute command and generate the output to display to the user if command: log = subprocess.Popen( - command, stdout=subprocess.PIPE, shell=True) + command, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) (log_output, _) = log.communicate() log.wait() log_output = str(log_output, 'latin-1') diff --git a/mycodo/scripts/measurement_db.py b/mycodo/scripts/measurement_db.py index c2ba8e88a..e7dfd7fbe 100644 --- a/mycodo/scripts/measurement_db.py +++ b/mycodo/scripts/measurement_db.py @@ -2,13 +2,17 @@ """Get measurement database info""" import argparse import json +import logging import os -import requests import subprocess import sys +import requests + sys.path.append(os.path.abspath(os.path.join(__file__, "../../.."))) +logger = logging.getLogger("mycodo.measurement_db") + def get_measurement_db_info(): dict_info = { @@ -21,13 +25,37 @@ def get_measurement_db_info(): settings = db_retrieve_table_daemon(Misc, entry='first') dict_info['db_name'] = settings.measurement_db_name dict_info['db_version'] = settings.measurement_db_version - except: + except Exception as err: + logger.debug(f"Could not determine influxdb version: {err}") dict_info['db_name'] = "ERROR" dict_info['db_version'] = "ERROR" return dict_info +def get_influxdb_host(): + try: + from mycodo.databases.models import Misc + from mycodo.utils.database import db_retrieve_table_daemon + settings = db_retrieve_table_daemon(Misc, entry='first') + if settings and settings.measurement_db_host: + return settings.measurement_db_host + except Exception as err: + logger.debug(f"Could not determine influxdb host from table: {err}") + + list_hosts = [ + 'localhost', + '127.0.0.1', + 'mycodo_influxdb' + ] + for host in list_hosts: + try: + if requests.get(f"http://{host}:8086/ping").status_code == 204: + return host + except Exception as err: + logger.debug(f"Could not determine localhost as influxdb host: {err}") + def get_influxdb_info(): + settings = None dict_info = { 'influxdb_installed': None, 'influxdb_version': '' @@ -36,13 +64,23 @@ def get_influxdb_info(): from mycodo.databases.models import Misc from mycodo.utils.database import db_retrieve_table_daemon settings = db_retrieve_table_daemon(Misc, entry='first') - r = requests.get(f'http://{settings.measurement_db_host}:{settings.measurement_db_port}/ping') + except Exception as err: + logger.debug(f"Could not determine influxdb info from table: {err}") + + try: + if settings: + host = settings.measurement_db_host + port = settings.measurement_db_port + else: + host = get_influxdb_host() + port = 8086 + r = requests.get(f'http://{host}:{port}/ping') if r.headers and "X-Influxdb-Version" in r.headers: dict_info['influxdb_installed'] = True dict_info['influxdb_version'] = r.headers["X-Influxdb-Version"] - except: - pass + except Exception as err: + logger.debug(f"Could not determine influxdb info: {err}") return dict_info diff --git a/mycodo/scripts/upgrade_commands.sh b/mycodo/scripts/upgrade_commands.sh index 42eaee312..0bdf1d204 100755 --- a/mycodo/scripts/upgrade_commands.sh +++ b/mycodo/scripts/upgrade_commands.sh @@ -369,7 +369,7 @@ case "${1:-''}" in printf "\n#### Uninstalling: Stopping and disabling Mycodo services (frontend/backend)\n" service mycodoflask stop service mycodo stop - /bin/bash "${MYCODO_PATH}"/mycodo/scripts/upgrade_download.sh web-server-disable + /bin/bash "${MYCODO_PATH}"/mycodo/scripts/upgrade_commands.sh web-server-disable /bin/bash "${MYCODO_PATH}"/mycodo/scripts/upgrade_commands.sh update-mycodo-service-disable ;; 'update-pigpiod') @@ -633,8 +633,8 @@ case "${1:-''}" in ;; 'web-server-update') printf "\n#### Installing and configuring nginx web server\n" - /bin/bash "${MYCODO_PATH}"/mycodo/scripts/upgrade_download.sh web-server-disable - /bin/bash "${MYCODO_PATH}"/mycodo/scripts/upgrade_download.sh web-server-enable + /bin/bash "${MYCODO_PATH}"/mycodo/scripts/upgrade_commands.sh web-server-disable + /bin/bash "${MYCODO_PATH}"/mycodo/scripts/upgrade_commands.sh web-server-enable ;;