Rewrite bash tests to python

mamonsu/lib/runner.py

@@ -224,8 +224,7 @@ def is_any_equal(array):
 #  extract pg version from input
 def define_pg_version(version_args):
     if len(version_args) < 4:
-        if version_args == "15" or version_args == "14" or version_args == "11" or version_args == "12" or version_args == "13" or version_args == "10" \
-                or version_args == "9.6" or version_args == "9.5":
+        if version_args in ["9.5", "9.6", "10", "11", "12", "13", "14", "15", "16", "17"]:
             version_number = version_args[0].split('.')
             for num in version_number:
                 if not num.isdigit():

mamonsu/plugins/pgsql/statements.py

@@ -275,13 +275,16 @@ def keys_and_queries(self, template_zabbix):
                     i + 1))
 
             if Pooler.server_version_greater("14"):
+                info_view = 'pgpro_stats_info'
                 if self.extension == "pg_stat_statements":
-                    for i, item in enumerate(self.Items_pg_14):
-                        keys = item[0].split("[")
-                        result.append(
-                            "{0}[*],$2 $1 -c \"{1}\" | awk -F  '|' '{{print ${2}}}'".format(
-                                "{0}{1}.{2}".format(self.key, keys[0], keys[1][:-1]),
-                                self.query_info.format(metrics=(item[1]), extension_schema=extension_schema),
+                    info_view = 'pg_stat_statements_info'
+                for i, item in enumerate(self.Items_pg_14):
+                    keys = item[0].split("[")
+                    result.append(
+                        "{0}[*],$2 $1 -c \"{1}\" | awk -F  '|' '{{print ${2}}}'".format(
+                            "{0}{1}.{2}".format(self.key, keys[0], keys[1][:-1]),
+                            self.query_info.format(metrics=(item[1]), extension_schema=extension_schema,
+                                                   info_view_name=info_view),
                             i + 1))
             return template_zabbix.key_and_query(result)
         else:

tests/.env

@@ -0,0 +1,29 @@
+WAIT_MAMONSU_TIMEOUT=180
+DEFAULT_HOSTGROUP="Zabbix servers"
+DEFAULT_TEMPLATE="Mamonsu PostgreSQL Linux"
+POSTGRES_VERSION=15
+
+# creds
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+POSTGRES_DB=mamonsu_test_db
+
+ZABBIX_ADMIN_USER=Admin
+ZABBIX_ADMIN_PASS=zabbix
+
+# hosts
+ZABBIX_EXT_URL=127.0.0.1:1337
+ZABBIX_INT_URL=zabbix-web:8080
+POSTGRES_EXT_HOST=127.0.0.1
+
+# external ports
+POSTGRES_EXT_PORT=15432
+MAMONSU_AGENT_EXT_PORT=11050
+ZABBIX_SERVER_EXT_PORT=11051
+ZABBIX_WEB_EXT_PORT=1337
+
+# internal ports
+POSTGRES_PORT=5432
+MAMONSU_AGENT_PORT=10050
+ZABBIX_SERVER_PORT=10051
+ZABBIX_WEB_PORT=8080

tests/README.md

@@ -0,0 +1,32 @@
+
+# Mamonsu autotests
+
+Mamonsu testing with different Postgres version, different operation systems(not supported yet). Uses docker-compose to run all services.
+
+
+## Installation
+
+
+```bash
+  pip3 install -e requirement.txt
+```
+
+## Usage/Examples
+
+You can simly run tests with only pytest mark "bash" and it will be ran with Postgres version from env variable POSTGRES_VERSION which is specified in .env file
+
+```bash
+pytest -m bash
+```
+
+You can run tests with different Postgres versions with  POSTGRES_VERSIONS variable
+
+```bash
+POSTGRES_VERSIONS=12,13 pytest -m bash
+```
+
+To run specific test you have to use -k flag with function name
+
+```bash
+POSTGRES_VERSIONS=12,13 pytest -k test_export_zabbix_params
+```

tests/config/config.py

@@ -0,0 +1,29 @@
+import os
+from pathlib import Path
+from typing import Any
+
+from dotenv import load_dotenv
+
+
+class Config:
+    def __init__(self, env_path: Path | None = None):
+        self._root_path = Path(__file__).parent.parent
+        load_dotenv(env_path or self._root_path / ".env")
+
+    def __getattr__(self, name: str) -> Any:
+        value = os.getenv(name)
+        if value is None:
+            return None
+        return self._convert_value(value)
+
+    @staticmethod
+    def _convert_value(value: str) -> Any:
+        if value.lower() in ("true", "false"):
+            return value.lower() == "true"
+        try:
+            return int(value)
+        except ValueError:
+            try:
+                return float(value)
+            except ValueError:
+                return value

tests/config/constants/containers.py

@@ -0,0 +1,8 @@
+from enum import StrEnum
+
+
+class ContainersEnum(StrEnum):
+    POSTGRES = "mamonsu-pg"
+    MAMONSU = "mamonsu-pg"
+    ZABBIX_WEB = "zabbix-web"
+    ZABBIX_SERVER = "zabbix-server"

tests/debian.Dockerfile

@@ -0,0 +1,44 @@
+# Стейдж нужен для того, что бы потом скопировать из него энтрипоинт нужной версии, т.к. в --from= нельзя использовать env
+ARG POSTGRES_VERSION=15
+FROM postgres:${POSTGRES_VERSION} AS postgres_base
+
+FROM debian:bookworm-slim AS builder
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    curl \
+    software-properties-common \
+    make \
+    dpkg-dev \
+    debhelper \
+    build-essential \
+    python3-dev \
+    python3-setuptools && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+COPY . /app
+RUN make deb
+
+FROM postgres:${POSTGRES_VERSION}
+
+COPY --from=builder /app/mamonsu*.deb /tmp/
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    python3 \
+    python3-setuptools \
+    sudo \
+    && rm -rf /var/lib/apt/lists/*
+RUN dpkg -i /tmp/mamonsu*.deb || apt-get install -f -y && \
+    rm /tmp/mamonsu*.deb
+RUN mkdir -p /var/log/mamonsu && \
+    chown postgres:postgres /var/log/mamonsu && \
+    chmod 755 /var/log/mamonsu
+
+COPY --from=postgres_base /usr/local/bin/docker-entrypoint.sh /usr/local/bin/
+COPY ./tests/service-scripts/mamonsu-pg/mamonsu.conf /etc/mamonsu/agent.conf
+COPY ./tests/service-scripts/mamonsu-pg/entrypoint.sh ./tests/service-scripts/mamonsu-pg/init_mamonsu_in_zbx.sh /app/
+
+RUN chmod +x /app/entrypoint.sh /app/init_mamonsu_in_zbx.sh
+
+ENTRYPOINT ["/app/entrypoint.sh"]

tests/docker-compose.yaml

@@ -0,0 +1,68 @@
+services:
+  mamonsu-pg:
+    build:
+      context: .
+      dockerfile: tests/debian.Dockerfile
+      args:
+        POSTGRES_VERSION: ${POSTGRES_VERSION}
+    container_name: mamonsu-pg
+    hostname: mamonsu-pg
+    image: mamonsu-pg
+    ports:
+      - "${MAMONSU_AGENT_EXT_PORT}:${MAMONSU_AGENT_PORT}"
+      - "${POSTGRES_EXT_PORT}:${POSTGRES_PORT}"
+    environment:
+      POSTGRES_VERSION: ${POSTGRES_VERSION}
+      MAMONSU_AGENT_PORT: ${MAMONSU_AGENT_PORT}
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_DB: postgres
+      POSTGRES_HOST_AUTH_METHOD: md5
+      ZABBIX_USER: ${ZABBIX_ADMIN_USER}
+      ZABBIX_PASSWD: ${ZABBIX_ADMIN_PASS}
+      ZABBIX_URL: http://${ZABBIX_INT_URL}/
+    restart: no
+
+  zabbix:
+    image: zabbix/zabbix-server-pgsql:6.4.13-ubuntu
+    container_name: zabbix
+    hostname: zabbix
+    environment:
+      - DB_SERVER_HOST=mamonsu-pg
+      - POSTGRES_USER=${POSTGRES_USER}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - POSTGRES_DB=zabbix
+      - PGPASSWORD=${POSTGRES_PASSWORD}
+    ports:
+      - "${ZABBIX_SERVER_EXT_PORT}:${ZABBIX_SERVER_PORT}"
+    depends_on:
+      - mamonsu-pg
+
+  zabbix-web:
+    image: zabbix/zabbix-web-nginx-pgsql:6.4.13-ubuntu
+    container_name: zabbix-web
+    hostname: zabbix-web
+    environment:
+      - DB_SERVER_HOST=mamonsu-pg
+      - POSTGRES_USER=${POSTGRES_USER}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - POSTGRES_DB=zabbix
+      - ZBX_SERVER_HOST=zabbix-server
+      - ZBX_SERVER_PORT=${ZABBIX_SERVER_PORT}
+      - ZABBIX_ADMIN_USER=Admin
+      - ZABBIX_ADMIN_PASS=zabbix
+    ports:
+      - "${ZABBIX_WEB_EXT_PORT}:${ZABBIX_WEB_PORT}"
+    depends_on:
+      - zabbix
+    healthcheck:
+      test: |
+        curl -fsS "http://localhost:${ZABBIX_WEB_PORT}/api_jsonrpc.php" \
+          -X POST \
+          -H "Content-Type: application/json-rpc" \
+          -d '{"jsonrpc":"2.0","method":"apiinfo.version","id":1,"auth":null,"params":{}}' \
+        | grep -q '"result"' || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 15
+      start_period: 30s

tests/pytest.ini

@@ -0,0 +1,8 @@
+
+[pytest]
+log_cli=true
+log_level=INFO
+log_format = %(asctime)s %(levelname)s %(message)s
+log_date_format = %Y-%m-%d %H:%M:%S
+markers =
+    bash

tests/requirements.txt

@@ -0,0 +1,5 @@
+pytest==8.3.5
+docker==7.1.0
+zabbix==1.3.1
+python-dotenv==1.1.0
+psycopg2==2.9.10

tests/service-scripts/mamonsu-pg/entrypoint.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+set -e
+
+RECOVERY_FILE="standby.signal"
+
+DATA_DIR=/var/lib/postgresql/data
+DATA_SLAVE_PHYSICAL_DIR=/var/lib/postgresql/data_slave_physical
+WAL_DIR=/var/lib/postgresql/wals
+DATA_SLAVE_LOGICAL_DIR=/var/lib/postgresql/data_slave_logical
+
+su postgres -c '/usr/local/bin/docker-entrypoint.sh postgres "$@" &'
+sleep 5
+su postgres -c "pg_ctl stop -D $DATA_DIR"
+
+sudo mkdir -p $DATA_SLAVE_PHYSICAL_DIR
+sudo mkdir -p $WAL_DIR
+sudo chown -R postgres:postgres $DATA_SLAVE_PHYSICAL_DIR $WAL_DIR
+sudo chmod 700 $DATA_SLAVE_PHYSICAL_DIR
+
+sudo -u postgres echo "shared_preload_libraries = 'pg_stat_statements'" >> $DATA_DIR/postgresql.conf
+sudo -u postgres echo "pg_stat_statements.track = all" >> $DATA_DIR/postgresql.conf
+sudo -u postgres echo "archive_mode=on" >> $DATA_DIR/postgresql.conf
+sudo -u postgres echo "archive_command='cp %p $WAL_DIR/%f'" >> $DATA_DIR/postgresql.conf
+sudo -u postgres echo "wal_level=replica" >> $DATA_DIR/postgresql.conf
+sudo -u postgres echo "max_wal_senders=4" >> $DATA_DIR/postgresql.conf
+sudo -u postgres echo "hot_standby=on" >> $DATA_DIR/postgresql.conf
+
+sudo -u postgres echo "track_io_timing = on" >> $DATA_DIR/postgresql.conf
+sudo -u postgres echo "track_functions = all" >> $DATA_DIR/postgresql.conf
+
+sudo -u postgres echo "host    replication     replicator      127.0.0.1/0             trust" >> $DATA_DIR/pg_hba.conf
+
+su postgres -c "pg_ctl start -D $DATA_DIR"
+sleep 3
+
+sudo -u postgres psql -c "CREATE DATABASE mamonsu_test_db;"
+sudo -u postgres psql -d mamonsu_test_db -c "CREATE EXTENSION pg_stat_statements;"
+sudo -u postgres psql -d mamonsu_test_db -c "CREATE EXTENSION pg_buffercache;"
+sudo -u postgres psql -d mamonsu_test_db -c "CREATE TABLE mamonsu_test_table(id serial, value integer);"
+sudo -u postgres psql -d mamonsu_test_db -c "INSERT INTO mamonsu_test_table(value) SELECT * FROM generate_series(1, 10000);"
+sudo -u postgres psql -c "CREATE USER replicator WITH REPLICATION ENCRYPTED PASSWORD 'secret';"
+sudo -u postgres pg_basebackup -h 127.0.0.1 -U replicator -Fp -Xs -P -R -D $DATA_SLAVE_PHYSICAL_DIR/
+sudo -u postgres sed -i '/^archive_mode/s/^\(.*\)$/#\1/' $DATA_SLAVE_PHYSICAL_DIR/postgresql.conf
+sudo -u postgres sed -i '/^archive_command/s/^\(.*\)$/#\1/' $DATA_SLAVE_PHYSICAL_DIR/postgresql.conf
+sudo -u postgres echo "port=5433" >> $DATA_SLAVE_PHYSICAL_DIR/postgresql.conf
+sudo -u postgres echo "restore_command = 'cp $WAL_DIR/%f %p'" >> $DATA_SLAVE_PHYSICAL_DIR/${RECOVERY_FILE}
+
+su postgres -c "pg_ctl start -D $DATA_SLAVE_PHYSICAL_DIR"
+
+# create logical slave
+if [ "$POSTGRES_VERSION" -ge 100 ]; then  # TODO: Пофиксить, пока что отключено
+    # create PGDATA directory
+    sudo mkdir -p $DATA_SLAVE_LOGICAL_DIR
+    sudo chown postgres:postgres $DATA_SLAVE_LOGICAL_DIR
+    sudo chmod 700 $DATA_SLAVE_LOGICAL_DIR
+
+    sudo -u postgres sed -i '/^wal_level/s/^\(.*\)$/#\1/' $DATA_DIR/postgresql.conf
+    sudo -u postgres echo "wal_level=logical" >> $DATA_DIR/postgresql.conf
+    su postgres -c "pg_ctl restart -D $DATA_DIR"
+    sleep 3
+    sudo -u postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE mamonsu_test_db TO replicator;"
+    sudo -u postgres psql -d mamonsu_test_db -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO replicator;"
+    sudo -u postgres psql -d mamonsu_test_db -c "CREATE PUBLICATION mamonsu_publication;"
+    sudo -u postgres psql -d mamonsu_test_db -c "ALTER PUBLICATION mamonsu_publication ADD TABLE mamonsu_test_table;"
+    sudo -u postgres echo "host    all             all             127.0.0.1/0             trust" >> $DATA_SLAVE_LOGICAL_DIR/pg_hba.conf
+    sudo -u postgres echo "port=5434" >> $DATA_SLAVE_LOGICAL_DIR/postgresql.conf
+    su postgres -c "pg_ctl start -D $DATA_SLAVE_LOGICAL_DIR"
+    sleep 3
+    sudo -u postgres psql -p 5434 -c "CREATE DATABASE mamonsu_test_db;"
+    sudo -u postgres psql -p 5434 -d mamonsu_test_db -c "CREATE TABLE mamonsu_test_table(id serial, value integer);"
+    sudo -u postgres psql -p 5434 -d mamonsu_test_db -c "CREATE SUBSCRIPTION mamonsu_subscription CONNECTION 'host=127.0.0.1 port=5432 user=replicator dbname=mamonsu_test_db' PUBLICATION mamonsu_publication;"
+fi
+
+mamonsu bootstrap -x --user postgres -d mamonsu_test_db
+service mamonsu restart
+
+tail -f /dev/null

tests/service-scripts/mamonsu-pg/init_mamonsu_in_zbx.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+INIT_MARKER="/app/.init_done"
+if [ ! -f "$INIT_MARKER" ]; then
+  echo "[INFO] Exporting templates"
+  mamonsu export template template.xml
+  mamonsu zabbix template export template.xml
+
+  echo "[INFO] Adding host in Zabbix"
+  mamonsu zabbix host create "$(hostname)" \
+    "$(mamonsu zabbix hostgroup id "Zabbix servers")" \
+    "$(mamonsu zabbix template id "Mamonsu PostgreSQL Linux")" \
+    "$(getent hosts "$(hostname)" | awk '{print $1}')"
+  service mamonsu start
+
+  echo "[INFO] Waiting for host to appear in Zabbix"
+  sleep 5
+  touch "$INIT_MARKER"
+else
+  echo "[INFO] Initialization already done. Skipping Mamonsu setup"
+fi
+