Merge pull request #66 from shantanoo-desai/develop

feat: QuestDB 7.x, data persistence

.github/workflows/tests.yml

@@ -38,7 +38,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Install ansible and pip dependencies
-        run: python -m pip install ansible docker passlib jsonschema
+        run: python -m pip install -r requirements.txt
 
       - name: Install ansible.utils collection
         run: ansible-galaxy collection install ansible.utils

README.md

@@ -52,13 +52,17 @@ file as that on the controller node (where Komponist is executed).
 This caveat may be mitigated if one uses a well-known directories like `/usr/share/` or `/etc/` because these paths are
 reflected similarly on the Controller as well as Remote machines
 
-### InfluxDBv2 UI rendering
+### QuestDB / InfluxDBv2 UI rendering
 
-Since InfluxDBv2's UI does not provide any configuration to render the UI behind a root path, it is not possible to render
-the UI under `/influxdbv2` path. At the moment, it is only possible to view the UI when visiting the following link:
+Since QuestDB / InfluxDBv2's UI does not provide any configuration to render the UI behind a root path, it is not possible to render
+the UI under `/influxdbv2` or `/questdb` path. At the moment, it is only possible to view the UI when visiting the following links:
 
     http://influxdbv2.localhost
 
+OR
+
+    http://questdb.localhost
+
 ## Usage
 
 Komponist relies on two core configuration files:
@@ -121,6 +125,7 @@ Currently the following services are available / planned to configure and run:
 | __InfluxDB__                       | `1.8`<br> `2.6`|
 | __TimescaleDB__                    | `v15`   |
 | __Grafana__                        | `9.5.1` |
+| __QuestDB__                        | `7.1.1` |
 
 ## Contributing / Development
 

requirements.txt

@@ -0,0 +1,5 @@
+ansible
+docker
+urllib3 < 2.0 # Pin this requirement to combat a current incompatibility with docker and urllib3
+jsonschema
+passlib

tasks/configure-questdb.yml

@@ -0,0 +1,24 @@
+# Komponist - Generate Your Favourite Compose Stack With the Least Effort
+#
+# Copyright (C) 2023  Shantanoo "Shan" Desai <sdes.softdev@gmail.com>
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as published
+#   by the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+# configure-questdb.yml: Tasks file that generates the settings and Compose service file for QuestDB
+---
+- name: '(QuestDB) Generating Compose Service File for Deployment'
+  ansible.builtin.template:
+    src: services/docker-compose.questdb.yml.j2
+    dest: "{{ komponist.deploy_dir }}/docker-compose.questdb.yml"
+    mode: "0755"

templates/config/grafana/provisioning/datasources/datasources.yml.j2

@@ -69,4 +69,23 @@ datasources:
     secureJsonData:
       password: {{ credentials.timescaledb.admin_password }}
     editable: true
+{% endif %}
+
+{% if 'questdb' in komponist.configuration.keys() %}
+  - name: Komponist-QuestDB
+    type: postgres
+    url: komponist_questdb:8812
+    user: admin
+    jsonData:
+      database: qdb
+      sslmode: 'disable'
+      maxOpenConns: 100
+      maxIdleConns: 100
+      maxIdleConnsAuto: true
+      connMaxLifetime: 14400
+      postgresVersion: 1000
+      timescaledb: true
+    secureJsonData:
+      password: quest
+    editable: true
 {% endif %}

templates/config/mosquitto/mosquitto.conf.j2

@@ -32,4 +32,9 @@ acl_file    /mosquitto_acl {# using Docker Config #}
 
 # Logging Configuration
 log_timestamp true
-log_type all
+log_type all
+
+{% if komponist.data_persistence | default(false) %}
+persistence true
+persistence_location /mosquitto/data/
+{% endif %}

templates/config/traefik/configurations/dynamic.yml.j2

@@ -43,6 +43,20 @@ http:
 # InfluxDBv2 UI router
 {% include 'routers-http/influxdbv2-ui-router.j2' %}
 {% endif %}
+
+{% if komponist.configuration.questdb.expose_http | default(false) %}
+# QuestDB HTTP API router
+{% include 'routers-http/questdb-router.j2' %}
+
+# QuestDB HTTP Health API Router
+{% include 'routers-http/questdb-health-router.j2' %}
+{% endif %}
+
+
+{% if not komponist.configuration.questdb.disable_ui | default(false) %}
+# QuestDB UI router
+{% include 'routers-http/questdb-ui-router.j2' %}
+{% endif %}
 {% endfilter %}
 
   # Middlewares
@@ -56,6 +70,9 @@ http:
 
 # Grafana Middleware
 {% include 'middlewares-http/grafana-mw.j2' %}
+
+# QuestDB Middleware
+{% include 'middlewares-http/questdb-mw.j2' %}
 {% endfilter +%}
 
   # Services
@@ -72,6 +89,12 @@ http:
 
 # InfluxDBv2 Service
 {% include 'services-http/influxdbv2-svc.j2' %}
+
+# QuestDB Service
+{% include 'services-http/questdb-svc.j2' %}
+
+# QuestDB Health Service
+{% include 'services-http/questdb-svc-health.j2'%}
 {% endfilter %}
 
 

templates/config/traefik/configurations/middlewares-http/questdb-mw.j2

@@ -0,0 +1,5 @@
+questdb-mw:
+  stripPrefix:
+    prefixes:
+      - /questdb
+    forceSlash: false

templates/config/traefik/configurations/routers-http/questdb-health-router.j2

@@ -0,0 +1,7 @@
+questdb-http-health-router:
+  entryPoints:
+    - web
+  middlewares:
+    - questdb-mw
+  service: questdb-svc-health
+  rule: PathPrefix(`/questdb/healthz`)

templates/config/traefik/configurations/routers-http/questdb-router.j2

@@ -0,0 +1,7 @@
+questdb-http-router:
+  entryPoints:
+    - web
+  middlewares:
+    - questdb-mw
+  service: questdb-svc
+  rule: PathPrefix(`/questdb`)

templates/config/traefik/configurations/routers-http/questdb-ui-router.j2

@@ -0,0 +1,5 @@
+questdb-ui-router:
+  entryPoints:
+    - web
+  service: questdb-svc
+  rule: Host(`questdb.localhost`)

templates/config/traefik/configurations/services-http/questdb-svc-health.j2

@@ -0,0 +1,4 @@
+questdb-svc-health:
+  loadBalancer:
+    servers:
+      - url: http://questdb:9003

templates/config/traefik/configurations/services-http/questdb-svc.j2

@@ -0,0 +1,4 @@
+questdb-svc:
+  loadBalancer:
+    servers:
+      - url: http://questdb:4000

templates/services/docker-compose.grafana.yml.j2

@@ -50,6 +50,13 @@ services:
       - /etc/timezone:/etc/timezone:ro
       - /etc/localtime:/etc/localtime:ro
       - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources
+{% if komponist.data_persistence | default(false) %}
+      - grafana:/var/lib/grafana
+
+volumes:
+  grafana:
+
+{% endif %}
 
 secrets:
     grafana_admin_username:

templates/services/docker-compose.influxdbv1.yml.j2

@@ -49,4 +49,11 @@ services:
       - "no-new-privileges:true"
     volumes:
       - /etc/timezone:/etc/timezone:ro
-      - /etc/localtime:/etc/localtime:ro
+      - /etc/localtime:/etc/localtime:ro
+{% if komponist.data_persistence | default(false) %}
+      - influxdbv1:/var/lib/influxdb
+
+volumes:
+  influxdbv1:
+
+{% endif %}  

templates/services/docker-compose.influxdbv2.yml.j2

@@ -55,6 +55,13 @@ services:
     volumes:
       - /etc/timezone:/etc/timezone:ro
       - /etc/localtime:/etc/localtime:ro
+{% if komponist.data_persistence | default(false) %}
+      - influxdbv2:/var/lib/influxdbv2
+
+volumes:
+  influxdbv2:
+
+{% endif %}
 
 secrets:
   influxdbv2_admin_username:

templates/services/docker-compose.mosquitto.yml.j2

@@ -25,7 +25,7 @@ services:
 {% if 'image' in komponist.configuration.mosquitto %}
     image: {{ komponist.configuration.mosquitto.image }}:{{ komponist.configuration.mosquito.version }}
 {% else %}
-    image: docker.io/eclipse-mosquitto:{{ configuration.mosquitto.version | default('latest') }}
+    image: docker.io/eclipse-mosquitto:{{ komponist.configuration.mosquitto.version | default('latest') }}
 {% endif %}
     container_name: komponist_mosquitto
     configs:
@@ -45,6 +45,13 @@ services:
     volumes:
       - /etc/timezone:/etc/timezone:ro
       - /etc/localtime:/etc/localtime:ro
+{% if komponist.data_persistence | default(false) %}
+      - mosquitto:/mosquitto/data
+
+volumes:
+  mosquitto:
+
+{% endif %}
 
 configs:
   mosquitto_conf:

templates/services/docker-compose.nodered.yml.j2

@@ -43,6 +43,13 @@ services:
     volumes:
       - /etc/timezone:/etc/timezone:ro
       - /etc/localtime:/etc/localtime:ro
+{% if komponist.data_persistence | default(false) %}
+      - nodered:/data
+
+volumes:
+  nodered:
+
+{% endif %}
 
 configs:
   nodered_settings:

templates/services/docker-compose.questdb.yml.j2

@@ -0,0 +1,44 @@
+# Komponist - Generate Your Favourite Compose Stack With the Least Effort
+#
+# Copyright (C) 2023  Shantanoo "Shan" Desai <sdes.softdev@gmail.com>
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as published
+#   by the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+{{ ansible_managed | comment }}
+
+{# docker-compose.questdb.yml.j2: Jinja2 Template for QuestDB Compose Service #}
+
+services:
+  questdb:
+{% if 'image' in komponist.configuration.questdb %}
+    image: {{ komponist.configuration.questdb.image }}:{{ komponist.configuration.questdb.version }}
+{% else %}
+    image: docker.io/questdb/questdb:{{ komponist.configuration.questdb.version }}
+{% endif %}
+    container_name: komponist_questdb
+    environment:
+      - QDB_SHARED_WORKER_COUNT=2
+      - QDB_HTTP_BIND_TO=0.0.0.0:4000 # use port 4000 to avoid conflicts with Portainer
+    security_opt:
+      - "no-new-privileges:true"
+    volumes:
+      - /etc/timezone:/etc/timezone:ro
+      - /etc/localtime:/etc/localtime:ro
+{% if komponist.data_persistence | default(false) %}
+      - questdb:/var/lib/questdb
+
+volumes:
+  questdb:
+
+{% endif %}

templates/services/docker-compose.timescaledb.yml.j2

@@ -40,6 +40,13 @@ services:
     volumes:
       - /etc/timezone:/etc/timezone:ro
       - /etc/localtime:/etc/localtime:ro
+{% if komponist.data_persistence | default(false) %}
+      - timescale:/home/postgres/pgdata/data
+
+volumes:
+  timescale:
+
+{% endif %}
 
 secrets:
   timescale_admin_username:

tests/test_generated_directories.yml

@@ -1,5 +1,11 @@
 ---
 - name: Directory Structure Creation Tests
+  vars:
+    # Services that do not have any specific configuration directories to test
+    excluded_services:
+      - influxdbv2
+      - timescaledb
+      - questdb
   vars_files:
     - "{{ playbook_dir }}/../vars/config.yml"
   hosts: localhost
@@ -32,6 +38,6 @@
         success_msg: "PASS: {{ item.item }} exist under {{ komponist.deploy_dir }} directory"
         quiet: false
       loop: "{{ test_deploy_dir_service_state.results | map('ansible.utils.keep_keys', ['item', 'exists', 'isdir', 'mode']) }}"
-      when: "item.item not in ['influxdbv2', 'timescaledb']"
+      when: "item.item not in excluded_services"
       loop_control:
         label: "{{ item.item }}"