Add Zabbix 7 compatibility, rewrite API code (#81)

* Add Zabbix 7.0 compatibility (#79)

* Add Zabbix 7.0 compatibility

* Add Host model comments

* Add missing type annotations to __init__ .py

* Fix state manager mypy issues

* Refactor failsafe OK file checking, add test

* Add type annotations to __init__

* Refactor process initialization

* Refactor hanging process handling

* Ignore missing mypy stubs in import

* Extract failsafe functions, add tests

* Fix zabbix_tags2zac_tags, add types

* Fix StateManager mypy stub hack

* Add type annotations for all processing.py methods

* Improve state.State comments+docstrings

* Refactor host modifier/source collector loading

* Refactor failsafe checking

Moves everything into failsafe.py module.

This allows us to test the failsafe checking more thoroughly.

* Fix incorrect variable usage

* Refactor DB host retrieval in `ZabbixUpdater`

* Fix broken tests

* Add check_failsafe tests

* Fix incorrect variable name

* Rewrite API internals with Pydantic (#82)

* Remove disabled hosts from maintenance

* Add periodic maintenance cleanup

* Add map_dir fixtures

* Add config options

* Fix mocks, use fixture

* Rewrite API internals with Pydantic

* Fix tests

* Fix and improve JSON serialization

* Fix changelog headers

* Add API param building functions

* Fix `set_hostgroups` not being able to remove groups

* Add read-only mode for ZabbixAPI

Activated during dryruns.

* Fix `ParamsType` docstring

* Document new config options in changelog

* Update changelog

* Add Py3.12 trove classifier

* Update sample config

* Fix ZabbixAPI method docstring tense

* README: update supported versions

* Create required host groups on startup

* README: fix JSON example

* README: Make host modifier example more relevant

* Update changelog

* Add notes on running source collectors standalone

* Warn if no proxies

* Remove redundant bool cast

* Use absolute import

* Use absolute imports

* Sort host groups when logging new and old

* Add note regarding Source Handler update interval

* Change "replaced" to "updated" for source hosts

* Remove trigger support in GC

* Remove validation of request params

* Add support for mysterious host.status==3

* Fix missing assignments in SignalHandler.__init__

* Fix missing parameter type annotation

* Move warning next to statement that caused it

* Add py.typed marker file

* Update README, run GC every 24h

* Show data in request errors

* Fix fetching templates via old API code

* Remove urllib3 logger, set logger on httpcore

* Fetch groups when fetching hosts

* Make bulk an optional field for CreateHostInterfaceDetails

* Remove duplicated code for host interfaces

* Refactor `set_interface`

* Fix setting proxies on Zabbix 7

* Comments, var name

* Update host group map when creating host group

* Changelog heading

* Fix README grammar

* Log names of templates

* Add NOTE comment

* Add configurable group prefix separator

CHANGELOG.md

@@ -0,0 +1,58 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+<!-- ## [Unreleased] -->
+
+## [0.2.0](https://github.com/unioslo/zabbix-auto-config/releases/tag/zac-v0.2.0)
+
+### Added
+
+- Zabbix 7 compatibility
+- Configuration option for setting group prefix separator.
+  - `[zabbix]`
+    - `prefix_separator`: Separator for group prefixes. Default is `-`.
+- Configuration options for each process.
+  - `[zac.process.garbage_collector]`
+    - `enabled`: Enable automatic garbage collection.
+    - `delete_empty_maintenance`: Delete maintenances that only contain disabled hosts.
+    - `update_interval`: Update interval in seconds.
+  - `[zac.process.host_updater]`
+    - `update_interval`: Update interval in seconds.
+  - `[zac.process.hostgroup_updater]`
+    - `update_interval`: Update interval in seconds.
+  - `[zac.process.template_updater]`
+    - `update_interval`: Update interval in seconds.
+  - `[zac.process.source_merger]`
+    - `update_interval`: Update interval in seconds.
+- Automatic garbage collection of maintenances (and more in the future.)
+  - Removes disabled hosts from maintenances.
+  - This feature is disabled by default, and must be opted into with `zac.process.garbage_collector.enabled`
+  - Optionally also delete maintenances that only contain disabled hosts with `zac.process.garbage_collector.delete_empty_maintenance`.
+  - If you have a large number of disabled hosts, it's recommended to set a long `update_interval` to avoid unnecessary load on the Zabbix server. The default is 300 seconds.
+- Automatic creation of required host groups.
+  - Creates the groups configured by the following options:
+    - `zabbix.hostgroup_all`
+    - `zabbix.hostgroup_disabled`
+- Utility functions for serializing source collector outputs:
+  - `zabbix_auto_config.models.hosts_to_json`
+  - `zabbix_auto_config.models.print_hosts`
+- `py.typed` marker file.
+
+### Changed
+
+- API internals rewritten to use Pydantic models.
+  - Borrows API code from Zabbix-cli v3.
+- Dry run mode now guarantees no changes are made to Zabbix by preventing all write operations via the API.
+
+### Deprecated
+
+- Zabbix 5 support.
+  - Should in most cases work with Zabbix 5, but it will not be actively supported going forward.
+
+## 0.1.0
+
+First version

README.md

@@ -2,13 +2,13 @@
 
 Zabbix-auto-config is an utility that aims to automatically configure hosts, host groups, host inventories, template groups and templates in the monitoring software [Zabbix](https://www.zabbix.com/).
 
-Note: Only tested with Zabbix 6.0 and 6.4.
+Note: Primarily tested with Zabbix 7.0 and 6.4, but should work with 6.0 and 5.2.
 
 ## Requirements
 
 * Python >=3.8
 * pip >=21.3
-* Zabbix >=5.0
+* Zabbix >=6.4
 
 # Quick start
 
@@ -19,23 +19,35 @@ This is a crash course in how to quickly get this application up and running in
 Setup a Zabbix test instance with [podman](https://podman.io/) and [podman-compose](https://github.com/containers/podman-compose/).
 
 ```bash
-TAG=alpine-5.0-latest ZABBIX_PASSWORD=secret podman-compose up -d
+TAG=7.0-ubuntu-latest ZABBIX_PASSWORD=secret podman-compose up -d
 ```
 
 ## Zabbix prerequisites
 
-It is currently assumed that you have the following hostgroups in Zabbix. You should logon to Zabbix and create them:
+The following host groups are created in Zabbix if they do not exist:
 
 * All-auto-disabled-hosts
 * All-hosts
 
+The name of these groups can be configured in `config.toml`:
+
+```toml
+[zabbix]
+hostgroup_all = "All-hosts"
+hostgroup_disabled = "All-auto-disabled-hosts"
+```
+
+These groups contain enabled and disabled hosts respectively.
+
 For automatic linking in templates you could create the templates:
 
 * Template-barry
 * Template-pizza
 
 ## Database
 
+The application requires a PostgreSQL database to store the state of the collected hosts. The database can be created with the following command:
+
 ```bash
 PGPASSWORD=secret psql -h localhost -U postgres -p 5432 -U zabbix << EOF
 CREATE DATABASE zac;
@@ -49,11 +61,13 @@ CREATE TABLE hosts_source (
 EOF
 ```
 
+Replace login credentials with your own when running against a different database. This is a one-time procedure per environment.
+
 ## Application
 
-### Installation (production)
+### Installation
 
-For production, installing the project in a virtual environment directly with pip is the recommended way to go:
+Installing the project in a virtual environment directly with pip is the recommended way to go:
 
 ```bash
 python -m venv venv
@@ -100,8 +114,9 @@ def collect(*args: Any, **kwargs: Any) -> List[Host]:
 
 
 if __name__ == "__main__":
-    for host in collect():
-        print(host.model_dump_json())
+    # Print hosts as a JSON array when running standalone
+    from zabbix_auto_config.models import print_hosts
+    print_hosts(collect())
 EOF
 cat > path/to/host_modifier_dir/mod.py << EOF
 from zabbix_auto_config.models import Host
@@ -133,7 +148,7 @@ zac
 
 ## Systemd unit
 
-You could run this as a systemd service:
+To add automatic startup of the application with systemd, create a unit file in `/etc/systemd/system/zabbix-auto-config.service`:
 
 ```ini
 [Unit]
@@ -147,34 +162,64 @@ WorkingDirectory=/home/zabbix/zabbix-auto-config
 Environment=PATH=/home/zabbix/zabbix-auto-config/venv/bin
 ExecStart=/home/zabbix/zabbix-auto-config/venv/bin/zac
 TimeoutSec=300
+Restart=always
+RestartSec=5s
 
 [Install]
 WantedBy=multi-user.target
 ```
 
 ## Source collectors
 
+ZAC relies on "Source Collectors" to fetch host data from various sources.
+A source can be anything: an API, a file, a database, etc. What matters is that
+the source is able to return a list of `zabbix_auto_config.models.Host` objects. ZAC uses these objects to create or update hosts in Zabbix. If a host with the same hostname is collected from multiple different sources, its information is combined into a single logical host object before being used to create/update the host in Zabbix.
+
+### Writing a source collector
+
 Source collectors are Python modules placed in a directory specified by the `source_collector_dir` option in the `[zac]` table of the configuration file. Zabbix-auto-config attempts to load all modules referenced by name in the configuration file from this directory. If any referenced modules cannot be found in the directory, they will be ignored.
 
-A source collector module contains a function named `collect` that returns a list of `Host` objects. These host objects are used by Zabbix-auto-config to create or update hosts in Zabbix.
+A source collector module contains a function named `collect()` that returns a list of `Host` objects. These host objects are used by Zabbix-auto-config to create or update hosts in Zabbix.
 
 Here's an example of a source collector module that reads hosts from a file:
 
 ```python
 # path/to/source_collector_dir/load_from_json.py
 
+import json
 from typing import Any, Dict, List
+
 from zabbix_auto_config.models import Host
 
 DEFAULT_FILE = "hosts.json"
 
 def collect(*args: Any, **kwargs: Any) -> List[Host]:
     filename = kwargs.get("filename", DEFAULT_FILE)
     with open(filename, "r") as f:
-        return [Host(**host) for host in f.read()]
+        return [Host(**host) for host in json.load(f)]
 ```
 
-A module is recognized as a source collector if it contains a `collect` function that accepts an arbitrary number of arguments and keyword arguments and returns a list of `Host` objects. Type annotations are optional but recommended.
+A module is recognized as a source collector if it contains a `collect()` function that accepts an arbitrary number of arguments and keyword arguments and returns a list of `Host` objects. Type annotations are optional but recommended.
+
+We can also provide a `if __name__ == "__main__"` block to run the collector standalone. This is useful for testing the collector module without running the entire application.
+
+```py
+if __name__ == "__main__":
+    # Print hosts as a JSON array when running standalone
+    from zabbix_auto_config.models import print_hosts
+    print_hosts(collect())
+```
+
+If you wish to collect just the JSON output and write it to a file or otherwise manipulate it, you can import the `hosts_to_json` function from `zabbix_auto_config.models` and use it like this:
+
+```py
+if __name__ == "__main__":
+    from zabbix_auto_config.models import hosts_to_json
+    with open("output.json", "w") as f:
+        f.write(hosts_to_json(collect()))
+```
+
+### Configuration
 
 The configuration entry for loading a source collector module, like the `load_from_json.py` module above, includes both mandatory and optional fields. Here's how it can be configured:
 
@@ -186,9 +231,12 @@ error_tolerance = 5
 error_duration = 360
 exit_on_error = false
 disable_duration = 3600
+# Extra keyword arguments to pass to the collect function:
 filename = "hosts.json"
 ```
 
+Only the extra `filename` option is passed in as a kwarg to the `collect()` function.
+
 The following configurations options are available:
 
 ### Mandatory configuration
@@ -197,13 +245,12 @@ The following configurations options are available:
 `module_name` is the name of the module to load. This is the name that will be used in the configuration file to reference the module. It must correspond with the name of the module file, without the `.py` extension.
 
 #### update_interval
-`update_interval` is the number of seconds between updates. This is the interval at which the `collect` function will be called.
+`update_interval` is the number of seconds between updates. This is the interval at which the `collect()` function will be called.
 
 ### Optional configuration (error handling)
 
 If `error_tolerance` number of errors occur within `error_duration` seconds, the collector is disabled. Source collectors do not tolerate errors by default and must opt-in to this behavior by setting `error_tolerance` and `error_duration` to non-zero values. If `exit_on_error` is set to `true`, the application will exit. Otherwise, the collector will be disabled for `disable_duration` seconds.
 
-
 #### error_tolerance
 
 `error_tolerance` (default: 0) is the maximum number of errors tolerated within `error_duration` seconds.
@@ -226,13 +273,16 @@ A useful guide is to set `error_duration` as `(error_tolerance + 1) * update_int
 
 ### Keyword arguments
 
-Any extra config options specified in the configuration file will be passed to the `collect` function as keyword arguments. In the example above, the `filename` option is passed to the `collect` function, and then accessed via `kwargs["filename"]`.
+Any extra config options specified in the configuration file will be passed to the `collect()` function as keyword arguments. In the example above, the `filename` option is passed to the `collect()` function, and then accessed via `kwargs["filename"]`.
 
 
 ## Host modifiers
 
 Host modifiers are Python modules (files) that are placed in a directory defined by the option `host_modifier_dir` in the `[zac]` table of the config file. A host modifier is a module that contains a function named `modify` that takes a `Host` object as its only argument, modifies it, and returns it. Zabbix-auto-config will attempt to load all modules in the given directory.
 
+
+### Writing a host modifier
+
 A host modifier module that adds a given siteadmin to all hosts could look like this:
 
 ```py
@@ -243,7 +293,8 @@ from zabbix_auto_config.models import Host
 SITEADMIN = "admin@example.com"
 
 def modify(host: Host) -> Host:
-    host.siteadmins.add(SITEADMIN)
+    if host.hostname.endswith(".example.com"):
+        host.siteadmins.add(SITEADMIN)
     return host
 ```
 
@@ -259,6 +310,34 @@ Zac manages only inventory properties configured as `managed_inventory` in `conf
 2. Remove the "location" property from the host in the source
 3. "location=x" will remain in Zabbix
 
+## Garbage Collection
+
+ZAC provides an optional Zabbix garbage collection module that cleans up stale data from Zabbix that is not otherwise managed by ZAC, such as maintenances.
+
+The garbage collector currently does the following:
+
+- Removes disabled hosts from maintenances.
+- Deletes maintenances that only contain disabled hosts.
+
+Under normal usage, hosts are removed from maintenances when being disabled by ZAC, but if hosts are disabled outside of ZAC, they will not be removed from maintenances. The GC module will remove these hosts, and optionally delete the maintenance altogether if it only contains disabled hosts.
+
+To enable garbage collection, add the following to your config:
+
+```toml
+[zac.process.garbage_collector]
+enabled = true
+delete_empty_maintenance = true
+```
+
+By default, the garbage collector runs every 24 hours. This can be adjusted with the `update_interval` option:
+
+```toml
+[zac.process.garbage_collector]
+update_interval = 3600 # Run every hour
+```
+
+----
+
 ## Development
 
 We use the project management tool [Hatch](https://hatch.pypa.io/latest/) for developing the project. The tool manages virtual environment creation, dependency installation, as well as building and publishing of the project, and more.

config.sample.toml

@@ -27,6 +27,29 @@ failsafe_ok_file = "/tmp/zac_failsafe_ok"
 # It is then up to the administrator to manually delete the file afterwards.
 failsafe_ok_file_strict = true
 
+# Configuration for ZAC processes.
+[zac.process.source_merger]
+# How often to run the source merger in seconds
+update_interval = 60
+
+[zac.process.host_updater]
+update_interval = 60
+
+[zac.process.hostgroup_updater]
+update_interval = 60
+
+[zac.process.template_updater]
+update_interval = 60
+
+[zac.process.garbage_collector]
+# Enable garbage collection, including:
+#   - Remove disabled hosts from maintenances
+enabled = false
+# Delete maintenances if all its hosts are disabled
+delete_empty_maintenance = false
+update_interval = 86400 # every 24 hours
+
+
 [zabbix]
 # Directory containing mapping files.
 map_dir = "path/to/map_dir/"
@@ -36,6 +59,7 @@ username = "Admin"
 password = "zabbix"
 
 # Preview changes without making them.
+# Disables all write operations to Zabbix.
 dryrun = true
 
 # Maximum number of hosts to add/remove in one go.
@@ -56,10 +80,15 @@ hostgroup_source_prefix = "Source-"
 hostgroup_importance_prefix = "Importance-"
 
 # Template group creation
+# If we have a host group named `Siteadmin-my-hosts`, ZAC creates a
+# template group named `Templates-my-hosts`
 # NOTE: will create host groups if enabled on Zabbix <6.2
 create_templategroups = true
 templategroup_prefix = "Templates-"
 
+# Separator used for group name prefixes
+prefix_separator = "-"
+
 extra_siteadmin_hostgroup_prefixes = []
 
 [source_collectors.mysource]

pyproject.toml

@@ -23,15 +23,16 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
 ]
 dependencies = [
-    "multiprocessing-logging==0.3.1",
+    "multiprocessing-logging>=0.3.1",
     "psycopg2>=2.9.5",
     "pydantic>=2.6.0",
-    "pyzabbix>=1.3.0",
-    "requests>=1.0.0",
+    "httpx>=0.27.0",
     "tomli>=2.0.0",
     "packaging>=23.2",
+    "typing_extensions>=4.12.0",
 ]
 
 [project.optional-dependencies]
@@ -65,6 +66,7 @@ extend-select = [
     "LOG", # flake8-logging
     "PLE1205", # pylint (too many logging args)
     "PLE1206", # pylint (too few logging args)
+    "TID252", # flake8-tidy-imports (prefer absolute imports)
 ]
 
 [tool.ruff.lint.isort]