feat(BA-593): Configure the logging module's config file to use Pydantic

changes/2834.feature.md

@@ -0,0 +1 @@
+Configure the logging module's config file to use Pydantic

configs/wsproxy/halfstack.toml

@@ -1,12 +1,12 @@
 [wsproxy]
-bind_host = "0.0.0.0"
-advertised_host = "127.0.0.1"
+bind-host = "0.0.0.0"
+advertised-host = "127.0.0.1"
 
-bind_api_port = 5050
-advertised_api_port = 5050
+bind-api-port = 5050
+advertised-api-port = 5050
 
 # replace these values with your passphrase
-jwt_encrypt_key = "QRX/ZX2nwKKpuTSD3ZycPA"
-permit_hash_key = "gHNAohmntRV0t9zlwTVQeQ"
+jwt-encrypt-key = "QRX/ZX2nwKKpuTSD3ZycPA"
+permit-hash-key = "gHNAohmntRV0t9zlwTVQeQ"
 
-api_secret = "v625xZLOgbMHhl0s49VuqQ"
+api-secret = "v625xZLOgbMHhl0s49VuqQ"

configs/wsproxy/sample.toml

@@ -1,39 +1,39 @@
 [wsproxy]
-ipc_base_path = "/tmp/backend.ai/ipc"
-event_loop = "asyncio"
-pid_file = "/run/backend.ai/wsproxy/wsproxy.pid"
+ipc-base-path = "/tmp/backend.ai/ipc"
+event-loop = "asyncio"
+pid-file = "/run/backend.ai/wsproxy/wsproxy.pid"
 id = "i-node01"
 user = 501
 group = 501
-bind_host = "0.0.0.0"
-advertised_host = "example.com"
-bind_api_port = 5050
-advertised_api_port = 15050
-bind_proxy_port_range = [
+bind-host = "0.0.0.0"
+advertised-host = "example.com"
+bind-api-port = 5050
+advertised-api-port = 15050
+bind-proxy-port-range = [
     10200,
     10300,
 ]
-advertised_proxy_port_range = [
+advertised-proxy-port-range = [
     20200,
     20300,
 ]
 protocol = "http"
 
 # replace these values with your passphrase
-jwt_encrypt_key = "50M3G00DL00KING53CR3T"
-permit_hash_key = "50M3G00DL00KING53CR3T"
-api_secret = "50M3G00DL00KING53CR3T"
+jwt-encrypt-key = "50M3G00DL00KING53CR3T"
+permit-hash-key = "50M3G00DL00KING53CR3T"
+api-secret = "50M3G00DL00KING53CR3T"
 
-aiomonitor_termui_port = 48500
-aiomonitor_webui_port = 49500
+aiomonitor-termui-port = 48500
+aiomonitor-webui-port = 49500
 
 [logging]
 level = "INFO"
 drivers = [
     "console",
 ]
 
-[logging.pkg_ns]
+[logging.pkg-ns]
 "" = "WARNING"
 "ai.backend" = "DEBUG"
 tests = "DEBUG"
@@ -46,14 +46,14 @@ format = "verbose"
 [logging.file]
 path = "/var/log/backend.ai"
 filename = "wsproxy.log"
-backup_count = 5
-rotation_size = "10M"
+backup-count = 5
+rotation-size = "10M"
 format = "verbose"
 
 [logging.logstash]
 protocol = "tcp"
-ssl_enabled = true
-ssl_verify = true
+ssl-enabled = true
+ssl-verify = true
 
 [logging.logstash.endpoint]
 host = "127.0.0.1"
@@ -63,14 +63,14 @@ port = 8001
 host = "127.0.0.1"
 port = 8000
 level = "INFO"
-ssl_verify = true
-ca_certs = "/etc/ssl/ca.pem"
+ssl-verify = true
+ca-certs = "/etc/ssl/ca.pem"
 keyfile = "/etc/backend.ai/graylog/privkey.pem"
 certfile = "/etc/backend.ai/graylog/cert.pem"
 
 [debug]
 enabled = false
 asyncio = false
-enhanced_aiomonitor_task_info = false
-log_events = false
+enhanced-aiomonitor-task-info = false
+log-events = false
 

src/ai/backend/logging/config_pydantic.py

@@ -0,0 +1,173 @@
+import enum
+from pathlib import Path
+from typing import Annotated
+
+from pydantic import BaseModel, ByteSize, ConfigDict, Field
+
+from .types import LogFormat, LogLevel
+
+default_pkg_ns = {"": "WARNING", "ai.backend": "DEBUG", "tests": "DEBUG", "aiohttp": "INFO"}
+
+
+class LogDriver(str, enum.Enum):
+    CONSOLE = "console"
+    LOGSTASH = "logstash"
+    FILE = "file"
+    GRAYLOG = "graylog"
+
+
+class LogstashProtocol(str, enum.Enum):
+    ZMQ_PUSH = "zmq.push"
+    ZMQ_PUB = "zmq.pub"
+    TCP = "tcp"
+    UDP = "udp"
+
+
+class BaseSchema(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+        from_attributes=True,
+        use_enum_values=True,
+    )
+
+
+class HostPortPair(BaseSchema):
+    host: Annotated[str, Field(examples=["127.0.0.1"])]
+    port: Annotated[int, Field(gt=0, lt=65536, examples=[8201])]
+
+    def __repr__(self) -> str:
+        return f"{self.host}:{self.port}"
+
+    def __str__(self) -> str:
+        return self.__repr__()
+
+    def __getitem__(self, *args) -> int | str:
+        if args[0] == 0:
+            return self.host
+        elif args[0] == 1:
+            return self.port
+        else:
+            raise KeyError(*args)
+
+
+class ConsoleConfig(BaseSchema):
+    colored: Annotated[
+        bool | None, Field(default=None, description="Opt to print colorized log.", examples=[True])
+    ]
+    format: Annotated[
+        LogFormat, Field(default=LogFormat.VERBOSE, description="Determine verbosity of log.")
+    ]
+
+
+class FileConfig(BaseSchema):
+    path: Annotated[Path, Field(description="Path to store log.", examples=["/var/log/backend.ai"])]
+    filename: Annotated[str, Field(description="Log file name.", examples=["wsproxy.log"])]
+    backup_count: Annotated[
+        int,
+        Field(
+            description="Number of outdated log files to retain.", default=5, alias="backup-count"
+        ),
+    ]
+    rotation_size: Annotated[
+        ByteSize,
+        Field(
+            description="Maximum size for a single log file.", default="10M", alias="rotation-size"
+        ),
+    ]
+    format: Annotated[
+        LogFormat, Field(default=LogFormat.VERBOSE, description="Determine verbosity of log.")
+    ]
+
+
+class LogstashConfig(BaseSchema):
+    endpoint: Annotated[
+        HostPortPair,
+        Field(
+            description="Connection information of logstash node.",
+            examples=[HostPortPair(host="127.0.0.1", port=8001)],
+        ),
+    ]
+    protocol: Annotated[
+        LogstashProtocol,
+        Field(
+            description="Protocol to communicate with logstash server.",
+            default=LogstashProtocol.TCP,
+        ),
+    ]
+    ssl_enabled: Annotated[
+        bool,
+        Field(
+            description="Use TLS to communicate with logstash server.",
+            default=True,
+            alias="ssl-enabled",
+        ),
+    ]
+    ssl_verify: Annotated[
+        bool,
+        Field(
+            description="Verify validity of TLS certificate when communicating with logstash.",
+            default=True,
+            alias="ssl-verify",
+        ),
+    ]
+
+
+class GraylogConfig(BaseSchema):
+    host: Annotated[str, Field(description="Graylog hostname.", examples=["127.0.0.1"])]
+    port: Annotated[int, Field(description="Graylog server port number.", examples=[8000])]
+    level: Annotated[LogLevel, Field(description="Log level.", default=LogLevel.INFO)]
+    ssl_verify: Annotated[
+        bool,
+        Field(
+            description="Verify validity of TLS certificate when communicating with logstash.",
+            default=True,
+            alias="ssl-verify",
+        ),
+    ]
+    ca_certs: Annotated[
+        str | None,
+        Field(
+            description="Path to Root CA certificate file.",
+            examples=["/etc/ssl/ca.pem"],
+            default=None,
+            alias="ca-certs",
+        ),
+    ]
+    keyfile: Annotated[
+        str | None,
+        Field(
+            description="Path to TLS private key file.",
+            examples=["/etc/backend.ai/graylog/privkey.pem"],
+            default=None,
+        ),
+    ]
+    certfile: Annotated[
+        str | None,
+        Field(
+            description="Path to TLS certificate file.",
+            examples=["/etc/backend.ai/graylog/cert.pem"],
+            default=None,
+        ),
+    ]
+
+
+class LoggingConfig(BaseSchema):
+    level: Annotated[LogLevel, Field(default=LogLevel.INFO, description="Log level.")]
+    drivers: Annotated[
+        list[LogDriver],
+        Field(default=[LogDriver.CONSOLE], description="Array of log drivers to print."),
+    ]
+    console: Annotated[
+        ConsoleConfig, Field(default=ConsoleConfig(colored=None, format=LogFormat.VERBOSE))
+    ]
+    file: Annotated[FileConfig | None, Field(default=None)]
+    logstash: Annotated[LogstashConfig | None, Field(default=None)]
+    graylog: Annotated[GraylogConfig | None, Field(default=None)]
+    pkg_ns: Annotated[
+        dict[str, LogLevel],
+        Field(
+            description="Override default log level for specific scope of package",
+            default=default_pkg_ns,
+            alias="pkg-ns",
+        ),
+    ]