cratedb-wtf: Add cratedb-wtf diagnostics program

CHANGES.md

@@ -14,6 +14,7 @@
 - MongoDB: Improve UX by using `ctk load table mongodb://...`
 - load table: Refactor to use more OO
 - Add `examples/cloud_import.py`
+- Add `cratedb-wtf` diagnostics program
 
 
 ## 2023/11/06 v0.0.2

cratedb_toolkit/cli.py

@@ -7,6 +7,7 @@
 from .io.cli import cli as io_cli
 from .job.cli import cli_list_jobs
 from .shell.cli import cli as shell_cli
+from .wtf.cli import cli as wtf_cli
 
 
 @click.group(cls=ClickAliasedGroup) # type: ignore[arg-type]
@@ -21,4 +22,5 @@ def cli(ctx: click.Context, verbose: bool, debug: bool):
 cli.add_command(cloud_cli, name="cluster")
 cli.add_command(io_cli, name="load")
 cli.add_command(shell_cli, name="shell")
+cli.add_command(wtf_cli, name="wtf")
 cli.add_command(cli_list_jobs)

cratedb_toolkit/util/cli.py

@@ -72,13 +72,15 @@ def boot_with_dburi():
  return dburi
 
 
-def make_command(cli, name, helpfun=None, aliases=None):
+def make_command(cli, name, help=None, aliases=None): # noqa: A002
  """
  Convenience shortcut for creating a subcommand.
  """
  kwargs = {}
- if helpfun:
- kwargs["help"] = docstring_format_verbatim(helpfun.__doc__)
+ if isinstance(help, str):
+ kwargs["help"] = help
+ elif callable(help):
+ kwargs["help"] = docstring_format_verbatim(help.__doc__)
  return cli.command(
  name,
  context_settings={"max_content_width": 120},

cratedb_toolkit/util/data.py

@@ -1,3 +1,4 @@
+import datetime as dt
 import json
 import sys
 import typing as t
@@ -7,7 +8,7 @@ def jd(data: t.Any):
  """
  Pretty-print JSON with indentation.
  """
- print(json.dumps(data, indent=2), file=sys.stdout) # noqa: T201
+ print(json.dumps(data, indent=2, cls=JSONEncoderPlus), file=sys.stdout) # noqa: T201
 
 
 def str_contains(haystack, *needles):
@@ -16,3 +17,15 @@ def str_contains(haystack, *needles):
  """
  haystack = str(haystack)
  return any(needle in haystack for needle in needles)
+
+
+class JSONEncoderPlus(json.JSONEncoder):
+ """
+ https://stackoverflow.com/a/27058505
+ """
+
+ def default(self, o):
+ if isinstance(o, dt.datetime):
+ return o.isoformat()
+
+ return json.JSONEncoder.default(self, o)

cratedb_toolkit/util/platform.py

@@ -0,0 +1,56 @@
+import io
+import json
+from contextlib import redirect_stdout
+
+
+class PlatformInfo:
+ @staticmethod
+ def application():
+ import platform
+
+ from cratedb_toolkit import __appname__, __version__
+
+ data = {}
+
+ data["platform"] = platform.platform()
+ data["version"] = __version__
+ data["name"] = __appname__
+ return data
+
+ @staticmethod
+ def libraries():
+ data = {}
+
+ # SQLAlchemy
+ from importlib.metadata import entry_points
+
+ try:
+ import sqlalchemy.dialects.plugins
+ import sqlalchemy.dialects.registry
+
+ data["sqlalchemy"] = {
+ "dialects_builtin": list(sqlalchemy.dialects.registry.impls.keys()),
+ "dialects_3rdparty": [dialect.name for dialect in entry_points(group="sqlalchemy.dialects")], # type: ignore[attr-defined,call-arg]
+ "plugins": list(sqlalchemy.dialects.plugins.impls.keys()),
+ }
+ except Exception: # noqa: S110
+ pass
+
+ # pandas
+ try:
+ import pandas
+
+ buffer = io.StringIO()
+ with redirect_stdout(buffer):
+ pandas.show_versions(as_json=True)
+ buffer.seek(0)
+ data["pandas"] = json.load(buffer)
+ except Exception: # noqa: S110
+ pass
+
+ # fsspec
+ import fsspec
+
+ data["fsspec"] = {"protocols": fsspec.available_protocols(), "compressions": fsspec.available_compressions()}
+
+ return data

cratedb_toolkit/util/service.py

@@ -0,0 +1,24 @@
+# Copyright (c) 2021-2023, Crate.io Inc.
+# Distributed under the terms of the AGPLv3 license, see LICENSE.
+import logging
+
+import typting as t
+
+from cratedb_toolkit.util.common import setup_logging
+
+logger = logging.getLogger(__name__)
+
+
+def start_service(app: str, listen_address: t.Union[str, None] = None, reload: bool = False): # pragma: no cover
+ setup_logging()
+ from uvicorn import run
+
+ if listen_address is None:
+ listen_address = "127.0.0.1:4242"
+
+ host, port = listen_address.split(":")
+ port_int = int(port)
+
+ logger.info(f"Starting HTTP web service on http://{listen_address}")
+
+ run(app=app, host=host, port=port_int, reload=reload)

cratedb_toolkit/wtf/README.md

@@ -0,0 +1,47 @@
+# cratedb-wtf
+
+A diagnostics utility in the spirit of [git-wtf], [grafana-wtf], and [pip.wtf].
+It is still a work-in-progress, but it is usable already.
+
+
+## Synopsis
+
+Define CrateDB database cluster address.
+```shell
+export CRATEDB_SQLALCHEMY_URL=crate://localhost/
+```
+
+Display system and database cluster information.
+```shell
+cratedb-wtf info
+```
+
+Display database cluster log messages.
+```shell
+cratedb-wtf logs
+```
+
+Statistics.
+```shell
+cratedb-wtf statistics quick
+cratedb-wtf statistics collect
+cratedb-wtf statistics view
+```
+
+
+## HTTP API
+
+Expose collected status information. 
+```shell
+cratedb-wtf --debug serve --reload
+```
+Consume collected status information via HTTP.
+```shell
+http http://127.0.0.1:4242/info/all
+```
+
+
+
+[git-wtf]: http://thrawn01.org/posts/2014/03/03/git-wtf/ 
+[grafana-wtf]: https://github.com/panodata/grafana-wtf
+[pip.wtf]: https://github.com/sabslikesobs/pip.wtf

cratedb_toolkit/wtf/backlog.md

@@ -0,0 +1,35 @@
+# cratedb-wtf backlog
+
+## Iteration +1
+- Complete collected queries and code snippets
+- Harvest queries from Admin UI, crash, crate-admin-alt
+- Harvest queries from experts
+ - https://tools.cr8.net/grafana/d/RkpNJx84z/cratedb-jobs-log?orgId=1&refresh=5m&var-datasource=crate-production
+ - https://tools.cr8.net/grafana/d/RkpNJx84z/cratedb-jobs-log?orgId=1&refresh=5m&var-datasource=crate-production&viewPanel=44
+- Add `description` field to each `InfoElement`
+
+## Iteration +2
+- Make `cratedb-wtf logs` also optionally consider `sys.` tables. 
+- cratedb-wtf explore table|shard|partition|node
+- High-level analysis, evaluating a set of threshold rules 
+- Network diagnostics?
+- Expose collected data via Glances-like UI
+
+## Notes
+```
+ctk cluster info
+ctk cluster health
+ctk cluster logs --slow-queries
+```
+
+Acknowledgements: @baur, @hammerhea, @walbeh.
+
+
+## Done
+- Make it work
+- Proper marshalling of timestamp values (ISO 8601)
+- Expose collected data via HTTP API
+ ```
+ cratedb-wtf serve
+ ```
+- Provide `scrub` option also via HTTP