From 8322086f88a1d14c60dcfdb14cea195cbc5fbac2 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 7 Jun 2021 16:22:37 -0400 Subject: [PATCH 1/4] Add scheduler log tab to per reports --- distributed/dashboard/components/scheduler.py | 41 +++++++++++++++++++ distributed/scheduler.py | 8 ++++ 2 files changed, 49 insertions(+) diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py index 28250ff9d10..833c37385ac 100644 --- a/distributed/dashboard/components/scheduler.py +++ b/distributed/dashboard/components/scheduler.py @@ -2162,6 +2162,47 @@ def update(self): self.source.data.update(data) +class SchedulerLogTable(DashboardComponent): + """Sortable table of the scheduler logs""" + + table_names = {"level": "log level", "msg": "message"} + + def __init__(self, scheduler, width=800, **kwargs): + self.scheduler = scheduler + self.source = ColumnDataSource({k: [] for k in self.table_names}) + + columns = { + name: TableColumn(field=name, title=title) + for name, title in self.table_names.items() + } + + table = DataTable( + source=self.source, + columns=list(columns.values()), + sortable=True, + width=width, + index_position=None, + ) + + if "sizing_mode" in kwargs: + sizing_mode = {"sizing_mode": kwargs["sizing_mode"]} + else: + sizing_mode = {} + + components = [table] + + self.root = column(*components, id="bk-scheduler-log-table", **sizing_mode) + + def get_data(self): + logs = self.scheduler.get_logs() + return {k: list(log[i] for log in logs) for i, k in enumerate(self.table_names)} + + @without_property_validation + def update(self): + with log_errors(): + self.source.stream(self.get_data(), 1000) + + def systemmonitor_doc(scheduler, extra, doc): with log_errors(): sysmon = SystemMonitor(scheduler, sizing_mode="stretch_both") diff --git a/distributed/scheduler.py b/distributed/scheduler.py index 863e05bc1c1..127d5cfa0b3 100644 --- a/distributed/scheduler.py +++ b/distributed/scheduler.py @@ -6905,6 +6905,12 @@ def profile_to_figure(state): sysmon = SystemMonitor(self, last_count=last_count, sizing_mode="stretch_both") sysmon.update() + # Scheduler logs + from distributed.dashboard.components.scheduler import SchedulerLogTable + + log_table = SchedulerLogTable(self, sizing_mode="stretch_both") + log_table.update() + from bokeh.models import Div, Panel, Tabs import distributed @@ -6963,12 +6969,14 @@ def profile_to_figure(state): ) bandwidth_types = Panel(child=bandwidth_types.root, title="Bandwidth (Types)") system = Panel(child=sysmon.root, title="System") + logs = Panel(child=log_table.root, title="Scheduler Logs") tabs = Tabs( tabs=[ html, task_stream, system, + logs, compute, workers, scheduler, From 1ad55831e8082293bba5ec6b638701f890c16b1f Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Fri, 11 Jun 2021 19:12:58 -0400 Subject: [PATCH 2/4] Use jinja template for logs component --- distributed/dashboard/components/scheduler.py | 68 +++++++++---------- distributed/scheduler.py | 7 +- 2 files changed, 35 insertions(+), 40 deletions(-) diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py index 833c37385ac..2107ca55570 100644 --- a/distributed/dashboard/components/scheduler.py +++ b/distributed/dashboard/components/scheduler.py @@ -32,10 +32,12 @@ value, ) from bokeh.models.widgets import DataTable, TableColumn +from bokeh.models.widgets.markups import Div from bokeh.palettes import Viridis11 from bokeh.plotting import figure from bokeh.themes import Theme from bokeh.transform import cumsum, factor_cmap, linear_cmap +from jinja2.environment import Template from tlz import curry, pipe from tlz.curried import concat, groupby, map from tornado import escape @@ -2162,45 +2164,39 @@ def update(self): self.source.data.update(data) -class SchedulerLogTable(DashboardComponent): - """Sortable table of the scheduler logs""" - - table_names = {"level": "log level", "msg": "message"} - - def __init__(self, scheduler, width=800, **kwargs): - self.scheduler = scheduler - self.source = ColumnDataSource({k: [] for k in self.table_names}) - - columns = { - name: TableColumn(field=name, title=title) - for name, title in self.table_names.items() - } - - table = DataTable( - source=self.source, - columns=list(columns.values()), - sortable=True, - width=width, - index_position=None, +class SchedulerLogs: + def __init__(self, scheduler): + template = Template( + """ + {% for level, message in logs %} +

+ {{ message }} +

+ {% endfor %} + + """ ) - if "sizing_mode" in kwargs: - sizing_mode = {"sizing_mode": kwargs["sizing_mode"]} - else: - sizing_mode = {} - - components = [table] - - self.root = column(*components, id="bk-scheduler-log-table", **sizing_mode) - - def get_data(self): - logs = self.scheduler.get_logs() - return {k: list(log[i] for log in logs) for i, k in enumerate(self.table_names)} + logs = scheduler.get_logs() - @without_property_validation - def update(self): - with log_errors(): - self.source.stream(self.get_data(), 1000) + self.root = Div(text=template.render(logs=logs)) def systemmonitor_doc(scheduler, extra, doc): diff --git a/distributed/scheduler.py b/distributed/scheduler.py index 127d5cfa0b3..8f06a3b6016 100644 --- a/distributed/scheduler.py +++ b/distributed/scheduler.py @@ -6906,10 +6906,9 @@ def profile_to_figure(state): sysmon.update() # Scheduler logs - from distributed.dashboard.components.scheduler import SchedulerLogTable + from distributed.dashboard.components.scheduler import SchedulerLogs - log_table = SchedulerLogTable(self, sizing_mode="stretch_both") - log_table.update() + logs = SchedulerLogs(self) from bokeh.models import Div, Panel, Tabs @@ -6969,7 +6968,7 @@ def profile_to_figure(state): ) bandwidth_types = Panel(child=bandwidth_types.root, title="Bandwidth (Types)") system = Panel(child=sysmon.root, title="System") - logs = Panel(child=log_table.root, title="Scheduler Logs") + logs = Panel(child=logs.root, title="Scheduler Logs") tabs = Tabs( tabs=[ From 2d678df1b150593fa37d884206ba717592d6b70a Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 14 Jun 2021 12:43:34 -0400 Subject: [PATCH 3/4] Refactor log HTML reprs with level styling --- distributed/dashboard/components/scheduler.py | 35 ++---------------- distributed/deploy/cluster.py | 13 +++---- distributed/tests/test_utils.py | 5 +-- distributed/utils.py | 37 ++++++++++++++----- 4 files changed, 38 insertions(+), 52 deletions(-) diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py index 2107ca55570..b00bcfb94b0 100644 --- a/distributed/dashboard/components/scheduler.py +++ b/distributed/dashboard/components/scheduler.py @@ -37,7 +37,6 @@ from bokeh.plotting import figure from bokeh.themes import Theme from bokeh.transform import cumsum, factor_cmap, linear_cmap -from jinja2.environment import Template from tlz import curry, pipe from tlz.curried import concat, groupby, map from tornado import escape @@ -71,7 +70,7 @@ from distributed.diagnostics.task_stream import color_of as ts_color_of from distributed.diagnostics.task_stream import colors as ts_color_lookup from distributed.metrics import time -from distributed.utils import format_time, log_errors, parse_timedelta +from distributed.utils import Logs, format_time, log_errors, parse_timedelta if dask.config.get("distributed.dashboard.export-tool"): from distributed.dashboard.export_tool import ExportTool @@ -2166,37 +2165,9 @@ def update(self): class SchedulerLogs: def __init__(self, scheduler): - template = Template( - """ - {% for level, message in logs %} -

- {{ message }} -

- {% endfor %} - - """ - ) - - logs = scheduler.get_logs() + logs = Logs(scheduler.get_logs())._repr_html_() - self.root = Div(text=template.render(logs=logs)) + self.root = Div(text=logs) def systemmonitor_doc(scheduler, extra, doc): diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py index d71f68f0a90..8ea7a989e24 100644 --- a/distributed/deploy/cluster.py +++ b/distributed/deploy/cluster.py @@ -14,8 +14,7 @@ from ..core import Status from ..utils import ( - Log, - Logs, + MultiLogs, format_dashboard_link, log_errors, parse_timedelta, @@ -208,21 +207,19 @@ def _log(self, log): print(log) async def _get_logs(self, cluster=True, scheduler=True, workers=True): - logs = Logs() + logs = MultiLogs() if cluster: - logs["Cluster"] = Log( - "\n".join(line[1] for line in self._cluster_manager_logs) - ) + logs["Cluster"] = self._cluster_manager_logs if scheduler: L = await self.scheduler_comm.get_logs() - logs["Scheduler"] = Log("\n".join(line for level, line in L)) + logs["Scheduler"] = L if workers: d = await self.scheduler_comm.worker_logs(workers=workers) for k, v in d.items(): - logs[k] = Log("\n".join(line for level, line in v)) + logs[k] = v return logs diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py index 6d9ab29520b..ed6dda137f0 100644 --- a/distributed/tests/test_utils.py +++ b/distributed/tests/test_utils.py @@ -19,9 +19,8 @@ from distributed.utils import ( LRU, All, - Log, - Logs, LoopRunner, + MultiLogs, TimeoutError, _maybe_complex, deprecated, @@ -559,7 +558,7 @@ def test_format_bytes_compat(): def test_logs(): - d = Logs({"123": Log("Hello"), "456": Log("World!")}) + d = MultiLogs({"123": [("INFO", "Hello")], "456": [("INFO", "World!")]}) text = d._repr_html_() assert is_valid_xml("
" + text + "
") assert "Hello" in text diff --git a/distributed/utils.py b/distributed/utils.py index dc63202486b..8e00046b6cf 100644 --- a/distributed/utils.py +++ b/distributed/utils.py @@ -1363,25 +1363,44 @@ def parse_ports(port): is_coroutine_function = iscoroutinefunction -class Log(str): - """A container for logs""" +class Log(tuple): + """A container for a single log entry""" + + level_styles = { + "WARNING": "font-weight: bold; color: orange;", + "CRITICAL": "font-weight: bold; color: orangered;", + "ERROR": "font-weight: bold; color: crimson;", + } def _repr_html_(self): - return "
\n{log}\n
".format( - log=html.escape(self.rstrip()) + level, message = self + + style = "font-family: monospace; margin: 0;" + style += self.level_styles.get(level, "") + + return '

{message}

'.format( + style=html.escape(style), + message=html.escape(message), ) -class Logs(dict): - """A container for multiple logs""" +class Logs(list): + """A container for a list of log entries""" + + def _repr_html_(self): + return "\n".join(Log(entry)._repr_html_() for entry in self) + + +class MultiLogs(dict): + """A container for a dict mapping strings to lists of log entries""" def _repr_html_(self): summaries = [ "
\n" "{title}\n" - "{log}\n" - "
".format(title=title, log=log._repr_html_()) - for title, log in sorted(self.items()) + "{logs}\n" + "".format(title=title, logs=Logs(entries)._repr_html_()) + for title, entries in sorted(self.items()) ] return "\n".join(summaries) From eddabdbf1e265e028064161578c1423d4778214e Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 14 Jun 2021 13:28:51 -0400 Subject: [PATCH 4/4] Add assertion for scheduler logs tab --- distributed/tests/test_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py index 4b86ae05c81..9f7a66bd169 100644 --- a/distributed/tests/test_client.py +++ b/distributed/tests/test_client.py @@ -6308,6 +6308,7 @@ async def f(stacklevel): assert "Dask Performance Report" in data assert "x = da.random" in data assert "Threads: 4" in data + assert "distributed.scheduler - INFO - Clear task state" in data assert dask.__version__ in data # Stacklevel two captures code two frames back -- which in this case