From d41d00b892b147dcf4060923bb93d5dd735c8742 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Mon, 30 Nov 2020 19:24:20 +0100 Subject: [PATCH 01/10] starting to implement a treemap To have more control and to spare the creation of a squarify debian package a simplified fork of https://github.com/laserson/squarify is created to enable creation of treemaps in svg directly without using matplotlib or alike. --- pheme/templatetags/charts/__init__.py | 28 ++ .../{charts.py => charts/h_bar.py} | 103 +------ pheme/templatetags/charts/pie.py | 107 +++++++ pheme/templatetags/charts/treemap.py | 287 ++++++++++++++++++ 4 files changed, 427 insertions(+), 98 deletions(-) create mode 100644 pheme/templatetags/charts/__init__.py rename pheme/templatetags/{charts.py => charts/h_bar.py} (66%) create mode 100644 pheme/templatetags/charts/pie.py create mode 100644 pheme/templatetags/charts/treemap.py diff --git a/pheme/templatetags/charts/__init__.py b/pheme/templatetags/charts/__init__.py new file mode 100644 index 00000000..a2cc7710 --- /dev/null +++ b/pheme/templatetags/charts/__init__.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# pheme/templatetags/__init__.py +# Copyright (C) 2020 Greenbone Networks GmbH +# +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django import template + +_severity_class_colors = { + 'High': "#d4003e", + 'Medium': "#fcb900", + 'Low': "#7db4d0", +} + +register = template.Library() diff --git a/pheme/templatetags/charts.py b/pheme/templatetags/charts/h_bar.py similarity index 66% rename from pheme/templatetags/charts.py rename to pheme/templatetags/charts/h_bar.py index 38d4b6f6..16220edb 100644 --- a/pheme/templatetags/charts.py +++ b/pheme/templatetags/charts/h_bar.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# pheme/templatetags/charts.py +# pheme/templatetags/bar_chart.py # Copyright (C) 2020 Greenbone Networks GmbH # # SPDX-License-Identifier: AGPL-3.0-or-later @@ -16,24 +16,13 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import io -import itertools -from typing import Callable, Optional, Union, Dict -from django import template +import itertools +from typing import Dict from django.utils.safestring import mark_safe -from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas -from matplotlib.figure import Figure - -__severity_class_colors = { - 'High': "#d4003e", - 'Medium': "#fcb900", - 'Low': "#7db4d0", -} +from pheme.templatetags.charts import register, _severity_class_colors -register = template.Library() - __ORIENTATION_LINE_TEMPLATE = """ """ @@ -110,7 +99,7 @@ def h_bar_chart( data = dict(itertools.islice(chart_data.items(), limit)) if not title_color: - title_color = __severity_class_colors + title_color = _severity_class_colors max_width = svg_width - 175 - 100 # key and total placeholder # highest sum of counts max_sum = max([sum(list(counts.values())) for counts in data.values()]) @@ -167,85 +156,3 @@ def __add_orientation(i: int, orientation_lines="", orientation_labels=""): bar_legend_y=len(data.keys()) * bar_jump + 20, ) return mark_safe(svg_chart) - - -def __create_default_figure(): - return Figure() - - -def __create_chart( - set_plot: Callable, - *, - fig: Union[Figure, Callable] = __create_default_figure, - modify_fig: Callable = None, -) -> Optional[str]: - fig = fig() if callable(fig) else fig - # there is a bug in 3.0.2 (debian buster) - # that canvas is not set automatically - canvas = FigureCanvas(fig) - ax = fig.subplots() - set_plot(ax) - if modify_fig: - modify_fig(fig) - buf = io.BytesIO() - fig.canvas = canvas - fig.savefig(buf, format='svg', dpi=300) - buf.seek(0) - # base64_fig = base64.b64encode(buf.read()) - # uri = 'data:image/png;base64,' + urllib.parse.quote(base64_fig) - return buf.read().decode() - - -@register.filter -def pie_chart(input_values, title_color=None, title=None) -> Optional[str]: - """ - creates a pie chart svg. - - The values parameter needs to be a dict with a categoryname: numeric_value. - E.g.: - { - "High": 5, - "Medium": 3, - "Low": 2 - } - - The keys need to match the title_color keys. As a default the - severity_classes: High , Medium, Low are getting used. - """ - if not title_color: - title_color = __severity_class_colors - category_names = list(input_values.keys()) - category_colors = list([title_color.get(key) for key in category_names]) - values = list(input_values.values()) - - total = sum(values) - - def raw_value_pct(pct): - value = pct * total / 100.0 - return "{:d}".format(int(round(value))) - - def modify_fig(fig: Figure): - for ax in fig.axes: - ax.set_axis_off() - - def set_plot(ax): - ax.set_title(title) - wedges, _, _ = ax.pie( - values, - colors=category_colors, - autopct=raw_value_pct, - wedgeprops=dict(width=0.5), - startangle=-40, - ) - - ax.legend( - wedges, - category_names, - bbox_to_anchor=(1, 0, 0, 1), - loc='lower right', - fontsize='small', - ) - - if total == 0: - return None - return mark_safe(__create_chart(set_plot, modify_fig=modify_fig)) diff --git a/pheme/templatetags/charts/pie.py b/pheme/templatetags/charts/pie.py new file mode 100644 index 00000000..5c1f5f45 --- /dev/null +++ b/pheme/templatetags/charts/pie.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +# pheme/templatetags/charts.py +# Copyright (C) 2020 Greenbone Networks GmbH +# +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +import io +from typing import Callable, Optional, Union + +from django.utils.safestring import mark_safe +from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas +from matplotlib.figure import Figure +from pheme.templatetags.charts import register, _severity_class_colors + + +def __create_default_figure(): + return Figure() + + +def __create_chart( + set_plot: Callable, + *, + fig: Union[Figure, Callable] = __create_default_figure, + modify_fig: Callable = None, +) -> Optional[str]: + fig = fig() if callable(fig) else fig + # there is a bug in 3.0.2 (debian buster) + # that canvas is not set automatically + canvas = FigureCanvas(fig) + ax = fig.subplots() + set_plot(ax) + if modify_fig: + modify_fig(fig) + buf = io.BytesIO() + fig.canvas = canvas + fig.savefig(buf, format='svg', dpi=300) + buf.seek(0) + # base64_fig = base64.b64encode(buf.read()) + # uri = 'data:image/png;base64,' + urllib.parse.quote(base64_fig) + return buf.read().decode() + + +@register.filter +def pie_chart(input_values, title_color=None, title=None) -> Optional[str]: + """ + creates a pie chart svg. + + The values parameter needs to be a dict with a categoryname: numeric_value. + E.g.: + { + "High": 5, + "Medium": 3, + "Low": 2 + } + + The keys need to match the title_color keys. As a default the + severity_classes: High , Medium, Low are getting used. + """ + if not title_color: + title_color = _severity_class_colors + category_names = list(input_values.keys()) + category_colors = list([title_color.get(key) for key in category_names]) + values = list(input_values.values()) + + total = sum(values) + + def raw_value_pct(pct): + value = pct * total / 100.0 + return "{:d}".format(int(round(value))) + + def modify_fig(fig: Figure): + for ax in fig.axes: + ax.set_axis_off() + + def set_plot(ax): + ax.set_title(title) + wedges, _, _ = ax.pie( + values, + colors=category_colors, + autopct=raw_value_pct, + wedgeprops=dict(width=0.5), + startangle=-40, + ) + + ax.legend( + wedges, + category_names, + bbox_to_anchor=(1, 0, 0, 1), + loc='lower right', + fontsize='small', + ) + + if total == 0: + return None + return mark_safe(__create_chart(set_plot, modify_fig=modify_fig)) diff --git a/pheme/templatetags/charts/treemap.py b/pheme/templatetags/charts/treemap.py new file mode 100644 index 00000000..6044a86a --- /dev/null +++ b/pheme/templatetags/charts/treemap.py @@ -0,0 +1,287 @@ +# -*- coding: utf-8 -*- +# /home/peder/squary.py +# Copyright (C) 2020 Greenbone Networks GmbH +# +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# Squarified Treemap Layout +# Implements algorithm from Bruls, Huizing, van Wijk, "Squarified Treemaps" +# (but not using their pseudocode) + +# It is a simplified, adapted version of squarify: +# https://github.com/laserson/squarify +# For more information see: +# https://www.win.tue.nl/~vanwijk/stm.pdf +# pylint: disable=C0103 +import math as m +import numbers +from dataclasses import dataclass +from typing import Dict, List + +from django.utils.safestring import mark_safe + +from pheme.templatetags.charts import _severity_class_colors, register + +__ELEMENT_TEMPLATE = """ + + + {label} + +""" + +__TEMPLATE = """ + + {rects} + +""" + + +@dataclass +class Rect: + x: float + y: float + dx: float + dy: float + + +def __create_rectangle(x, y, dx, dy) -> Rect: + # add spacing of 1px + if dx > 2: + x += 1 + dx -= 2 + if dy > 2: + y += 1 + dy -= 2 + return Rect(x, y, dx, dy) + + +def __layoutrow(sizes, x, y, _, dy) -> List[Rect]: + covered_area = sum(sizes) + width = covered_area / dy + rects = [] + for size in sizes: + rects.append(__create_rectangle(x, y, width, size / width)) + y += size / width + return rects + + +def __layoutcol(sizes, x, y, dx, _) -> List[Rect]: + covered_area = sum(sizes) + height = covered_area / dx + rects = [] + for size in sizes: + rects.append(__create_rectangle(x, y, size / height, height)) + x += size / height + return rects + + +def __layout(sizes, x, y, dx, dy) -> List[Rect]: + if dx >= dy: + return __layoutrow(sizes, x, y, dx, dy) + return __layoutcol(sizes, x, y, dx, dy) + + +def __leftoverrow(sizes, x, y, dx, dy): + covered_area = sum(sizes) + width = covered_area / dy + leftover_x = x + width + leftover_y = y + leftover_dx = dx - width + leftover_dy = dy + return leftover_x, leftover_y, leftover_dx, leftover_dy + + +def __leftovercol(sizes, x, y, dx, dy): + covered_area = sum(sizes) + height = covered_area / dx + leftover_x = x + leftover_y = y + height + leftover_dx = dx + leftover_dy = dy - height + return leftover_x, leftover_y, leftover_dx, leftover_dy + + +def __leftover(sizes, x, y, dx, dy): + if dx >= dy: + return __leftoverrow(sizes, x, y, dx, dy) + return __leftovercol(sizes, x, y, dx, dy) + + +def __find_split(sizes, x, y, dx, dy) -> int: + """ + returns the index to split the sizes based on worst ratio to get the + remaining and current space. + + For an example the area ratio of 5, 3, 2, 1 with the area of 90 * 50 + >>> test_data = [2045.5, 1227.3, 818.2, 409.0 ] + >>> __find_split(test_data, 0, 0, 90, 50) + 1 + + The first two elements can be put into the first space and the rest needs to + calculated for the remaining space. + + """ + + def worst(i: int) -> float: + return max( + [ + max(rect.dx / rect.dy, rect.dy / rect.dx) + for rect in __layout(sizes[:i], x, y, dx, dy) + ] + ) + + for i in range(1, len(sizes)): + if worst(i) < worst(i + 1): + return i + return len(sizes) - 1 + + +def __squarify(sizes, x, y, dx, dy) -> List[Rect]: + """ + calculates treemap rectangles using an algorithm based on Bruls, Huizing, + van Wijk, "Squarified Treemaps" and "squarify": + https://github.com/laserson/squarify + + >>> __squarify([5, 1], 0, 0, 90, 50) + [Rect(x=1, y=1, dx=73.0, dy=48.0), Rect(x=76.0, y=1, dx=13.0, dy=48.0)] + + Parameters: + sizes : list-like of numeric values + The set of values to compute a treemap for. `sizes` must be sorted, + positive values + x, y : numeric + The coordinates of the "origin". + dx, dy : numeric + The full width (`dx`) and height (`dy`) of the treemap. + + Returns: + List[Rect] + Each dict in the returned list represents a single rectangle in the + treemap. The order corresponds to the input order. + """ + + if len(sizes) == 0: + return [] + + total_size = sum(sizes) + total_area = dx * dy + + sizes = list([size * total_area / total_size for size in sizes]) + + if len(sizes) == 1: + return __layout(sizes, x, y, dx, dy) + + i = __find_split(sizes, x, y, dx, dy) + current = sizes[:i] + remaining = sizes[i:] + + return __layout(current, x, y, dx, dy) + __squarify( + remaining, *__leftover(current, x, y, dx, dy) + ) + + +def __transform_to_tree_data(data) -> List[Dict]: + """ + tansforms given data to treemap compatible format. + The support types are: + >>> example_host_data = dict( + ... host1=dict(high=12, medium=4, low=0), + ... host2=dict(high=0, medium=4, low=0), + ... ) + >>> __transform_to_tree_data(example_host_data) + ([16, 4], ['host1', 'host2'], ['high', 'medium']) + + On unsupported input data it returns empty lists: + >>> __transform_to_tree_data(dict(k="hello", v="world")) + ([], [], []) + >>> __transform_to_tree_data(12) + ([], [], []) + """ + values = [] + labels = [] + color_keys = [] + if isinstance(data, dict): + for key, val in data.items(): + color = None + val_sum = 0 + if isinstance(val, dict): + for item_key, item_val in val.items(): + if isinstance(item_val, numbers.Number): + val_sum += item_val + if color is None and item_val > 0: + color = item_key + if color is not None and val_sum > 0: + values.append(val_sum) + labels.append(key) + color_keys.append(color) + return values, labels, color_keys + + +@register.filter +def treemap( + data: List[Dict], + width=1024, + height=768, + fontsize=11, + border_color="#ffffff", + title_color=None, +): + """ + expects a descending sorted list of a dict. + + It will transform given data to an useful format and than create + rectangle dimeniosn out of it. Thise dimensions will be used to create + the svg. + + The color key within the data must be consistant with the colors. + + Parameters: + + Returns: + """ + if not title_color: + title_color = _severity_class_colors + sizes, label, color_keys = __transform_to_tree_data(data) + sizes = __squarify(sizes, 0, 0, width, height) + elements = "" + for i, d in enumerate(sizes): + label_size_in_px = len(label[i]) * fontsize + label_x = d.x + 1 + # move half of the font size down and add a buffer for different + # heights of characters (depends on font) to display the label + # on the upper left corner of an rectangle. + label_y = d.y + fontsize / 2 + 5 + max_label_len = m.ceil(d.dx / label_size_in_px * len(label[i])) + if d.dy <= fontsize: + max_label_len = 0 + + elements += __ELEMENT_TEMPLATE.format( + x=d.x, + y=d.y, + width=d.dx, + height=d.dy, + color=title_color.get(color_keys[i]), + border_color=border_color, + label_x=label_x, + label_y=label_y, + label=label[i][:max_label_len], + ) + return mark_safe( + __TEMPLATE.format(width=width, height=height, rects=elements) + ) From 2bc1ce557e271f9b054b28dc51f871da3771f7e3 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Tue, 1 Dec 2020 14:02:43 +0100 Subject: [PATCH 02/10] add doctests to pytest.ini remove asgi.py Due to version switch from 3 to 2 asgi is not supported and is removed. Add doctests to treemap.py for demonstration purpose, add doc test execution to pytest due to that. --- pheme/asgi.py | 34 ---------------------------------- pytest.ini | 1 + 2 files changed, 1 insertion(+), 34 deletions(-) delete mode 100644 pheme/asgi.py diff --git a/pheme/asgi.py b/pheme/asgi.py deleted file mode 100644 index 6e5a3330..00000000 --- a/pheme/asgi.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- -# pheme/asgi.py -# Copyright (C) 2020 Greenbone Networks GmbH -# -# SPDX-License-Identifier: AGPL-3.0-or-later -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -""" -ASGI config for pheme project. - -It exposes the ASGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/3.1/howto/deployment/asgi/ -""" - -import os - -from django.core.asgi import get_asgi_application - -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'pheme.settings') - -application = get_asgi_application() diff --git a/pytest.ini b/pytest.ini index fb3c2fa6..42e01fe0 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,6 @@ # -- FILE: pytest.ini (or tox.ini) [pytest] +addopts = --doctest-modules DJANGO_SETTINGS_MODULE = pheme.settings # -- recommended but optional: python_files = tests.py test_*.py *_tests.py From 76a60660bc7997bb55e6382734e95a17564ce2c8 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Tue, 1 Dec 2020 14:19:17 +0100 Subject: [PATCH 03/10] add description of treemap function --- pheme/templatetags/charts/treemap.py | 30 ++++++++++++++++++++++------ tests/test_report_generation.py | 16 ++++++--------- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/pheme/templatetags/charts/treemap.py b/pheme/templatetags/charts/treemap.py index 6044a86a..8b284e17 100644 --- a/pheme/templatetags/charts/treemap.py +++ b/pheme/templatetags/charts/treemap.py @@ -241,19 +241,37 @@ def treemap( fontsize=11, border_color="#ffffff", title_color=None, -): +) -> str: """ - expects a descending sorted list of a dict. + Expects a sorted dict containing a str, and a dict with values in it. - It will transform given data to an useful format and than create - rectangle dimeniosn out of it. Thise dimensions will be used to create - the svg. + An example can be: - The color key within the data must be consistant with the colors. + { + 'host1': {'high': 12, 'medium': 4, 'low': 0}, + 'host2': {'high': 0, 'medium': 4, 'low': 0}, + } + + With that host1 and host2 will be used as a label, high and medium will be + used to find the color and values will be summed for the rectangle + calculation. + + Data will be transformed to lits of labels, color_keys and the actual + values. + + The color key within the data must be consistant with title_color. Parameters: + data: needs to be an dict containing label, color_key and values. + width: width of the svg (default 1024) + height: height of the svg (default 768) + fontsize: used fontsize (default 11) + border_color: color of the rectangle border (default white) + title_color: the color_key to color lookup map + (default _severity_class_colors) Returns: + the treemap in svg as a SafeString. """ if not title_color: title_color = _severity_class_colors diff --git a/tests/test_report_generation.py b/tests/test_report_generation.py index 75e46359..09025646 100644 --- a/tests/test_report_generation.py +++ b/tests/test_report_generation.py @@ -82,15 +82,8 @@ def test_workaround_for_inline_svg_and_weasyprint(html_contains): assert contains in result -@pytest.mark.parametrize( - "http_accept", - [ - "application/pdf", - "text/html", - ], -) -def test_chart_keyswords(http_accept): - subtype = http_accept.split('/')[-1] +def test_chart_keyswords(): + subtype = "html" css_key = 'vulnerability_report_{}_css'.format(subtype) template_key = 'vulnerability_report_{}_template'.format(subtype) client = APIClient() @@ -102,6 +95,7 @@ def test_chart_keyswords(http_accept): {{ overview.nvts | pie_chart}} {{ overview.hosts | h_bar_chart }} + {{ overview.hosts | treemap }} """ response = client.put( @@ -113,7 +107,9 @@ def test_chart_keyswords(http_accept): HTTP_X_API_KEY=SECRET_KEY, ) assert response.status_code == 200 - test_http_accept(http_accept) + response = test_http_accept("text/html") + html_report = response.getvalue().decode('utf-8') + assert html_report.count(" Date: Tue, 1 Dec 2020 15:54:22 +0100 Subject: [PATCH 04/10] fix just the first svg will be replaced, fix bar chart The bar chart is missing an closing g element and the svg image replacement worked just for the first svg and will then produce gibberish since the index has changed on html change. In the moment it is using recursion, which should be replaced because python does not nor will ever optimize tail recursion. --- pheme/templatetags/charts/h_bar.py | 9 ++-- pheme/transformation/scanreport/renderer.py | 57 +++++++-------------- 2 files changed, 24 insertions(+), 42 deletions(-) diff --git a/pheme/templatetags/charts/h_bar.py b/pheme/templatetags/charts/h_bar.py index 16220edb..fc801fd0 100644 --- a/pheme/templatetags/charts/h_bar.py +++ b/pheme/templatetags/charts/h_bar.py @@ -28,7 +28,7 @@ """ __ORIENTATION_LINE_TEXT_TEMPLATE = """ -{} """ @@ -38,7 +38,7 @@ __BAR_TEMPLATE = """ -{key} @@ -46,16 +46,17 @@ {bar_elements} -{total} """ __BAR_CHART_TEMPLATE = """ - + {bars} {bar_legend} + """ diff --git a/pheme/transformation/scanreport/renderer.py b/pheme/transformation/scanreport/renderer.py index 3b0b2c20..b39a208c 100644 --- a/pheme/transformation/scanreport/renderer.py +++ b/pheme/transformation/scanreport/renderer.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . import logging -from typing import Dict, Generator, Tuple +from typing import Dict from base64 import b64encode @@ -109,33 +109,9 @@ def apply(self, name: str, data: Dict, parameter: Dict): ) -def __find_all_tags_in( - html: str, open_tag: str, close_tag: str -) -> Generator[Tuple[int, int], None, None]: - - """ - Is used within __replace_inline_svg_with_img_tag - - It searches for open and close tags and returns the start of the opening - and the end of the close_tag so that it is easy to extract and replace an - svg image. - """ - open_index = 0 - close_end_index = 0 - # abort conditions are: open or close_tag not found (-1) - while True: - open_index = html.find(open_tag, open_index) - if open_index == -1: - return - close_end_index = html.find(close_tag, open_index + len(open_tag)) - if close_end_index == -1: - return - close_end_index += len(close_tag) - yield (open_index, close_end_index) - open_index = close_end_index - - -def _replace_inline_svg_with_img_tags(html: str) -> str: +def _replace_inline_svg_with_img_tags( + html: str, from_index: int = 0, open_tag=' str: """ Is a workaround because WeasyPrint is not capable of dealing with inline svg @@ -148,16 +124,21 @@ def _replace_inline_svg_with_img_tags(html: str) -> str: Please replace this method as soon as possible with a proper solution. """ - for from_index, to_index in __find_all_tags_in(html, ''): - to_encode = html[from_index:to_index] - encoded = b64encode(to_encode.encode()).decode() - img = ( - ''.format( - encoded - ) - ) - html = html[:from_index] + img + html[to_index:] - return html + from_index = html.find(open_tag, from_index) + if from_index == -1: + return html + to_index = html.find(close_tag, from_index + len(open_tag)) + if to_index == -1: + return html + to_index += len(close_tag) + to_encode = html[from_index:to_index] + encoded = b64encode(to_encode.encode()).decode() + img = ''.format( + encoded + ) + + html = html[:from_index] + img + html[to_index:] + return _replace_inline_svg_with_img_tags(html, to_index) class VulnerabilityPDFReport(Report): From 5fb4afbe2bdfb90b74336e4c9116f3db07ed64a5 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Tue, 1 Dec 2020 16:02:41 +0100 Subject: [PATCH 05/10] replace recurision with while loop to reduce te overhead on a recurision it replace_inline_svg is using a while loop to find all svgs if no svg was found than it returns the html. --- pheme/transformation/scanreport/renderer.py | 33 +++++++++++---------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/pheme/transformation/scanreport/renderer.py b/pheme/transformation/scanreport/renderer.py index b39a208c..9d404795 100644 --- a/pheme/transformation/scanreport/renderer.py +++ b/pheme/transformation/scanreport/renderer.py @@ -124,21 +124,24 @@ def _replace_inline_svg_with_img_tags( Please replace this method as soon as possible with a proper solution. """ - from_index = html.find(open_tag, from_index) - if from_index == -1: - return html - to_index = html.find(close_tag, from_index + len(open_tag)) - if to_index == -1: - return html - to_index += len(close_tag) - to_encode = html[from_index:to_index] - encoded = b64encode(to_encode.encode()).decode() - img = ''.format( - encoded - ) - - html = html[:from_index] + img + html[to_index:] - return _replace_inline_svg_with_img_tags(html, to_index) + # abort condition is when either open_tag or close_tag was not found + while True: + from_index = html.find(open_tag, from_index) + if from_index == -1: + return html + to_index = html.find(close_tag, from_index + len(open_tag)) + if to_index == -1: + return html + to_index += len(close_tag) + to_encode = html[from_index:to_index] + encoded = b64encode(to_encode.encode()).decode() + img = ( + ''.format( + encoded + ) + ) + html = html[:from_index] + img + html[to_index:] + from_index = to_index class VulnerabilityPDFReport(Report): From c44a8d9714c54b8afe6e095086e3d5142ba09f54 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Tue, 1 Dec 2020 16:27:11 +0100 Subject: [PATCH 06/10] add changelog entry --- CHANGELOG.md | 1 + pheme/templatetags/charts/__init__.py | 1 - pheme/templatetags/charts/treemap.py | 1 - 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2da99c3c..1f8ff124 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - nvt threat information in host result [114](https://github.com/greenbone/pheme/pull/114) - nvt severity information in host result [121](https://github.com/greenbone/pheme/pull/121) +- treemap as svg [128](https://github.com/greenbone/pheme/pull/128) ### Changed - remove pandas due to too old debian version [112](https://github.com/greenbone/pheme/pull/112) - add workaround for svg in pdf with wasyprint [120](https://github.com/greenbone/pheme/pull/120) diff --git a/pheme/templatetags/charts/__init__.py b/pheme/templatetags/charts/__init__.py index a2cc7710..d3368379 100644 --- a/pheme/templatetags/charts/__init__.py +++ b/pheme/templatetags/charts/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -# pheme/templatetags/__init__.py # Copyright (C) 2020 Greenbone Networks GmbH # # SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/pheme/templatetags/charts/treemap.py b/pheme/templatetags/charts/treemap.py index 8b284e17..ab306972 100644 --- a/pheme/templatetags/charts/treemap.py +++ b/pheme/templatetags/charts/treemap.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -# /home/peder/squary.py # Copyright (C) 2020 Greenbone Networks GmbH # # SPDX-License-Identifier: AGPL-3.0-or-later From f55c3300a8e70628dc41831b7d627bc49a3887da Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Wed, 2 Dec 2020 07:53:24 +0100 Subject: [PATCH 07/10] Apply suggestions from code review use Rect class instead of x, y, dx, dy as parameter Co-authored-by: Jaspar L. --- pheme/templatetags/charts/treemap.py | 50 ++++++++++++---------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/pheme/templatetags/charts/treemap.py b/pheme/templatetags/charts/treemap.py index ab306972..b33b2c2c 100644 --- a/pheme/templatetags/charts/treemap.py +++ b/pheme/templatetags/charts/treemap.py @@ -70,56 +70,50 @@ def __create_rectangle(x, y, dx, dy) -> Rect: return Rect(x, y, dx, dy) -def __layoutrow(sizes, x, y, _, dy) -> List[Rect]: +def __layoutrow(sizes, rect: Rect) -> List[Rect]: covered_area = sum(sizes) - width = covered_area / dy + width = covered_area / rect.dy rects = [] + y = rect.y for size in sizes: - rects.append(__create_rectangle(x, y, width, size / width)) + rects.append(__create_rectangle(rect.x, y, width, size / width)) y += size / width return rects -def __layoutcol(sizes, x, y, dx, _) -> List[Rect]: +def __layoutcol(sizes, rect: Rect) -> List[Rect]: covered_area = sum(sizes) - height = covered_area / dx + height = covered_area / rect.dx rects = [] + x = rect.x for size in sizes: - rects.append(__create_rectangle(x, y, size / height, height)) + rects.append(__create_rectangle(x, rect.y, size / height, height)) x += size / height return rects -def __layout(sizes, x, y, dx, dy) -> List[Rect]: - if dx >= dy: - return __layoutrow(sizes, x, y, dx, dy) - return __layoutcol(sizes, x, y, dx, dy) +def __layout(sizes, rect: Rect) -> List[Rect]: + if rect.dx >= rect.dy: + return __layoutrow(sizes, rect) + return __layoutcol(sizes, rect) -def __leftoverrow(sizes, x, y, dx, dy): +def __leftoverrow(sizes, rect: Rect): covered_area = sum(sizes) - width = covered_area / dy - leftover_x = x + width - leftover_y = y - leftover_dx = dx - width - leftover_dy = dy - return leftover_x, leftover_y, leftover_dx, leftover_dy + width = covered_area / rect.dy + return Rect(rect.x + width, rect.y, rect.dx - width, rect.dy) -def __leftovercol(sizes, x, y, dx, dy): - covered_area = sum(sizes) +def __leftovercol(covered_area, rect: Rect) -> Rect: height = covered_area / dx - leftover_x = x - leftover_y = y + height - leftover_dx = dx - leftover_dy = dy - height - return leftover_x, leftover_y, leftover_dx, leftover_dy + return Rect(rect.x, rect.y + height, rect.dx, rect.dy - height) -def __leftover(sizes, x, y, dx, dy): - if dx >= dy: - return __leftoverrow(sizes, x, y, dx, dy) - return __leftovercol(sizes, x, y, dx, dy) +def __leftover(sizes, rect: Rect): + covered_area = sum(sizes) + if rect.dx >=rect. dy: + return __leftoverrow(covered_area, rect) + return __leftovercol(covered_area, rect) def __find_split(sizes, x, y, dx, dy) -> int: From 75c9fefd47051fb411cd0f0659b26edbc7236e33 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Wed, 2 Dec 2020 08:30:53 +0100 Subject: [PATCH 08/10] using Rect class instead of x, y, dx, dy parameter instead of using 4 parameter everywhere use the data class Rect for it. --- pheme/templatetags/charts/treemap.py | 38 +++++++++++++--------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/pheme/templatetags/charts/treemap.py b/pheme/templatetags/charts/treemap.py index b33b2c2c..61387cd7 100644 --- a/pheme/templatetags/charts/treemap.py +++ b/pheme/templatetags/charts/treemap.py @@ -70,7 +70,7 @@ def __create_rectangle(x, y, dx, dy) -> Rect: return Rect(x, y, dx, dy) -def __layoutrow(sizes, rect: Rect) -> List[Rect]: +def __layoutrow(sizes: List[numbers.Number], rect: Rect) -> List[Rect]: covered_area = sum(sizes) width = covered_area / rect.dy rects = [] @@ -81,7 +81,7 @@ def __layoutrow(sizes, rect: Rect) -> List[Rect]: return rects -def __layoutcol(sizes, rect: Rect) -> List[Rect]: +def __layoutcol(sizes: List[numbers.Number], rect: Rect) -> List[Rect]: covered_area = sum(sizes) height = covered_area / rect.dx rects = [] @@ -92,38 +92,37 @@ def __layoutcol(sizes, rect: Rect) -> List[Rect]: return rects -def __layout(sizes, rect: Rect) -> List[Rect]: +def __layout(sizes: List[numbers.Number], rect: Rect) -> List[Rect]: if rect.dx >= rect.dy: return __layoutrow(sizes, rect) return __layoutcol(sizes, rect) -def __leftoverrow(sizes, rect: Rect): - covered_area = sum(sizes) +def __leftoverrow(covered_area, rect: Rect): width = covered_area / rect.dy return Rect(rect.x + width, rect.y, rect.dx - width, rect.dy) def __leftovercol(covered_area, rect: Rect) -> Rect: - height = covered_area / dx + height = covered_area / rect.dx return Rect(rect.x, rect.y + height, rect.dx, rect.dy - height) -def __leftover(sizes, rect: Rect): +def __leftover(sizes: List[numbers.Number], rect: Rect): covered_area = sum(sizes) - if rect.dx >=rect. dy: + if rect.dx >= rect.dy: return __leftoverrow(covered_area, rect) return __leftovercol(covered_area, rect) -def __find_split(sizes, x, y, dx, dy) -> int: +def __find_split(sizes: List[numbers.Number], rect: Rect) -> int: """ returns the index to split the sizes based on worst ratio to get the remaining and current space. For an example the area ratio of 5, 3, 2, 1 with the area of 90 * 50 >>> test_data = [2045.5, 1227.3, 818.2, 409.0 ] - >>> __find_split(test_data, 0, 0, 90, 50) + >>> __find_split(test_data, Rect(0, 0, 90, 50)) 1 The first two elements can be put into the first space and the rest needs to @@ -135,7 +134,7 @@ def worst(i: int) -> float: return max( [ max(rect.dx / rect.dy, rect.dy / rect.dx) - for rect in __layout(sizes[:i], x, y, dx, dy) + for rect in __layout(sizes[:i], rect) ] ) @@ -145,13 +144,13 @@ def worst(i: int) -> float: return len(sizes) - 1 -def __squarify(sizes, x, y, dx, dy) -> List[Rect]: +def __squarify(sizes: List[numbers.Number], rect: Rect) -> List[Rect]: """ calculates treemap rectangles using an algorithm based on Bruls, Huizing, van Wijk, "Squarified Treemaps" and "squarify": https://github.com/laserson/squarify - >>> __squarify([5, 1], 0, 0, 90, 50) + >>> __squarify([5, 1], Rect(0, 0, 90, 50)) [Rect(x=1, y=1, dx=73.0, dy=48.0), Rect(x=76.0, y=1, dx=13.0, dy=48.0)] Parameters: @@ -168,24 +167,23 @@ def __squarify(sizes, x, y, dx, dy) -> List[Rect]: Each dict in the returned list represents a single rectangle in the treemap. The order corresponds to the input order. """ - if len(sizes) == 0: return [] total_size = sum(sizes) - total_area = dx * dy + total_area = rect.dx * rect.dy sizes = list([size * total_area / total_size for size in sizes]) if len(sizes) == 1: - return __layout(sizes, x, y, dx, dy) + return __layout(sizes, rect) - i = __find_split(sizes, x, y, dx, dy) + i = __find_split(sizes, rect) current = sizes[:i] remaining = sizes[i:] - return __layout(current, x, y, dx, dy) + __squarify( - remaining, *__leftover(current, x, y, dx, dy) + return __layout(current, rect) + __squarify( + remaining, __leftover(current, rect) ) @@ -269,7 +267,7 @@ def treemap( if not title_color: title_color = _severity_class_colors sizes, label, color_keys = __transform_to_tree_data(data) - sizes = __squarify(sizes, 0, 0, width, height) + sizes = __squarify(sizes, Rect(0, 0, width, height)) elements = "" for i, d in enumerate(sizes): label_size_in_px = len(label[i]) * fontsize From 335aa706ab762c5a616f0bb9a5f35877f5eaace6 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Wed, 2 Dec 2020 08:44:18 +0100 Subject: [PATCH 09/10] use named format parameter in all svg templates To be more consistent and easier to read svg templates are using just named form parameter instead of only when > 3 and reusing. --- pheme/templatetags/charts/h_bar.py | 16 +++++++++------- pheme/templatetags/charts/treemap.py | 4 ++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pheme/templatetags/charts/h_bar.py b/pheme/templatetags/charts/h_bar.py index fc801fd0..660bfa9d 100644 --- a/pheme/templatetags/charts/h_bar.py +++ b/pheme/templatetags/charts/h_bar.py @@ -24,16 +24,16 @@ __ORIENTATION_LINE_TEMPLATE = """ - + """ __ORIENTATION_LINE_TEXT_TEMPLATE = """ -{} +{label} """ __BAR_ELEMENT_TEMPLATE = """ - + """ __BAR_TEMPLATE = """ @@ -113,10 +113,10 @@ def __add_orientation(i: int, orientation_lines="", orientation_labels=""): x_pos = i * orientation_basis / max_sum * max_width label = str(i * orientation_basis) orientation_lines += __ORIENTATION_LINE_TEMPLATE.format( - x_pos, bar_jump + 10 + x=x_pos, height=bar_jump + 10 ) orientation_labels += __ORIENTATION_LINE_TEXT_TEMPLATE.format( - x_pos, len(label), label + x=x_pos, width=len(label), label=label ) return orientation_lines, orientation_labels @@ -139,7 +139,9 @@ def __add_orientation(i: int, orientation_lines="", orientation_labels=""): for category, count in counts.items(): color = title_color.get(category) width = count / max_sum * max_width - elements += __BAR_ELEMENT_TEMPLATE.format(element_x, width, color) + elements += __BAR_ELEMENT_TEMPLATE.format( + x=element_x, width=width, color=color + ) element_x += width bars += __BAR_TEMPLATE.format( diff --git a/pheme/templatetags/charts/treemap.py b/pheme/templatetags/charts/treemap.py index 61387cd7..6d1d5256 100644 --- a/pheme/templatetags/charts/treemap.py +++ b/pheme/templatetags/charts/treemap.py @@ -98,12 +98,12 @@ def __layout(sizes: List[numbers.Number], rect: Rect) -> List[Rect]: return __layoutcol(sizes, rect) -def __leftoverrow(covered_area, rect: Rect): +def __leftoverrow(covered_area: float, rect: Rect): width = covered_area / rect.dy return Rect(rect.x + width, rect.y, rect.dx - width, rect.dy) -def __leftovercol(covered_area, rect: Rect) -> Rect: +def __leftovercol(covered_area: float, rect: Rect) -> Rect: height = covered_area / rect.dx return Rect(rect.x, rect.y + height, rect.dx, rect.dy - height) From db06229829fa911b12ef9a16eaa7dc63dc9141fb Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Wed, 2 Dec 2020 09:19:56 +0100 Subject: [PATCH 10/10] Apply suggestions from code review indicate Rect return value Co-authored-by: Jaspar L. --- pheme/templatetags/charts/treemap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pheme/templatetags/charts/treemap.py b/pheme/templatetags/charts/treemap.py index 6d1d5256..5177f10a 100644 --- a/pheme/templatetags/charts/treemap.py +++ b/pheme/templatetags/charts/treemap.py @@ -98,7 +98,7 @@ def __layout(sizes: List[numbers.Number], rect: Rect) -> List[Rect]: return __layoutcol(sizes, rect) -def __leftoverrow(covered_area: float, rect: Rect): +def __leftoverrow(covered_area: float, rect: Rect) -> Rect: width = covered_area / rect.dy return Rect(rect.x + width, rect.y, rect.dx - width, rect.dy) @@ -108,7 +108,7 @@ def __leftovercol(covered_area: float, rect: Rect) -> Rect: return Rect(rect.x, rect.y + height, rect.dx, rect.dy - height) -def __leftover(sizes: List[numbers.Number], rect: Rect): +def __leftover(sizes: List[numbers.Number], rect: Rect) -> Rect: covered_area = sum(sizes) if rect.dx >= rect.dy: return __leftoverrow(covered_area, rect)