From 617e577ecf76fa8273934ff047013ebae97a8cfc Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 28 Sep 2019 18:02:46 +0300 Subject: [PATCH 01/33] FIX(build): py2 needs pinning networkx-2.2 --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bd7883f4..d3dfec84 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,10 @@ author_email='huyng@yahoo-inc.com', url='http://github.com/yahoo/graphkit', packages=['graphkit'], - install_requires=['networkx'], + install_requires=[ + "networkx; python_version >= '3.5'", + "networkx == 2.2; python_version < '3.5'", + ], extras_require={ 'plot': ['pydot', 'matplotlib'] }, From f58d14865f45f07e5125aed9b0a3f151073fa122 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Mon, 30 Sep 2019 00:36:57 +0300 Subject: [PATCH 02/33] FIX(#13): BUG in plot-diagram writtin from PY2 era, were writing in text-mode in PY3. and failing as encoding error. --- graphkit/network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/graphkit/network.py b/graphkit/network.py index 0df3ddf8..24c3ac37 100644 --- a/graphkit/network.py +++ b/graphkit/network.py @@ -422,8 +422,8 @@ def get_node_name(a): # save plot if filename: - basename, ext = os.path.splitext(filename) - with open(filename, "w") as fh: + _basename, ext = os.path.splitext(filename) + with open(filename, "wb") as fh: if ext.lower() == ".png": fh.write(g.create_png()) elif ext.lower() == ".dot": From c75a2c0cf571430161ae1bf28f84f1c982c7d760 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Mon, 30 Sep 2019 00:20:40 +0300 Subject: [PATCH 03/33] doc(#13): sample code to plot workflow diagram in intro --- README.md | 13 +++++++++++++ docs/source/index.rst | 12 ++++++++++++ 2 files changed, 25 insertions(+) diff --git a/README.md b/README.md index 0e1e95a4..af414020 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,19 @@ print(out) As you can see, any function can be used as an operation in GraphKit, even ones imported from system modules! +For debugging, you may plot the workflow with one of these methods: + +```python + graph.net.plot(show=True) # open a matplotlib window + graph.net.plot("path/to/workflow.png") # supported files: .png .dot .jpg .jpeg .pdf .svg +``` + +> **NOTE**: For plots, `graphviz` must be in your PATH, and `pydot` & `matplotlib` python packages installed. +> You may install both when installing *graphkit* with its `plot` extras: +> ```python +> pip install graphkit[plot] +> ``` + # License Code licensed under the Apache License, Version 2.0 license. See LICENSE file for terms. diff --git a/docs/source/index.rst b/docs/source/index.rst index 5c5e505c..6b5cb690 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -69,6 +69,18 @@ Here's a Python script with an example GraphKit computation graph that produces As you can see, any function can be used as an operation in GraphKit, even ones imported from system modules! +For debugging, you may plot the workflow with one of these methods:: + + graph.net.plot(show=True) # open a matplotlib window + graph.net.plot("path/to/workflow.png") # supported files: .png .dot .jpg .jpeg .pdf .svg + +.. NOTE:: + For plots, ``graphviz`` must be in your PATH, and ``pydot` & ``matplotlib`` python packages installed. + You may install both when installing *graphkit* with its `plot` extras:: + + pip install graphkit[plot] + + License ------- From a005bd6b2c209c91871ea9ea266dba8e2ec7ab86 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Mon, 30 Sep 2019 01:06:15 +0300 Subject: [PATCH 04/33] enh(plot): provide help msg on supported file-exts --- graphkit/network.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/graphkit/network.py b/graphkit/network.py index 24c3ac37..33b8363e 100644 --- a/graphkit/network.py +++ b/graphkit/network.py @@ -435,7 +435,10 @@ def get_node_name(a): elif ext.lower() == ".svg": fh.write(g.create_svg()) else: - raise Exception("Unknown file format for saving graph: %s" % ext) + raise Exception( + "Unknown file format for saving graph: %s" + " File extensions must be one of: .png .dot .jpg .jpeg .pdf .svg" + % ext) # display graph via matplotlib if show: From 32409f6342b5dc136c1c6d150171a2abe9912a3e Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Thu, 3 Oct 2019 20:34:44 +0300 Subject: [PATCH 05/33] enh(build): replace numpy with pytest... numpy was used just for its assert_raise --- setup.py | 2 +- test/test_graphkit.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index d3dfec84..d4b7c378 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ extras_require={ 'plot': ['pydot', 'matplotlib'] }, - tests_require=['numpy'], + tests_require=['pytest'], license='Apache-2.0', keywords=['graph', 'computation graph', 'DAG', 'directed acyclical graph'], classifiers=[ diff --git a/test/test_graphkit.py b/test/test_graphkit.py index bd97b317..7db2e973 100644 --- a/test/test_graphkit.py +++ b/test/test_graphkit.py @@ -6,7 +6,8 @@ from pprint import pprint from operator import add -from numpy.testing import assert_raises + +import pytest import graphkit.network as network import graphkit.modifiers as modifiers @@ -180,9 +181,10 @@ def test_pruning_raises_for_bad_output(): # Request two outputs we can compute and one we can't compute. Assert # that this raises a ValueError. - assert_raises(ValueError, net, {'a': 1, 'b': 2, 'c': 3, 'd': 4}, - outputs=['sum1', 'sum3', 'sum4']) - + with pytest.raises(ValueError) as exinfo: + net({'a': 1, 'b': 2, 'c': 3, 'd': 4}, + outputs=['sum1', 'sum3', 'sum4']) + assert exinfo.match('sum4') def test_optional(): # Test that optional() needs work as expected. From f606ed1f147cba8956012c0a35475af052bcd361 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Thu, 3 Oct 2019 20:35:56 +0300 Subject: [PATCH 06/33] feat(build): add pip-extras [test] --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d4b7c378..51d606fc 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,8 @@ "networkx == 2.2; python_version < '3.5'", ], extras_require={ - 'plot': ['pydot', 'matplotlib'] + 'plot': ['pydot', 'matplotlib'], + 'test': ['pydot', 'matplotlib', 'pytest'], }, tests_require=['pytest'], license='Apache-2.0', From cd1370b9c6ee022ff056e020633b15c1089fbf87 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Mon, 30 Sep 2019 01:20:28 +0300 Subject: [PATCH 07/33] TEST(plot,ci): test plotting; pip install extras in Travis --- .travis.yml | 7 ++++++- test/test_graphkit.py | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d8657a8f..588f64a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,11 +4,16 @@ python: - "2.7" - "3.4" - "3.5" +addons: + apt: + packages: + - graphviz + install: - pip install Sphinx sphinx_rtd_theme codecov packaging - "python -c $'import os, packaging.version as version\\nv = version.parse(os.environ.get(\"TRAVIS_TAG\", \"1.0\")).public\\nwith open(\"VERSION\", \"w\") as f: f.write(v)'" - - python setup.py install + - pip install .[plot] - cd docs - make clean html - cd .. diff --git a/test/test_graphkit.py b/test/test_graphkit.py index bd97b317..b4681121 100644 --- a/test/test_graphkit.py +++ b/test/test_graphkit.py @@ -3,6 +3,10 @@ import math import pickle +import os.path as osp +import shutil +import tempfile + from pprint import pprint from operator import add @@ -317,6 +321,22 @@ def infer(i): pool.close() +def test_plotting(): + sum_op1 = operation(name='sum_op1', needs=['a', 'b'], provides='sum1')(add) + sum_op2 = operation(name='sum_op2', needs=['a', 'b'], provides='sum2')(add) + sum_op3 = operation(name='sum_op3', needs=['sum1', 'c'], provides='sum3')(add) + net1 = compose(name='my network 1')(sum_op1, sum_op2, sum_op3) + + for ext in ".png .dot .jpg .jpeg .pdf .svg".split(): + tdir = tempfile.mkdtemp(suffix=ext) + png_file = osp.join(tdir, "workflow.png") + net1.net.plot(png_file) + try: + assert osp.exists(png_file) + finally: + shutil.rmtree(tdir, ignore_errors=True) + + #################################### # Backwards compatibility #################################### From f6766627bb5e5f0226d3625cd4d6906655e7ff56 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Mon, 30 Sep 2019 01:38:41 +0300 Subject: [PATCH 08/33] fix(plot): don't create file on unsupported formats thanks to @andres-fr. --- graphkit/network.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/graphkit/network.py b/graphkit/network.py index 33b8363e..9280a891 100644 --- a/graphkit/network.py +++ b/graphkit/network.py @@ -422,23 +422,23 @@ def get_node_name(a): # save plot if filename: + supported_plot_formaters = { + ".png": g.create_png, + ".dot": g.to_string, + ".jpg": g.create_jpeg, + ".jpeg": g.create_jpeg, + ".pdf": g.create_pdf, + ".svg": g.create_svg, + } _basename, ext = os.path.splitext(filename) + plot_formater = supported_plot_formaters.get(ext.lower()) + if not plot_formater: + raise Exception( + "Unknown file format for saving graph: %s" + " File extensions must be one of: .png .dot .jpg .jpeg .pdf .svg" + % ext) with open(filename, "wb") as fh: - if ext.lower() == ".png": - fh.write(g.create_png()) - elif ext.lower() == ".dot": - fh.write(g.to_string()) - elif ext.lower() in [".jpg", ".jpeg"]: - fh.write(g.create_jpeg()) - elif ext.lower() == ".pdf": - fh.write(g.create_pdf()) - elif ext.lower() == ".svg": - fh.write(g.create_svg()) - else: - raise Exception( - "Unknown file format for saving graph: %s" - " File extensions must be one of: .png .dot .jpg .jpeg .pdf .svg" - % ext) + fh.write(plot_formater()) # display graph via matplotlib if show: From 65d1816b39a08fc9dfd12910d68e3ebcb68aba06 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Mon, 30 Sep 2019 01:54:03 +0300 Subject: [PATCH 09/33] enh(plot.TC): expose supported writers and TC on them --- graphkit/network.py | 32 ++++++++++++++++++-------------- test/test_graphkit.py | 11 ++++++++++- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/graphkit/network.py b/graphkit/network.py index 9280a891..f0c7444a 100644 --- a/graphkit/network.py +++ b/graphkit/network.py @@ -375,6 +375,17 @@ def _compute_sequential_method(self, named_inputs, outputs): return {k: cache[k] for k in iter(cache) if k in outputs} + @staticmethod + def supported_plot_writers(): + return { + ".png": lambda gplot: gplot.create_png(), + ".dot": lambda gplot: gplot.to_string(), + ".jpg": lambda gplot: gplot.create_jpeg(), + ".jpeg": lambda gplot: gplot.create_jpeg(), + ".pdf": lambda gplot: gplot.create_pdf(), + ".svg": lambda gplot: gplot.create_svg(), + } + def plot(self, filename=None, show=False): """ Plot the graph. @@ -422,23 +433,16 @@ def get_node_name(a): # save plot if filename: - supported_plot_formaters = { - ".png": g.create_png, - ".dot": g.to_string, - ".jpg": g.create_jpeg, - ".jpeg": g.create_jpeg, - ".pdf": g.create_pdf, - ".svg": g.create_svg, - } _basename, ext = os.path.splitext(filename) - plot_formater = supported_plot_formaters.get(ext.lower()) - if not plot_formater: - raise Exception( + writers = Network.supported_plot_writers() + plot_writer = Network.supported_plot_writers().get(ext.lower()) + if not plot_writer: + raise ValueError( "Unknown file format for saving graph: %s" - " File extensions must be one of: .png .dot .jpg .jpeg .pdf .svg" - % ext) + " File extensions must be one of: %s" + % (ext, ' '.join(writers))) with open(filename, "wb") as fh: - fh.write(plot_formater()) + fh.write(plot_writer(g)) # display graph via matplotlib if show: diff --git a/test/test_graphkit.py b/test/test_graphkit.py index b4681121..bdd0ab37 100644 --- a/test/test_graphkit.py +++ b/test/test_graphkit.py @@ -327,7 +327,7 @@ def test_plotting(): sum_op3 = operation(name='sum_op3', needs=['sum1', 'c'], provides='sum3')(add) net1 = compose(name='my network 1')(sum_op1, sum_op2, sum_op3) - for ext in ".png .dot .jpg .jpeg .pdf .svg".split(): + for ext in network.Network.supported_plot_writers(): tdir = tempfile.mkdtemp(suffix=ext) png_file = osp.join(tdir, "workflow.png") net1.net.plot(png_file) @@ -335,6 +335,15 @@ def test_plotting(): assert osp.exists(png_file) finally: shutil.rmtree(tdir, ignore_errors=True) + try: + net1.net.plot('bad.format') + assert False, "Should had failed writting arbitrary file format!" + except ValueError as ex: + assert "Unknown file format" in str(ex) + + ## Check help msg lists all siupported formats + for ext in network.Network.supported_plot_writers(): + assert ext in str(ex) #################################### From 4e55b30310d04660c53553bb9e2e87669271b9bd Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Thu, 3 Oct 2019 20:54:53 +0300 Subject: [PATCH 10/33] enh(build,ci): use pytest in travis --- .travis.yml | 4 ++-- setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index d8657a8f..bb7d875e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,13 +8,13 @@ python: install: - pip install Sphinx sphinx_rtd_theme codecov packaging - "python -c $'import os, packaging.version as version\\nv = version.parse(os.environ.get(\"TRAVIS_TAG\", \"1.0\")).public\\nwith open(\"VERSION\", \"w\") as f: f.write(v)'" - - python setup.py install + - pip install -e .[test] - cd docs - make clean html - cd .. script: - - python setup.py nosetests --with-coverage --cover-package=graphkit + - pytest -v --cov=graphkit deploy: provider: pypi diff --git a/setup.py b/setup.py index 51d606fc..c8d231a5 100644 --- a/setup.py +++ b/setup.py @@ -34,9 +34,9 @@ ], extras_require={ 'plot': ['pydot', 'matplotlib'], - 'test': ['pydot', 'matplotlib', 'pytest'], + 'test': ['pydot', 'matplotlib', 'pytest', "pytest-cov"], }, - tests_require=['pytest'], + tests_require=['pytest', "pytest-cov"], license='Apache-2.0', keywords=['graph', 'computation graph', 'DAG', 'directed acyclical graph'], classifiers=[ From 47b50f6bcd9734d20314afaa3f829739137e08e6 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 5 Oct 2019 02:03:36 +0300 Subject: [PATCH 11/33] fix(plot): NetOp did not return pydot instance --- graphkit/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graphkit/base.py b/graphkit/base.py index 1c04e8d5..631c66ab 100644 --- a/graphkit/base.py +++ b/graphkit/base.py @@ -172,7 +172,7 @@ def set_execution_method(self, method): self._execution_method = method def plot(self, filename=None, show=False): - self.net.plot(filename=filename, show=show) + return self.net.plot(filename=filename, show=show) def __getstate__(self): state = Operation.__getstate__(self) From b1d02a1918da028fc163a1cc132280e2468f8560 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 5 Oct 2019 02:10:22 +0300 Subject: [PATCH 12/33] refact(plot): extract plot function out of Network class... to use it also on stary DAGs. Keep delegation. --- graphkit/network.py | 158 ++++++++++++++++++++++-------------------- test/test_graphkit.py | 4 +- 2 files changed, 86 insertions(+), 76 deletions(-) diff --git a/graphkit/network.py b/graphkit/network.py index f0c7444a..3f9cbdaa 100644 --- a/graphkit/network.py +++ b/graphkit/network.py @@ -375,84 +375,14 @@ def _compute_sequential_method(self, named_inputs, outputs): return {k: cache[k] for k in iter(cache) if k in outputs} - @staticmethod - def supported_plot_writers(): - return { - ".png": lambda gplot: gplot.create_png(), - ".dot": lambda gplot: gplot.to_string(), - ".jpg": lambda gplot: gplot.create_jpeg(), - ".jpeg": lambda gplot: gplot.create_jpeg(), - ".pdf": lambda gplot: gplot.create_pdf(), - ".svg": lambda gplot: gplot.create_svg(), - } - def plot(self, filename=None, show=False): """ - Plot the graph. - - params: - :param str filename: - Write the output to a png, pdf, or graphviz dot file. The extension - controls the output format. - - :param boolean show: - If this is set to True, use matplotlib to show the graph diagram - (Default: False) - - :returns: - An instance of the pydot graph + Plot a *Graphviz* graph and return it, if no other argument provided. + Supported arguments: filename, show + See :func:`network.plot_graph()` """ - import pydot - import matplotlib.pyplot as plt - import matplotlib.image as mpimg - - assert self.graph is not None - - def get_node_name(a): - if isinstance(a, DataPlaceholderNode): - return a - return a.name - - g = pydot.Dot(graph_type="digraph") - - # draw nodes - for nx_node in self.graph.nodes(): - if isinstance(nx_node, DataPlaceholderNode): - node = pydot.Node(name=nx_node, shape="rect") - else: - node = pydot.Node(name=nx_node.name, shape="circle") - g.add_node(node) - - # draw edges - for src, dst in self.graph.edges(): - src_name = get_node_name(src) - dst_name = get_node_name(dst) - edge = pydot.Edge(src=src_name, dst=dst_name) - g.add_edge(edge) - - # save plot - if filename: - _basename, ext = os.path.splitext(filename) - writers = Network.supported_plot_writers() - plot_writer = Network.supported_plot_writers().get(ext.lower()) - if not plot_writer: - raise ValueError( - "Unknown file format for saving graph: %s" - " File extensions must be one of: %s" - % (ext, ' '.join(writers))) - with open(filename, "wb") as fh: - fh.write(plot_writer(g)) - - # display graph via matplotlib - if show: - png = g.create_png() - sio = StringIO(png) - img = mpimg.imread(sio) - plt.imshow(img, aspect="equal") - plt.show() - - return g + return plot_graph(self.graph, filename=filename, show=show) def ready_to_schedule_operation(op, has_executed, graph): @@ -501,3 +431,83 @@ def get_data_node(name, graph): if node == name and isinstance(node, DataPlaceholderNode): return node return None + + +def supported_plot_writers(): + return { + ".png": lambda gplot: gplot.create_png(), + ".dot": lambda gplot: gplot.to_string(), + ".jpg": lambda gplot: gplot.create_jpeg(), + ".jpeg": lambda gplot: gplot.create_jpeg(), + ".pdf": lambda gplot: gplot.create_pdf(), + ".svg": lambda gplot: gplot.create_svg(), + } + + +def plot_graph(graph, filename=None, show=False): + """ + Plot a *Graphviz* graph and return it, if no other argument provided. + + :param graph: + what to plot + :param str filename: + Write the output to a png, pdf, or graphviz dot file. The extension + controls the output format. + :param boolean show: + If this is set to True, use matplotlib to show the graph diagram + (Default: False) + + :returns: + An instance of the pydot graph + + """ + import pydot + import matplotlib.pyplot as plt + import matplotlib.image as mpimg + + assert graph is not None + + def get_node_name(a): + if isinstance(a, DataPlaceholderNode): + return a + return a.name + + g = pydot.Dot(graph_type="digraph") + + # draw nodes + for nx_node in graph.nodes(): + if isinstance(nx_node, DataPlaceholderNode): + node = pydot.Node(name=nx_node, shape="rect") + else: + node = pydot.Node(name=nx_node.name, shape="circle") + g.add_node(node) + + # draw edges + for src, dst in graph.edges(): + src_name = get_node_name(src) + dst_name = get_node_name(dst) + edge = pydot.Edge(src=src_name, dst=dst_name) + g.add_edge(edge) + + # save plot + if filename: + _basename, ext = os.path.splitext(filename) + writers = Network.supported_plot_writers() + plot_writer = Network.supported_plot_writers().get(ext.lower()) + if not plot_writer: + raise ValueError( + "Unknown file format for saving graph: %s" + " File extensions must be one of: %s" + % (ext, ' '.join(writers))) + with open(filename, "wb") as fh: + fh.write(plot_writer(g)) + + # display graph via matplotlib + if show: + png = g.create_png() + sio = StringIO(png) + img = mpimg.imread(sio) + plt.imshow(img, aspect="equal") + plt.show() + + return g diff --git a/test/test_graphkit.py b/test/test_graphkit.py index bdd0ab37..bb08cf15 100644 --- a/test/test_graphkit.py +++ b/test/test_graphkit.py @@ -327,7 +327,7 @@ def test_plotting(): sum_op3 = operation(name='sum_op3', needs=['sum1', 'c'], provides='sum3')(add) net1 = compose(name='my network 1')(sum_op1, sum_op2, sum_op3) - for ext in network.Network.supported_plot_writers(): + for ext in network.supported_plot_writers(): tdir = tempfile.mkdtemp(suffix=ext) png_file = osp.join(tdir, "workflow.png") net1.net.plot(png_file) @@ -342,7 +342,7 @@ def test_plotting(): assert "Unknown file format" in str(ex) ## Check help msg lists all siupported formats - for ext in network.Network.supported_plot_writers(): + for ext in network.supported_plot_writers(): assert ext in str(ex) From c11af2ae384340b675e8d5aec0b48c28250f8757 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 5 Oct 2019 02:31:33 +0300 Subject: [PATCH 13/33] fix(plot): matplotlib plot was failing in PY3 due IO io misuse --- graphkit/network.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/graphkit/network.py b/graphkit/network.py index 3f9cbdaa..265b048f 100644 --- a/graphkit/network.py +++ b/graphkit/network.py @@ -1,8 +1,9 @@ # Copyright 2016, Yahoo Inc. # Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms. -import time +import io import os +import time import networkx as nx from io import StringIO @@ -505,7 +506,7 @@ def get_node_name(a): # display graph via matplotlib if show: png = g.create_png() - sio = StringIO(png) + sio = io.BytesIO(png) img = mpimg.imread(sio) plt.imshow(img, aspect="equal") plt.show() From 344490be8cafcc466674bc986fa7118bd68c99d7 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 5 Oct 2019 03:04:06 +0300 Subject: [PATCH 14/33] FEAT(plot): overlay Execution STEPS on diagrams --- graphkit/network.py | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/graphkit/network.py b/graphkit/network.py index 265b048f..035facd5 100644 --- a/graphkit/network.py +++ b/graphkit/network.py @@ -6,7 +6,6 @@ import time import networkx as nx -from io import StringIO from .base import Operation @@ -383,7 +382,7 @@ def plot(self, filename=None, show=False): Supported arguments: filename, show See :func:`network.plot_graph()` """ - return plot_graph(self.graph, filename=filename, show=show) + return plot_graph(self.graph, filename, show, self.steps) def ready_to_schedule_operation(op, has_executed, graph): @@ -445,9 +444,9 @@ def supported_plot_writers(): } -def plot_graph(graph, filename=None, show=False): +def plot_graph(graph, filename=None, show=False, steps=None): """ - Plot a *Graphviz* graph and return it, if no other argument provided. + Plot a *Graphviz* graph/steps and return it, if no other argument provided. :param graph: what to plot @@ -457,6 +456,8 @@ def plot_graph(graph, filename=None, show=False): :param boolean show: If this is set to True, use matplotlib to show the graph diagram (Default: False) + :param steps: + a list of nodes & instructions to overlay on the diagram :returns: An instance of the pydot graph @@ -469,18 +470,23 @@ def plot_graph(graph, filename=None, show=False): assert graph is not None def get_node_name(a): - if isinstance(a, DataPlaceholderNode): - return a - return a.name + if isinstance(a, Operation): + return a.name + return a g = pydot.Dot(graph_type="digraph") # draw nodes - for nx_node in graph.nodes(): + for nx_node in graph.nodes: + kw = {} if isinstance(nx_node, DataPlaceholderNode): - node = pydot.Node(name=nx_node, shape="rect") + if nx_node in steps: + kw = {'color': 'red', 'style': 'bold'} + node = pydot.Node(name=nx_node, shape="rect", **kw) else: - node = pydot.Node(name=nx_node.name, shape="circle") + if nx_node in steps: + kw = {'style': 'bold'} + node = pydot.Node(name=nx_node.name, shape="circle", **kw) g.add_node(node) # draw edges @@ -490,6 +496,18 @@ def get_node_name(a): edge = pydot.Edge(src=src_name, dst=dst_name) g.add_edge(edge) + # draw steps sequence + if steps and len(steps) > 1: + it1 = iter(steps) + it2 = iter(steps); next(it2) + for i, (src, dst) in enumerate(zip(it1, it2), 1): + src_name = get_node_name(src) + dst_name = get_node_name(dst) + edge = pydot.Edge( + src=src_name, dst=dst_name, label=str(i), style="dotted", + penwidth='2') + g.add_edge(edge) + # save plot if filename: _basename, ext = os.path.splitext(filename) From 23ef81e3cf08af2074a14195c101da814e408097 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 5 Oct 2019 03:44:01 +0300 Subject: [PATCH 15/33] ENH(plot): +inputs, +outputs, +solution modify plotting (see #13 for an example): --- graphkit/base.py | 5 +++-- graphkit/network.py | 39 ++++++++++++++++++++++++++++++++------- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/graphkit/base.py b/graphkit/base.py index 631c66ab..608a96d0 100644 --- a/graphkit/base.py +++ b/graphkit/base.py @@ -171,8 +171,9 @@ def set_execution_method(self, method): assert method in options self._execution_method = method - def plot(self, filename=None, show=False): - return self.net.plot(filename=filename, show=show) + def plot(self, filename=None, show=False, + inputs=None, outputs=None, solution=None): + return self.net.plot(filename, show, inputs, outputs, solution) def __getstate__(self): state = Operation.__getstate__(self) diff --git a/graphkit/network.py b/graphkit/network.py index 035facd5..9fdf71c2 100644 --- a/graphkit/network.py +++ b/graphkit/network.py @@ -375,14 +375,15 @@ def _compute_sequential_method(self, named_inputs, outputs): return {k: cache[k] for k in iter(cache) if k in outputs} - def plot(self, filename=None, show=False): + def plot(self, filename=None, show=False, + inputs=None, outputs=None, solution=None): """ Plot a *Graphviz* graph and return it, if no other argument provided. - Supported arguments: filename, show See :func:`network.plot_graph()` """ - return plot_graph(self.graph, filename, show, self.steps) + return plot_graph(self.graph, filename, show, self.steps, + inputs, outputs, solution) def ready_to_schedule_operation(op, has_executed, graph): @@ -444,7 +445,8 @@ def supported_plot_writers(): } -def plot_graph(graph, filename=None, show=False, steps=None): +def plot_graph(graph, filename=None, show=False, steps=None, + inputs=None, outputs=None, solution=None): """ Plot a *Graphviz* graph/steps and return it, if no other argument provided. @@ -458,6 +460,13 @@ def plot_graph(graph, filename=None, show=False, steps=None): (Default: False) :param steps: a list of nodes & instructions to overlay on the diagram + :param inputs: + an optional list, any nodes in there are plotted as `"house" + `_ + :param outputs: + an optional list, any nodes in there are plotted as `"invhouse" + :param outputs: + an optional dict, any values in there are included in the node-name :returns: An instance of the pydot graph @@ -480,9 +489,25 @@ def get_node_name(a): for nx_node in graph.nodes: kw = {} if isinstance(nx_node, DataPlaceholderNode): + # Only DeleteInstructions data in steps. if nx_node in steps: kw = {'color': 'red', 'style': 'bold'} - node = pydot.Node(name=nx_node, shape="rect", **kw) + + # SHAPE change if in inputs/outputs. + shape="rect" + if inputs and nx_node in inputs: + shape="invhouse" + if outputs and nx_node in outputs: + if inputs and nx_node in inputs: + shape="polygon" + else: + shape="house" + + # LABEL change from solution. + name = str(nx_node) + if solution and nx_node in solution: + name = "%s: %s" % (nx_node, solution.get(nx_node)) + node = pydot.Node(name=nx_node, label=name, shape=shape, **kw) else: if nx_node in steps: kw = {'style': 'bold'} @@ -511,8 +536,8 @@ def get_node_name(a): # save plot if filename: _basename, ext = os.path.splitext(filename) - writers = Network.supported_plot_writers() - plot_writer = Network.supported_plot_writers().get(ext.lower()) + writers = supported_plot_writers() + plot_writer = supported_plot_writers().get(ext.lower()) if not plot_writer: raise ValueError( "Unknown file format for saving graph: %s" From 4e8601ce9c4a9372ed398cd6402c2a83bb059f25 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 5 Oct 2019 12:43:10 +0300 Subject: [PATCH 16/33] refact(plot.TC): move plot tests to early beggining --- test/test_graphkit.py | 50 +++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/test/test_graphkit.py b/test/test_graphkit.py index bb08cf15..dff4e655 100644 --- a/test/test_graphkit.py +++ b/test/test_graphkit.py @@ -107,6 +107,31 @@ def test_network_deep_merge(): pprint(net3({'a': 1, 'b': 2, 'c': 4})) +def test_plotting(): + sum_op1 = operation(name='sum_op1', needs=['a', 'b'], provides='sum1')(add) + sum_op2 = operation(name='sum_op2', needs=['a', 'b'], provides='sum2')(add) + sum_op3 = operation(name='sum_op3', needs=['sum1', 'c'], provides='sum3')(add) + net1 = compose(name='my network 1')(sum_op1, sum_op2, sum_op3) + + for ext in network.supported_plot_writers(): + tdir = tempfile.mkdtemp(suffix=ext) + png_file = osp.join(tdir, "workflow.png") + net1.net.plot(png_file) + try: + assert osp.exists(png_file) + finally: + shutil.rmtree(tdir, ignore_errors=True) + try: + net1.net.plot('bad.format') + assert False, "Should had failed writting arbitrary file format!" + except ValueError as ex: + assert "Unknown file format" in str(ex) + + ## Check help msg lists all siupported formats + for ext in network.supported_plot_writers(): + assert ext in str(ex) + + def test_input_based_pruning(): # Tests to make sure we don't need to pass graph inputs if we're provided # with data further downstream in the graph as an input. @@ -321,31 +346,6 @@ def infer(i): pool.close() -def test_plotting(): - sum_op1 = operation(name='sum_op1', needs=['a', 'b'], provides='sum1')(add) - sum_op2 = operation(name='sum_op2', needs=['a', 'b'], provides='sum2')(add) - sum_op3 = operation(name='sum_op3', needs=['sum1', 'c'], provides='sum3')(add) - net1 = compose(name='my network 1')(sum_op1, sum_op2, sum_op3) - - for ext in network.supported_plot_writers(): - tdir = tempfile.mkdtemp(suffix=ext) - png_file = osp.join(tdir, "workflow.png") - net1.net.plot(png_file) - try: - assert osp.exists(png_file) - finally: - shutil.rmtree(tdir, ignore_errors=True) - try: - net1.net.plot('bad.format') - assert False, "Should had failed writting arbitrary file format!" - except ValueError as ex: - assert "Unknown file format" in str(ex) - - ## Check help msg lists all siupported formats - for ext in network.supported_plot_writers(): - assert ext in str(ex) - - #################################### # Backwards compatibility #################################### From 834a8b0784d3ac58df2e4530b76cbbc4681a1582 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 5 Oct 2019 12:44:33 +0300 Subject: [PATCH 17/33] doc(plot): tell supported formats in doctest, +TC --- graphkit/network.py | 4 ++-- test/test_graphkit.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/graphkit/network.py b/graphkit/network.py index 9fdf71c2..987629f3 100644 --- a/graphkit/network.py +++ b/graphkit/network.py @@ -453,8 +453,8 @@ def plot_graph(graph, filename=None, show=False, steps=None, :param graph: what to plot :param str filename: - Write the output to a png, pdf, or graphviz dot file. The extension - controls the output format. + Write the output to a file. + The extension must be one of: ``.png .dot .jpg .jpeg .pdf .svg`` :param boolean show: If this is set to True, use matplotlib to show the graph diagram (Default: False) diff --git a/test/test_graphkit.py b/test/test_graphkit.py index dff4e655..826719d6 100644 --- a/test/test_graphkit.py +++ b/test/test_graphkit.py @@ -132,6 +132,11 @@ def test_plotting(): assert ext in str(ex) +def test_plotting_docstring(): + for ext in network.supported_plot_writers(): + assert ext in network.plot_graph.__doc__ + + def test_input_based_pruning(): # Tests to make sure we don't need to pass graph inputs if we're provided # with data further downstream in the graph as an input. From c2e28a405a46ee08a4ae5f20d13811f01759dcaf Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 5 Oct 2019 12:46:10 +0300 Subject: [PATCH 18/33] doc(plot): add legend & example; docstring in netop.plot() --- graphkit/base.py | 5 +++++ graphkit/network.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/graphkit/base.py b/graphkit/base.py index 608a96d0..512a95bc 100644 --- a/graphkit/base.py +++ b/graphkit/base.py @@ -173,6 +173,11 @@ def set_execution_method(self, method): def plot(self, filename=None, show=False, inputs=None, outputs=None, solution=None): + """ + Plot a *Graphviz* graph and return it, if no other argument provided. + + See :func:`network.plot_graph()` for arguments, legend, and example code. + """ return self.net.plot(filename, show, inputs, outputs, solution) def __getstate__(self): diff --git a/graphkit/network.py b/graphkit/network.py index 987629f3..1f7d81e9 100644 --- a/graphkit/network.py +++ b/graphkit/network.py @@ -450,6 +450,23 @@ def plot_graph(graph, filename=None, show=False, steps=None, """ Plot a *Graphviz* graph/steps and return it, if no other argument provided. + Legend: + + NODES: + + - **circle**: function + - **house**: input (given) + - **inversed-house**: output (asked) + - **polygon**: given both as input & asked as output (what?) + - **square**: intermediate data (neither given nor asked) + - **red frame**: delete-instruction (to free up memory) + + ARROWS + + - **solid black arrows**: dependencies (target ``need`` source, + sources ``provide`` target) + - **green-dotted arrows**: execution steps labeled in succession + :param graph: what to plot :param str filename: @@ -471,6 +488,18 @@ def plot_graph(graph, filename=None, show=False, steps=None, :returns: An instance of the pydot graph + **Example:** + + >>> netop = compose(name="netop")( + ... operation(name="add", needs=["a", "b1"], provides=["ab1"])(add), + ... operation(name="sub", needs=["a", optional("b2")], provides=["ab2"])(lambda a, b=1: a-b), + ... operation(name="abb", needs=["ab1", "ab2"], provides=["asked"])(add), + ... ) + + >>> inputs = {'a': 1, 'b1': 2} + >>> solution=netop(inputs) + >>> netop.plot('plot.svg', inputs=inputs, solution=solution, outputs=['asked', 'b1']); + """ import pydot import matplotlib.pyplot as plt From e38c8ad72f2c21e8ca923199f4be10b88794f997 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 5 Oct 2019 13:54:49 +0300 Subject: [PATCH 19/33] enh(plot): mark optional "needs" --- graphkit/network.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/graphkit/network.py b/graphkit/network.py index 1f7d81e9..e06427c2 100644 --- a/graphkit/network.py +++ b/graphkit/network.py @@ -463,8 +463,9 @@ def plot_graph(graph, filename=None, show=False, steps=None, ARROWS - - **solid black arrows**: dependencies (target ``need`` source, - sources ``provide`` target) + - **solid black arrows**: dependencies (source-data are``need``\ed + by target-operations, sources-operations ``provide`` target-data) + - **dashed black arrows**: optional needs - **green-dotted arrows**: execution steps labeled in succession :param graph: @@ -547,7 +548,12 @@ def get_node_name(a): for src, dst in graph.edges(): src_name = get_node_name(src) dst_name = get_node_name(dst) - edge = pydot.Edge(src=src_name, dst=dst_name) + kw = {} + if isinstance(dst, Operation) and any(n == src + and isinstance(n, optional) + for n in dst.needs): + kw["style"] = "dashed" + edge = pydot.Edge(src=src_name, dst=dst_name, **kw) g.add_edge(edge) # draw steps sequence From d855bf688f607e6a4b33961b751b68d53dc5f2b7 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 5 Oct 2019 13:55:10 +0300 Subject: [PATCH 20/33] ENH(plot): visual enhamcents on nodes & edges --- graphkit/network.py | 58 +++++++++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/graphkit/network.py b/graphkit/network.py index e06427c2..1c246530 100644 --- a/graphkit/network.py +++ b/graphkit/network.py @@ -7,7 +7,8 @@ import networkx as nx -from .base import Operation +from .base import Operation, NetworkOperation +from .modifiers import optional class DataPlaceholderNode(str): @@ -380,7 +381,7 @@ def plot(self, filename=None, show=False, """ Plot a *Graphviz* graph and return it, if no other argument provided. - See :func:`network.plot_graph()` + See :func:`network.plot_graph()` for arguments, legend, and example code. """ return plot_graph(self.graph, filename, show, self.steps, inputs, outputs, solution) @@ -452,14 +453,18 @@ def plot_graph(graph, filename=None, show=False, steps=None, Legend: + NODES: - **circle**: function - - **house**: input (given) - - **inversed-house**: output (asked) + - **oval**: subgraph function + - **house**: given input + - **inversed-house**: asked output - **polygon**: given both as input & asked as output (what?) - - **square**: intermediate data (neither given nor asked) - - **red frame**: delete-instruction (to free up memory) + - **square**: intermediate data, neither given nor asked. + - **red frame**: delete-instruction, to free up memory. + - **filled**: data node has a value in `solution`, shown in tooltip. + - **thick frame**: function/data node visited. ARROWS @@ -473,6 +478,7 @@ def plot_graph(graph, filename=None, show=False, steps=None, :param str filename: Write the output to a file. The extension must be one of: ``.png .dot .jpg .jpeg .pdf .svg`` + Prefer ``.pdf`` or ``.svg`` to see solution-values in tooltips. :param boolean show: If this is set to True, use matplotlib to show the graph diagram (Default: False) @@ -518,34 +524,39 @@ def get_node_name(a): # draw nodes for nx_node in graph.nodes: kw = {} - if isinstance(nx_node, DataPlaceholderNode): + if isinstance(nx_node, str): # Only DeleteInstructions data in steps. if nx_node in steps: - kw = {'color': 'red', 'style': 'bold'} - + kw = {'color': 'red', 'penwidth': 2} + # SHAPE change if in inputs/outputs. shape="rect" - if inputs and nx_node in inputs: - shape="invhouse" - if outputs and nx_node in outputs: + if inputs and outputs and nx_node in inputs and nx_node in outputs: + shape="hexagon" + else: if inputs and nx_node in inputs: - shape="polygon" - else: + shape="invhouse" + if outputs and nx_node in outputs: shape="house" # LABEL change from solution. - name = str(nx_node) if solution and nx_node in solution: - name = "%s: %s" % (nx_node, solution.get(nx_node)) - node = pydot.Node(name=nx_node, label=name, shape=shape, **kw) - else: + kw["style"] = "filled" + kw["fillcolor"] = "gray" + # kw["tooltip"] = nx_node, solution.get(nx_node) + node = pydot.Node(name=nx_node, shape=shape, + URL="fdgfdf", **kw) + else: # Operation + kw = {} + shape = "oval" if isinstance(nx_node, NetworkOperation) else "circle" if nx_node in steps: - kw = {'style': 'bold'} - node = pydot.Node(name=nx_node.name, shape="circle", **kw) + kw["style"] = "bold" + node = pydot.Node(name=nx_node.name, shape=shape, **kw) + g.add_node(node) # draw edges - for src, dst in graph.edges(): + for src, dst in graph.edges: src_name = get_node_name(src) dst_name = get_node_name(dst) kw = {} @@ -564,8 +575,9 @@ def get_node_name(a): src_name = get_node_name(src) dst_name = get_node_name(dst) edge = pydot.Edge( - src=src_name, dst=dst_name, label=str(i), style="dotted", - penwidth='2') + src=src_name, dst=dst_name, label=str(i), style='dotted', + color="green", fontcolor="green", fontname="bold", fontsize=18, + penwidth=3, arrowhead="vee") g.add_edge(edge) # save plot From ca5d243369bb8911a4fdd06c3eeb0b36d5045f57 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 5 Oct 2019 14:01:41 +0300 Subject: [PATCH 21/33] test(plot): enhance plot test to try all #13 features; + test all chained plot() methods from netop. --- test/test_graphkit.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/test/test_graphkit.py b/test/test_graphkit.py index 826719d6..927632e3 100644 --- a/test/test_graphkit.py +++ b/test/test_graphkit.py @@ -108,21 +108,24 @@ def test_network_deep_merge(): def test_plotting(): - sum_op1 = operation(name='sum_op1', needs=['a', 'b'], provides='sum1')(add) - sum_op2 = operation(name='sum_op2', needs=['a', 'b'], provides='sum2')(add) - sum_op3 = operation(name='sum_op3', needs=['sum1', 'c'], provides='sum3')(add) - net1 = compose(name='my network 1')(sum_op1, sum_op2, sum_op3) + pipeline = compose(name="netop")( + operation(name="add", needs=["a", "b1"], provides=["ab1"])(add), + operation(name="sub", needs=["a", modifiers.optional("b2")], provides=["ab2"])(lambda a, b=1: a-b), + operation(name="abb", needs=["ab1", "ab2"], provides=["asked"])(add), + ) + inputs = {'a': 1, 'b1': 2} + solution=pipeline(inputs) for ext in network.supported_plot_writers(): tdir = tempfile.mkdtemp(suffix=ext) png_file = osp.join(tdir, "workflow.png") - net1.net.plot(png_file) + pipeline.plot(png_file, inputs=inputs, solution=solution, outputs=['asked', 'b1']) try: assert osp.exists(png_file) finally: shutil.rmtree(tdir, ignore_errors=True) try: - net1.net.plot('bad.format') + pipeline.plot('bad.format') assert False, "Should had failed writting arbitrary file format!" except ValueError as ex: assert "Unknown file format" in str(ex) From a2de9efb2fbbb2fdb2a8e65c957db09297e0099f Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 5 Oct 2019 16:32:26 +0300 Subject: [PATCH 22/33] doc(plot); explain also params in user-facing API --- graphkit/base.py | 23 ++++++++++++++++++++--- graphkit/network.py | 43 ++++++++++++++++++++++++++++++++----------- 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/graphkit/base.py b/graphkit/base.py index 512a95bc..1298b566 100644 --- a/graphkit/base.py +++ b/graphkit/base.py @@ -174,9 +174,26 @@ def set_execution_method(self, method): def plot(self, filename=None, show=False, inputs=None, outputs=None, solution=None): """ - Plot a *Graphviz* graph and return it, if no other argument provided. - - See :func:`network.plot_graph()` for arguments, legend, and example code. + :param str filename: + Write diagram into a file. + The extension must be one of: ``.png .dot .jpg .jpeg .pdf .svg`` + Prefer ``.pdf`` or ``.svg`` to see solution-values in tooltips. + :param boolean show: + If it evaluates to true, opens the diagram in a matplotlib window. + :param inputs: + an optional name list, any nodes in there are plotted + as a "house" + :param outputs: + an optional name list, any nodes in there are plotted + as an "inverted-house" + :param solution: + an optional dict with values to annotate nodes + (currently content not shown, but node drawn as "filled") + + :return: + An instance of the :mod`pydot` graph + + See :func:`network.plot_graph()` for the plot legend and example code. """ return self.net.plot(filename, show, inputs, outputs, solution) diff --git a/graphkit/network.py b/graphkit/network.py index 1c246530..ba9d3a76 100644 --- a/graphkit/network.py +++ b/graphkit/network.py @@ -381,7 +381,26 @@ def plot(self, filename=None, show=False, """ Plot a *Graphviz* graph and return it, if no other argument provided. - See :func:`network.plot_graph()` for arguments, legend, and example code. + :param str filename: + Write diagram into a file. + The extension must be one of: ``.png .dot .jpg .jpeg .pdf .svg`` + Prefer ``.pdf`` or ``.svg`` to see solution-values in tooltips. + :param boolean show: + If it evaluates to true, opens the diagram in a matplotlib window. + :param inputs: + an optional name list, any nodes in there are plotted + as a "house" + :param outputs: + an optional name list, any nodes in there are plotted + as an "inverted-house" + :param solution: + an optional dict with values to annotate nodes + (currently content not shown, but node drawn as "filled") + + :return: + An instance of the :mod`pydot` graph + + See :func:`network.plot_graph` for the plot legend and example code. """ return plot_graph(self.graph, filename, show, self.steps, inputs, outputs, solution) @@ -476,24 +495,25 @@ def plot_graph(graph, filename=None, show=False, steps=None, :param graph: what to plot :param str filename: - Write the output to a file. + Write diagram into a file. The extension must be one of: ``.png .dot .jpg .jpeg .pdf .svg`` Prefer ``.pdf`` or ``.svg`` to see solution-values in tooltips. :param boolean show: - If this is set to True, use matplotlib to show the graph diagram - (Default: False) + If it evaluates to true, opens the diagram in a matplotlib window. :param steps: a list of nodes & instructions to overlay on the diagram :param inputs: - an optional list, any nodes in there are plotted as `"house" - `_ + an optional name list, any nodes in there are plotted + as a "house" :param outputs: - an optional list, any nodes in there are plotted as `"invhouse" - :param outputs: - an optional dict, any values in there are included in the node-name + an optional name list, any nodes in there are plotted + as an "inverted-house" + :param solution: + an optional dict with values to annotate nodes + (currently content not shown, but node drawn as "filled") - :returns: - An instance of the pydot graph + :return: + An instance of the :mod`pydot` graph **Example:** @@ -530,6 +550,7 @@ def get_node_name(a): kw = {'color': 'red', 'penwidth': 2} # SHAPE change if in inputs/outputs. + # tip: https://graphviz.gitlab.io/_pages/doc/info/shapes.html shape="rect" if inputs and outputs and nx_node in inputs and nx_node in outputs: shape="hexagon" From dc5a21a6128a9710ec855b9d08465452caed9d89 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 5 Oct 2019 18:29:36 +0300 Subject: [PATCH 23/33] FIX(PLOT.TC): TC was always testing PNG, ... + retorfitted to try all available formats. + list of forbidden formats based on my failres --- graphkit/base.py | 4 ++-- graphkit/network.py | 32 +++++++++++++------------------- test/test_graphkit.py | 34 ++++++++++++++++++++++++---------- 3 files changed, 39 insertions(+), 31 deletions(-) diff --git a/graphkit/base.py b/graphkit/base.py index 1298b566..5f425028 100644 --- a/graphkit/base.py +++ b/graphkit/base.py @@ -176,8 +176,8 @@ def plot(self, filename=None, show=False, """ :param str filename: Write diagram into a file. - The extension must be one of: ``.png .dot .jpg .jpeg .pdf .svg`` - Prefer ``.pdf`` or ``.svg`` to see solution-values in tooltips. + Common extensions are ``.png .dot .jpg .jpeg .pdf .svg`` + call :func:`network.supported_plot_formats()` for more. :param boolean show: If it evaluates to true, opens the diagram in a matplotlib window. :param inputs: diff --git a/graphkit/network.py b/graphkit/network.py index ba9d3a76..82b7d128 100644 --- a/graphkit/network.py +++ b/graphkit/network.py @@ -383,8 +383,8 @@ def plot(self, filename=None, show=False, :param str filename: Write diagram into a file. - The extension must be one of: ``.png .dot .jpg .jpeg .pdf .svg`` - Prefer ``.pdf`` or ``.svg`` to see solution-values in tooltips. + Common extensions are ``.png .dot .jpg .jpeg .pdf .svg`` + call :func:`network.supported_plot_formats()` for more. :param boolean show: If it evaluates to true, opens the diagram in a matplotlib window. :param inputs: @@ -454,15 +454,10 @@ def get_data_node(name, graph): return None -def supported_plot_writers(): - return { - ".png": lambda gplot: gplot.create_png(), - ".dot": lambda gplot: gplot.to_string(), - ".jpg": lambda gplot: gplot.create_jpeg(), - ".jpeg": lambda gplot: gplot.create_jpeg(), - ".pdf": lambda gplot: gplot.create_pdf(), - ".svg": lambda gplot: gplot.create_svg(), - } +def supported_plot_formats(): + import pydot + + return [".%s" % f for f in pydot.Dot().formats] def plot_graph(graph, filename=None, show=False, steps=None, @@ -496,8 +491,8 @@ def plot_graph(graph, filename=None, show=False, steps=None, what to plot :param str filename: Write diagram into a file. - The extension must be one of: ``.png .dot .jpg .jpeg .pdf .svg`` - Prefer ``.pdf`` or ``.svg`` to see solution-values in tooltips. + Common extensions are ``.png .dot .jpg .jpeg .pdf .svg`` + call :func:`network.supported_plot_formats()` for more. :param boolean show: If it evaluates to true, opens the diagram in a matplotlib window. :param steps: @@ -603,16 +598,15 @@ def get_node_name(a): # save plot if filename: + formats = supported_plot_formats() _basename, ext = os.path.splitext(filename) - writers = supported_plot_writers() - plot_writer = supported_plot_writers().get(ext.lower()) - if not plot_writer: + if not ext.lower() in formats: raise ValueError( "Unknown file format for saving graph: %s" " File extensions must be one of: %s" - % (ext, ' '.join(writers))) - with open(filename, "wb") as fh: - fh.write(plot_writer(g)) + % (ext, " ".join(formats))) + + g.write(filename, format=ext.lower()[1:]) # display graph via matplotlib if show: diff --git a/test/test_graphkit.py b/test/test_graphkit.py index 927632e3..fb7620b8 100644 --- a/test/test_graphkit.py +++ b/test/test_graphkit.py @@ -116,14 +116,27 @@ def test_plotting(): inputs = {'a': 1, 'b1': 2} solution=pipeline(inputs) - for ext in network.supported_plot_writers(): - tdir = tempfile.mkdtemp(suffix=ext) - png_file = osp.join(tdir, "workflow.png") - pipeline.plot(png_file, inputs=inputs, solution=solution, outputs=['asked', 'b1']) - try: - assert osp.exists(png_file) - finally: - shutil.rmtree(tdir, ignore_errors=True) + # ...not working on my PC ... + forbidden_formats = ".dia .hpgl .mif .pcl .pic .vtx .xlib".split() + tdir = tempfile.mkdtemp() + counter = 0 + try: + for ext in network.supported_plot_formats(): + if ext in forbidden_formats: + continue + + counter += 1 + fpath = osp.join(tdir, "workflow-%i%s" % (counter, ext)) + pipeline.plot(fpath, inputs=inputs, solution=solution, outputs=['asked', 'b1']) + assert osp.exists(fpath) + + counter += 1 + fpath = osp.join(tdir, "workflow-%i%s" % (counter, ext)) + pipeline.plot(fpath) + assert osp.exists(fpath) + finally: + shutil.rmtree(tdir, ignore_errors=True) + try: pipeline.plot('bad.format') assert False, "Should had failed writting arbitrary file format!" @@ -131,12 +144,13 @@ def test_plotting(): assert "Unknown file format" in str(ex) ## Check help msg lists all siupported formats - for ext in network.supported_plot_writers(): + for ext in network.supported_plot_formats(): assert ext in str(ex) def test_plotting_docstring(): - for ext in network.supported_plot_writers(): + common_formats = ".png .dot .jpg .jpeg .pdf .svg".split() + for ext in common_formats: assert ext in network.plot_graph.__doc__ From 782d9b9ee96bacd55732a657309358e7d2d2e8c5 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 5 Oct 2019 18:31:27 +0300 Subject: [PATCH 24/33] fix(plot): don't require Matplotlib if no Window asked --- graphkit/network.py | 6 ++++-- test/test_graphkit.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/graphkit/network.py b/graphkit/network.py index 82b7d128..3351ee94 100644 --- a/graphkit/network.py +++ b/graphkit/network.py @@ -455,6 +455,7 @@ def get_data_node(name, graph): def supported_plot_formats(): + """return automatically all `pydot` extensions withlike ``.png``""" import pydot return [".%s" % f for f in pydot.Dot().formats] @@ -524,8 +525,6 @@ def plot_graph(graph, filename=None, show=False, steps=None, """ import pydot - import matplotlib.pyplot as plt - import matplotlib.image as mpimg assert graph is not None @@ -610,6 +609,9 @@ def get_node_name(a): # display graph via matplotlib if show: + import matplotlib.pyplot as plt + import matplotlib.image as mpimg + png = g.create_png() sio = io.BytesIO(png) img = mpimg.imread(sio) diff --git a/test/test_graphkit.py b/test/test_graphkit.py index fb7620b8..d41572cc 100644 --- a/test/test_graphkit.py +++ b/test/test_graphkit.py @@ -117,7 +117,8 @@ def test_plotting(): solution=pipeline(inputs) # ...not working on my PC ... - forbidden_formats = ".dia .hpgl .mif .pcl .pic .vtx .xlib".split() + forbidden_formats = ".dia .hpgl .mif .mp .pcl .pic .vtx .xlib".split() + tdir = tempfile.mkdtemp() counter = 0 try: From 7d389c3f34b9c897aabd55d4dcdf7de8850fa349 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 5 Oct 2019 18:50:11 +0300 Subject: [PATCH 25/33] test(plot): check also matplotlib show=True --- graphkit/network.py | 4 +++- setup.py | 6 +++++- test/test_graphkit.py | 12 ++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/graphkit/network.py b/graphkit/network.py index 3351ee94..140d0b2e 100644 --- a/graphkit/network.py +++ b/graphkit/network.py @@ -496,6 +496,7 @@ def plot_graph(graph, filename=None, show=False, steps=None, call :func:`network.supported_plot_formats()` for more. :param boolean show: If it evaluates to true, opens the diagram in a matplotlib window. + If it equals ``-1``, it plots but does not open the Window. :param steps: a list of nodes & instructions to overlay on the diagram :param inputs: @@ -616,6 +617,7 @@ def get_node_name(a): sio = io.BytesIO(png) img = mpimg.imread(sio) plt.imshow(img, aspect="equal") - plt.show() + if show != -1: + plt.show() return g diff --git a/setup.py b/setup.py index d3dfec84..4ed30ff8 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,11 @@ extras_require={ 'plot': ['pydot', 'matplotlib'] }, - tests_require=['numpy'], + tests_require=[ + "numpy", + "pydot", # to test plot + "matplotlib" # to test plot + ], license='Apache-2.0', keywords=['graph', 'computation graph', 'DAG', 'directed acyclical graph'], classifiers=[ diff --git a/test/test_graphkit.py b/test/test_graphkit.py index d41572cc..c4d8a20f 100644 --- a/test/test_graphkit.py +++ b/test/test_graphkit.py @@ -5,6 +5,7 @@ import pickle import os.path as osp import shutil +import sys import tempfile @@ -138,6 +139,17 @@ def test_plotting(): finally: shutil.rmtree(tdir, ignore_errors=True) + ## Don't open matplotlib window. + # + if sys.version_info < (3, 5): + # On PY< 3.5 it fails with: + # nose.proxy.TclError: no display name and no $DISPLAY environment variable + # eg https://travis-ci.org/ankostis/graphkit/jobs/593957996 + import matplotlib + matplotlib.use("Agg") + # do not open window in headless travis + assert pipeline.plot(show=-1) + try: pipeline.plot('bad.format') assert False, "Should had failed writting arbitrary file format!" From 3fe0b404594d347b7f481197be89b17f1a5ae8c1 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 5 Oct 2019 19:19:05 +0300 Subject: [PATCH 26/33] ENH(plot): return SVG rendered in JUPYTER, ... + doc: rename in sample code: netop --> pipeline. + enh(build): add `ipython` in test dependencies. + include it in the plot TC. --- graphkit/base.py | 10 +++++++--- graphkit/network.py | 44 ++++++++++++++++++++++++++++++------------- setup.py | 1 + test/test_graphkit.py | 9 ++++++++- 4 files changed, 47 insertions(+), 17 deletions(-) diff --git a/graphkit/base.py b/graphkit/base.py index 5f425028..36ccca3c 100644 --- a/graphkit/base.py +++ b/graphkit/base.py @@ -171,15 +171,19 @@ def set_execution_method(self, method): assert method in options self._execution_method = method - def plot(self, filename=None, show=False, + def plot(self, filename=None, show=False, jupyter=None, inputs=None, outputs=None, solution=None): """ :param str filename: Write diagram into a file. Common extensions are ``.png .dot .jpg .jpeg .pdf .svg`` call :func:`network.supported_plot_formats()` for more. - :param boolean show: + :param show: If it evaluates to true, opens the diagram in a matplotlib window. + If it equals `-1`, it plots but does not open the Window. + :param jupyter: + If it evaluates to true, return an SVG suitable to render + in *jupyter notebook cells* (`ipython` must be installed). :param inputs: an optional name list, any nodes in there are plotted as a "house" @@ -195,7 +199,7 @@ def plot(self, filename=None, show=False, See :func:`network.plot_graph()` for the plot legend and example code. """ - return self.net.plot(filename, show, inputs, outputs, solution) + return self.net.plot(filename, show, jupyter, inputs, outputs, solution) def __getstate__(self): state = Operation.__getstate__(self) diff --git a/graphkit/network.py b/graphkit/network.py index 140d0b2e..6dc4d48a 100644 --- a/graphkit/network.py +++ b/graphkit/network.py @@ -376,7 +376,7 @@ def _compute_sequential_method(self, named_inputs, outputs): return {k: cache[k] for k in iter(cache) if k in outputs} - def plot(self, filename=None, show=False, + def plot(self, filename=None, show=False, jupyter=None, inputs=None, outputs=None, solution=None): """ Plot a *Graphviz* graph and return it, if no other argument provided. @@ -385,8 +385,12 @@ def plot(self, filename=None, show=False, Write diagram into a file. Common extensions are ``.png .dot .jpg .jpeg .pdf .svg`` call :func:`network.supported_plot_formats()` for more. - :param boolean show: + :param show: If it evaluates to true, opens the diagram in a matplotlib window. + If it equals `-1``, it plots but does not open the Window. + :param jupyter: + If it evaluates to true, return an SVG suitable to render + in *jupyter notebook cells* (`ipython` must be installed). :param inputs: an optional name list, any nodes in there are plotted as a "house" @@ -402,8 +406,8 @@ def plot(self, filename=None, show=False, See :func:`network.plot_graph` for the plot legend and example code. """ - return plot_graph(self.graph, filename, show, self.steps, - inputs, outputs, solution) + return plot_graph(self.graph, filename, show, jupyter, + self.steps, inputs, outputs, solution) def ready_to_schedule_operation(op, has_executed, graph): @@ -461,8 +465,8 @@ def supported_plot_formats(): return [".%s" % f for f in pydot.Dot().formats] -def plot_graph(graph, filename=None, show=False, steps=None, - inputs=None, outputs=None, solution=None): +def plot_graph(graph, filename=None, show=False, jupyter=False, + steps=None, inputs=None, outputs=None, solution=None): """ Plot a *Graphviz* graph/steps and return it, if no other argument provided. @@ -494,9 +498,12 @@ def plot_graph(graph, filename=None, show=False, steps=None, Write diagram into a file. Common extensions are ``.png .dot .jpg .jpeg .pdf .svg`` call :func:`network.supported_plot_formats()` for more. - :param boolean show: + :param show: If it evaluates to true, opens the diagram in a matplotlib window. - If it equals ``-1``, it plots but does not open the Window. + If it equals `-1``, it plots but does not open the Window. + :param jupyter: + If it evaluates to true, return an SVG suitable to render + in *jupyter notebook cells* (`ipython` must be installed). :param steps: a list of nodes & instructions to overlay on the diagram :param inputs: @@ -514,15 +521,18 @@ def plot_graph(graph, filename=None, show=False, steps=None, **Example:** - >>> netop = compose(name="netop")( + >>> from graphkit import compose, operation + >>> from graphkit.modifiers import optional + + >>> pipeline = compose(name="pipeline")( ... operation(name="add", needs=["a", "b1"], provides=["ab1"])(add), ... operation(name="sub", needs=["a", optional("b2")], provides=["ab2"])(lambda a, b=1: a-b), ... operation(name="abb", needs=["ab1", "ab2"], provides=["asked"])(add), ... ) >>> inputs = {'a': 1, 'b1': 2} - >>> solution=netop(inputs) - >>> netop.plot('plot.svg', inputs=inputs, solution=solution, outputs=['asked', 'b1']); + >>> solution=pipeline(inputs) + >>> pipeline.plot('plot.svg', inputs=inputs, solution=solution, outputs=['asked', 'b1']); """ import pydot @@ -596,7 +606,8 @@ def get_node_name(a): penwidth=3, arrowhead="vee") g.add_edge(edge) - # save plot + # Save plot + # if filename: formats = supported_plot_formats() _basename, ext = os.path.splitext(filename) @@ -608,7 +619,14 @@ def get_node_name(a): g.write(filename, format=ext.lower()[1:]) - # display graph via matplotlib + ## Return an SVG renderable in jupyter. + # + if jupyter: + from IPython.display import SVG + g = SVG(data=g.create_svg()) + + ## Display graph via matplotlib + # if show: import matplotlib.pyplot as plt import matplotlib.image as mpimg diff --git a/setup.py b/setup.py index 4ed30ff8..1448a241 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,7 @@ }, tests_require=[ "numpy", + "ipython; python_version >= '3.5'", # to test jupyter plot. "pydot", # to test plot "matplotlib" # to test plot ], diff --git a/test/test_graphkit.py b/test/test_graphkit.py index c4d8a20f..4cd283f2 100644 --- a/test/test_graphkit.py +++ b/test/test_graphkit.py @@ -139,7 +139,8 @@ def test_plotting(): finally: shutil.rmtree(tdir, ignore_errors=True) - ## Don't open matplotlib window. + ## Try matplotlib Window, but + # without opening a Window. # if sys.version_info < (3, 5): # On PY< 3.5 it fails with: @@ -150,6 +151,12 @@ def test_plotting(): # do not open window in headless travis assert pipeline.plot(show=-1) + ## Try Jupyter SVG. + # + # but latest ipython-7+ dropped < PY3.4 + if sys.version_info >= (3, 5): + assert "display.SVG" in str(type(pipeline.plot(jupyter=True))) + try: pipeline.plot('bad.format') assert False, "Should had failed writting arbitrary file format!" From 1471551b30ce22ab668b4d18bf8d20ceaea5061f Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 5 Oct 2019 19:40:47 +0300 Subject: [PATCH 27/33] refact(plot.TC): avoid writting multiple temp-files --- test/test_graphkit.py | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/test/test_graphkit.py b/test/test_graphkit.py index 4cd283f2..07c2654d 100644 --- a/test/test_graphkit.py +++ b/test/test_graphkit.py @@ -117,25 +117,35 @@ def test_plotting(): inputs = {'a': 1, 'b1': 2} solution=pipeline(inputs) - # ...not working on my PC ... - forbidden_formats = ".dia .hpgl .mif .mp .pcl .pic .vtx .xlib".split() + ## Generate all formats + # (not needing to save files) + # + # ...these are not working on my PC, or travis. + forbidden_formats = ".dia .hpgl .mif .mp .pcl .pic .vtx .xlib".split() + prev_dot = None + for ext in network.supported_plot_formats(): + if ext in forbidden_formats: + continue + + dot = pipeline.plot(inputs=inputs, solution=solution, outputs=['asked', 'b1']) + assert dot + assert dot != prev_dot + prev_dot = dot + + dot = pipeline.plot() + assert dot + assert dot != prev_dot + prev_dot = dot + + ## Try saving one file. + # tdir = tempfile.mkdtemp() - counter = 0 + fpath = osp.join(tdir, "workflow.png") try: - for ext in network.supported_plot_formats(): - if ext in forbidden_formats: - continue - - counter += 1 - fpath = osp.join(tdir, "workflow-%i%s" % (counter, ext)) - pipeline.plot(fpath, inputs=inputs, solution=solution, outputs=['asked', 'b1']) - assert osp.exists(fpath) - - counter += 1 - fpath = osp.join(tdir, "workflow-%i%s" % (counter, ext)) - pipeline.plot(fpath) - assert osp.exists(fpath) + dot = pipeline.plot(fpath, inputs=inputs, solution=solution, outputs=['asked', 'b1']) + assert osp.exists(fpath) + assert dot finally: shutil.rmtree(tdir, ignore_errors=True) From b4401963eb103cd7b6fa225fb0dad0153efa89a2 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 5 Oct 2019 20:06:48 +0300 Subject: [PATCH 28/33] fix(build): reuse dependencies definitions --- setup.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 6ce52009..654f4e13 100644 --- a/setup.py +++ b/setup.py @@ -19,6 +19,16 @@ with io.open('graphkit/__init__.py', 'rt', encoding='utf8') as f: version = re.search(r'__version__ = \'(.*?)\'', f.read()).group(1) +plot_reqs = [ + "ipython; python_version >= '3.5'", # to test jupyter plot. + "matplotlib", # to test plot + "pydot", # to test plot +] +test_reqs = plot_reqs + [ + "pytest", + "pytest-cov", +] + setup( name='graphkit', version=version, @@ -33,16 +43,10 @@ "networkx == 2.2; python_version < '3.5'", ], extras_require={ - 'plot': ['pydot', 'matplotlib'], - 'test': ['pydot', 'matplotlib', 'pytest', "pytest-cov"], + 'plot': plot_reqs, + 'test': test_reqs, }, - tests_require=[ - "pytest", - "pytest-cov", - "ipython; python_version >= '3.5'", # to test jupyter plot. - "pydot", # to test plot - "matplotlib" # to test plot - ], + tests_require=test_reqs, license='Apache-2.0', keywords=['graph', 'computation graph', 'DAG', 'directed acyclical graph'], classifiers=[ From bde9b64a0000d5b0ca798d39d34c9043d2bc80fd Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 5 Oct 2019 21:17:10 +0300 Subject: [PATCH 29/33] REFACT(plot.TC): PYTESTize and parametrize --- test/test_graphkit.py | 81 -------------------------------- test/test_plot.py | 106 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 81 deletions(-) create mode 100644 test/test_plot.py diff --git a/test/test_graphkit.py b/test/test_graphkit.py index 1afdb740..7db2e973 100644 --- a/test/test_graphkit.py +++ b/test/test_graphkit.py @@ -3,11 +3,6 @@ import math import pickle -import os.path as osp -import shutil -import sys -import tempfile - from pprint import pprint from operator import add @@ -109,82 +104,6 @@ def test_network_deep_merge(): pprint(net3({'a': 1, 'b': 2, 'c': 4})) -def test_plotting(): - pipeline = compose(name="netop")( - operation(name="add", needs=["a", "b1"], provides=["ab1"])(add), - operation(name="sub", needs=["a", modifiers.optional("b2")], provides=["ab2"])(lambda a, b=1: a-b), - operation(name="abb", needs=["ab1", "ab2"], provides=["asked"])(add), - ) - inputs = {'a': 1, 'b1': 2} - solution=pipeline(inputs) - - - ## Generate all formats - # (not needing to save files) - # - # ...these are not working on my PC, or travis. - forbidden_formats = ".dia .hpgl .mif .mp .pcl .pic .vtx .xlib".split() - prev_dot = None - for ext in network.supported_plot_formats(): - if ext in forbidden_formats: - continue - - dot = pipeline.plot(inputs=inputs, solution=solution, outputs=['asked', 'b1']) - assert dot - assert dot != prev_dot - prev_dot = dot - - dot = pipeline.plot() - assert dot - assert dot != prev_dot - prev_dot = dot - - ## Try saving one file. - # - tdir = tempfile.mkdtemp() - fpath = osp.join(tdir, "workflow.png") - try: - dot = pipeline.plot(fpath, inputs=inputs, solution=solution, outputs=['asked', 'b1']) - assert osp.exists(fpath) - assert dot - finally: - shutil.rmtree(tdir, ignore_errors=True) - - ## Try matplotlib Window, but - # without opening a Window. - # - if sys.version_info < (3, 5): - # On PY< 3.5 it fails with: - # nose.proxy.TclError: no display name and no $DISPLAY environment variable - # eg https://travis-ci.org/ankostis/graphkit/jobs/593957996 - import matplotlib - matplotlib.use("Agg") - # do not open window in headless travis - assert pipeline.plot(show=-1) - - ## Try Jupyter SVG. - # - # but latest ipython-7+ dropped < PY3.4 - if sys.version_info >= (3, 5): - assert "display.SVG" in str(type(pipeline.plot(jupyter=True))) - - try: - pipeline.plot('bad.format') - assert False, "Should had failed writting arbitrary file format!" - except ValueError as ex: - assert "Unknown file format" in str(ex) - - ## Check help msg lists all siupported formats - for ext in network.supported_plot_formats(): - assert ext in str(ex) - - -def test_plotting_docstring(): - common_formats = ".png .dot .jpg .jpeg .pdf .svg".split() - for ext in common_formats: - assert ext in network.plot_graph.__doc__ - - def test_input_based_pruning(): # Tests to make sure we don't need to pass graph inputs if we're provided # with data further downstream in the graph as an input. diff --git a/test/test_plot.py b/test/test_plot.py new file mode 100644 index 00000000..90ba87f9 --- /dev/null +++ b/test/test_plot.py @@ -0,0 +1,106 @@ +# Copyright 2016, Yahoo Inc. +# Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms. + +from operator import add + +import pytest +import sys + +from graphkit import compose, network, operation +from graphkit.modifiers import optional + + +@pytest.fixture +def pipeline(): + return compose(name="netop")( + operation(name="add", needs=["a", "b1"], provides=["ab1"])(add), + operation(name="sub", needs=["a", optional("b2")], provides=["ab2"])( + lambda a, b=1: a - b + ), + operation(name="abb", needs=["ab1", "ab2"], provides=["asked"])(add), + ) + + +@pytest.fixture(params=[{"a": 1}, {"a": 1, "b1": 2}]) +def inputs(request): + return {"a": 1, "b1": 2} + + +@pytest.fixture(params=[None, ("a", "b1")]) +def input_names(request): + return request.param + + +@pytest.fixture(params=[None, ["asked", "b1"]]) +def outputs(request): + return request.param + + +@pytest.fixture(params=[None, 1]) +def solution(pipeline, inputs, outputs, request): + return request.param and pipeline(inputs, outputs) + + +###### TEST CASES ####### +## + + +def test_plotting_docstring(): + common_formats = ".png .dot .jpg .jpeg .pdf .svg".split() + for ext in common_formats: + assert ext in network.plot_graph.__doc__ + + +def test_plot_formats(pipeline, input_names, outputs, solution, tmp_path): + ## Generate all formats (not needing to save files) + + # ...these are not working on my PC, or travis. + forbidden_formats = ".dia .hpgl .mif .mp .pcl .pic .vtx .xlib".split() + prev_dot = None + for ext in network.supported_plot_formats(): + if ext not in forbidden_formats: + dot = pipeline.plot(inputs=input_names, outputs=outputs, solution=solution) + assert dot + assert ext == ".jpg" or dot != prev_dot + prev_dot = dot + + +def test_plot_bad_format(pipeline, tmp_path): + with pytest.raises(ValueError, match="Unknown file format") as exinfo: + pipeline.plot(filename="bad.format") + + ## Check help msg lists all siupported formats + for ext in network.supported_plot_formats(): + assert exinfo.match(ext) + + +def test_plot_write_file(pipeline, tmp_path): + # Try saving a file from one format. + + fpath = tmp_path / "workflow.png" + + dot = pipeline.plot(str(fpath)) + assert fpath.exists() + assert dot + + +def test_plot_matplib(pipeline, tmp_path): + ## Try matplotlib Window, but # without opening a Window. + + if sys.version_info < (3, 5): + # On PY< 3.5 it fails with: + # nose.proxy.TclError: no display name and no $DISPLAY environment variable + # eg https://travis-ci.org/ankostis/graphkit/jobs/593957996 + import matplotlib + + matplotlib.use("Agg") + # do not open window in headless travis + assert pipeline.plot(show=-1) + + +@pytest.mark.skipif(sys.version_info < (3, 5), reason="ipython-7+ dropped PY3.4-") +def test_plot_jupyter(pipeline, tmp_path): + ## Try returned Jupyter SVG. + + dot = pipeline.plot(jupyter=True) + assert "display.SVG" in str(type(dot)) From 8e361e6b3e463afe48eb94ce0feac59a37421c5e Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 5 Oct 2019 21:34:00 +0300 Subject: [PATCH 30/33] REFACT(PLOT): MOVE PLOT in own module --- graphkit/base.py | 2 +- graphkit/network.py | 186 ++------------------------------------- graphkit/plot.py | 208 ++++++++++++++++++++++++++++++++++++++++++++ test/test_plot.py | 8 +- 4 files changed, 219 insertions(+), 185 deletions(-) create mode 100644 graphkit/plot.py diff --git a/graphkit/base.py b/graphkit/base.py index 36ccca3c..140b7a97 100644 --- a/graphkit/base.py +++ b/graphkit/base.py @@ -197,7 +197,7 @@ def plot(self, filename=None, show=False, jupyter=None, :return: An instance of the :mod`pydot` graph - See :func:`network.plot_graph()` for the plot legend and example code. + See :func:`graphkit.plot.plot_graph()` for the plot legend and example code. """ return self.net.plot(filename, show, jupyter, inputs, outputs, solution) diff --git a/graphkit/network.py b/graphkit/network.py index 6dc4d48a..fa94f822 100644 --- a/graphkit/network.py +++ b/graphkit/network.py @@ -1,13 +1,11 @@ # Copyright 2016, Yahoo Inc. # Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms. -import io -import os import time import networkx as nx -from .base import Operation, NetworkOperation +from .base import Operation from .modifiers import optional @@ -404,9 +402,11 @@ def plot(self, filename=None, show=False, jupyter=None, :return: An instance of the :mod`pydot` graph - See :func:`network.plot_graph` for the plot legend and example code. + See :func:`graphkit.plot.plot_graph()` for the plot legend and example code. """ - return plot_graph(self.graph, filename, show, jupyter, + from . import plot + + return plot.plot_graph(self.graph, filename, show, jupyter, self.steps, inputs, outputs, solution) @@ -463,179 +463,3 @@ def supported_plot_formats(): import pydot return [".%s" % f for f in pydot.Dot().formats] - - -def plot_graph(graph, filename=None, show=False, jupyter=False, - steps=None, inputs=None, outputs=None, solution=None): - """ - Plot a *Graphviz* graph/steps and return it, if no other argument provided. - - Legend: - - - NODES: - - - **circle**: function - - **oval**: subgraph function - - **house**: given input - - **inversed-house**: asked output - - **polygon**: given both as input & asked as output (what?) - - **square**: intermediate data, neither given nor asked. - - **red frame**: delete-instruction, to free up memory. - - **filled**: data node has a value in `solution`, shown in tooltip. - - **thick frame**: function/data node visited. - - ARROWS - - - **solid black arrows**: dependencies (source-data are``need``\ed - by target-operations, sources-operations ``provide`` target-data) - - **dashed black arrows**: optional needs - - **green-dotted arrows**: execution steps labeled in succession - - :param graph: - what to plot - :param str filename: - Write diagram into a file. - Common extensions are ``.png .dot .jpg .jpeg .pdf .svg`` - call :func:`network.supported_plot_formats()` for more. - :param show: - If it evaluates to true, opens the diagram in a matplotlib window. - If it equals `-1``, it plots but does not open the Window. - :param jupyter: - If it evaluates to true, return an SVG suitable to render - in *jupyter notebook cells* (`ipython` must be installed). - :param steps: - a list of nodes & instructions to overlay on the diagram - :param inputs: - an optional name list, any nodes in there are plotted - as a "house" - :param outputs: - an optional name list, any nodes in there are plotted - as an "inverted-house" - :param solution: - an optional dict with values to annotate nodes - (currently content not shown, but node drawn as "filled") - - :return: - An instance of the :mod`pydot` graph - - **Example:** - - >>> from graphkit import compose, operation - >>> from graphkit.modifiers import optional - - >>> pipeline = compose(name="pipeline")( - ... operation(name="add", needs=["a", "b1"], provides=["ab1"])(add), - ... operation(name="sub", needs=["a", optional("b2")], provides=["ab2"])(lambda a, b=1: a-b), - ... operation(name="abb", needs=["ab1", "ab2"], provides=["asked"])(add), - ... ) - - >>> inputs = {'a': 1, 'b1': 2} - >>> solution=pipeline(inputs) - >>> pipeline.plot('plot.svg', inputs=inputs, solution=solution, outputs=['asked', 'b1']); - - """ - import pydot - - assert graph is not None - - def get_node_name(a): - if isinstance(a, Operation): - return a.name - return a - - g = pydot.Dot(graph_type="digraph") - - # draw nodes - for nx_node in graph.nodes: - kw = {} - if isinstance(nx_node, str): - # Only DeleteInstructions data in steps. - if nx_node in steps: - kw = {'color': 'red', 'penwidth': 2} - - # SHAPE change if in inputs/outputs. - # tip: https://graphviz.gitlab.io/_pages/doc/info/shapes.html - shape="rect" - if inputs and outputs and nx_node in inputs and nx_node in outputs: - shape="hexagon" - else: - if inputs and nx_node in inputs: - shape="invhouse" - if outputs and nx_node in outputs: - shape="house" - - # LABEL change from solution. - if solution and nx_node in solution: - kw["style"] = "filled" - kw["fillcolor"] = "gray" - # kw["tooltip"] = nx_node, solution.get(nx_node) - node = pydot.Node(name=nx_node, shape=shape, - URL="fdgfdf", **kw) - else: # Operation - kw = {} - shape = "oval" if isinstance(nx_node, NetworkOperation) else "circle" - if nx_node in steps: - kw["style"] = "bold" - node = pydot.Node(name=nx_node.name, shape=shape, **kw) - - g.add_node(node) - - # draw edges - for src, dst in graph.edges: - src_name = get_node_name(src) - dst_name = get_node_name(dst) - kw = {} - if isinstance(dst, Operation) and any(n == src - and isinstance(n, optional) - for n in dst.needs): - kw["style"] = "dashed" - edge = pydot.Edge(src=src_name, dst=dst_name, **kw) - g.add_edge(edge) - - # draw steps sequence - if steps and len(steps) > 1: - it1 = iter(steps) - it2 = iter(steps); next(it2) - for i, (src, dst) in enumerate(zip(it1, it2), 1): - src_name = get_node_name(src) - dst_name = get_node_name(dst) - edge = pydot.Edge( - src=src_name, dst=dst_name, label=str(i), style='dotted', - color="green", fontcolor="green", fontname="bold", fontsize=18, - penwidth=3, arrowhead="vee") - g.add_edge(edge) - - # Save plot - # - if filename: - formats = supported_plot_formats() - _basename, ext = os.path.splitext(filename) - if not ext.lower() in formats: - raise ValueError( - "Unknown file format for saving graph: %s" - " File extensions must be one of: %s" - % (ext, " ".join(formats))) - - g.write(filename, format=ext.lower()[1:]) - - ## Return an SVG renderable in jupyter. - # - if jupyter: - from IPython.display import SVG - g = SVG(data=g.create_svg()) - - ## Display graph via matplotlib - # - if show: - import matplotlib.pyplot as plt - import matplotlib.image as mpimg - - png = g.create_png() - sio = io.BytesIO(png) - img = mpimg.imread(sio) - plt.imshow(img, aspect="equal") - if show != -1: - plt.show() - - return g diff --git a/graphkit/plot.py b/graphkit/plot.py new file mode 100644 index 00000000..65d7d4a2 --- /dev/null +++ b/graphkit/plot.py @@ -0,0 +1,208 @@ +# Copyright 2016, Yahoo Inc. +# Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms. + +import io +import os + +from .base import NetworkOperation, Operation +from .modifiers import optional + + +def supported_plot_formats(): + """return automatically all `pydot` extensions withlike ``.png``""" + import pydot + + return [".%s" % f for f in pydot.Dot().formats] + + +def plot_graph( + graph, + filename=None, + show=False, + jupyter=False, + steps=None, + inputs=None, + outputs=None, + solution=None, +): + """ + Plot a *Graphviz* graph/steps and return it, if no other argument provided. + + Legend: + + + NODES: + + - **circle**: function + - **oval**: subgraph function + - **house**: given input + - **inversed-house**: asked output + - **polygon**: given both as input & asked as output (what?) + - **square**: intermediate data, neither given nor asked. + - **red frame**: delete-instruction, to free up memory. + - **filled**: data node has a value in `solution`, shown in tooltip. + - **thick frame**: function/data node visited. + + ARROWS + + - **solid black arrows**: dependencies (source-data are``need``\ed + by target-operations, sources-operations ``provide`` target-data) + - **dashed black arrows**: optional needs + - **green-dotted arrows**: execution steps labeled in succession + + :param graph: + what to plot + :param str filename: + Write diagram into a file. + Common extensions are ``.png .dot .jpg .jpeg .pdf .svg`` + call :func:`network.supported_plot_formats()` for more. + :param show: + If it evaluates to true, opens the diagram in a matplotlib window. + If it equals `-1``, it plots but does not open the Window. + :param jupyter: + If it evaluates to true, return an SVG suitable to render + in *jupyter notebook cells* (`ipython` must be installed). + :param steps: + a list of nodes & instructions to overlay on the diagram + :param inputs: + an optional name list, any nodes in there are plotted + as a "house" + :param outputs: + an optional name list, any nodes in there are plotted + as an "inverted-house" + :param solution: + an optional dict with values to annotate nodes + (currently content not shown, but node drawn as "filled") + + :return: + An instance of the :mod`pydot` graph + + **Example:** + + >>> from graphkit import compose, operation + >>> from graphkit.modifiers import optional + + >>> pipeline = compose(name="pipeline")( + ... operation(name="add", needs=["a", "b1"], provides=["ab1"])(add), + ... operation(name="sub", needs=["a", optional("b2")], provides=["ab2"])(lambda a, b=1: a-b), + ... operation(name="abb", needs=["ab1", "ab2"], provides=["asked"])(add), + ... ) + + >>> inputs = {'a': 1, 'b1': 2} + >>> solution=pipeline(inputs) + >>> pipeline.plot('plot.svg', inputs=inputs, solution=solution, outputs=['asked', 'b1']); + + """ + import pydot + + assert graph is not None + + def get_node_name(a): + if isinstance(a, Operation): + return a.name + return a + + g = pydot.Dot(graph_type="digraph") + + # draw nodes + for nx_node in graph.nodes: + kw = {} + if isinstance(nx_node, str): + # Only DeleteInstructions data in steps. + if nx_node in steps: + kw = {"color": "red", "penwidth": 2} + + # SHAPE change if in inputs/outputs. + # tip: https://graphviz.gitlab.io/_pages/doc/info/shapes.html + shape = "rect" + if inputs and outputs and nx_node in inputs and nx_node in outputs: + shape = "hexagon" + else: + if inputs and nx_node in inputs: + shape = "invhouse" + if outputs and nx_node in outputs: + shape = "house" + + # LABEL change from solution. + if solution and nx_node in solution: + kw["style"] = "filled" + kw["fillcolor"] = "gray" + # kw["tooltip"] = nx_node, solution.get(nx_node) + node = pydot.Node(name=nx_node, shape=shape, URL="fdgfdf", **kw) + else: # Operation + kw = {} + shape = "oval" if isinstance(nx_node, NetworkOperation) else "circle" + if nx_node in steps: + kw["style"] = "bold" + node = pydot.Node(name=nx_node.name, shape=shape, **kw) + + g.add_node(node) + + # draw edges + for src, dst in graph.edges: + src_name = get_node_name(src) + dst_name = get_node_name(dst) + kw = {} + if isinstance(dst, Operation) and any( + n == src and isinstance(n, optional) for n in dst.needs + ): + kw["style"] = "dashed" + edge = pydot.Edge(src=src_name, dst=dst_name, **kw) + g.add_edge(edge) + + # draw steps sequence + if steps and len(steps) > 1: + it1 = iter(steps) + it2 = iter(steps) + next(it2) + for i, (src, dst) in enumerate(zip(it1, it2), 1): + src_name = get_node_name(src) + dst_name = get_node_name(dst) + edge = pydot.Edge( + src=src_name, + dst=dst_name, + label=str(i), + style="dotted", + color="green", + fontcolor="green", + fontname="bold", + fontsize=18, + penwidth=3, + arrowhead="vee", + ) + g.add_edge(edge) + + # Save plot + # + if filename: + formats = supported_plot_formats() + _basename, ext = os.path.splitext(filename) + if not ext.lower() in formats: + raise ValueError( + "Unknown file format for saving graph: %s" + " File extensions must be one of: %s" % (ext, " ".join(formats)) + ) + + g.write(filename, format=ext.lower()[1:]) + + ## Return an SVG renderable in jupyter. + # + if jupyter: + from IPython.display import SVG + + g = SVG(data=g.create_svg()) + + ## Display graph via matplotlib + # + if show: + import matplotlib.pyplot as plt + import matplotlib.image as mpimg + + png = g.create_png() + sio = io.BytesIO(png) + img = mpimg.imread(sio) + plt.imshow(img, aspect="equal") + if show != -1: + plt.show() + + return g diff --git a/test/test_plot.py b/test/test_plot.py index 90ba87f9..1ba2cc0b 100644 --- a/test/test_plot.py +++ b/test/test_plot.py @@ -1,12 +1,12 @@ # Copyright 2016, Yahoo Inc. # Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms. +import sys from operator import add import pytest -import sys -from graphkit import compose, network, operation +from graphkit import base, compose, network, operation, plot from graphkit.modifiers import optional @@ -48,7 +48,9 @@ def solution(pipeline, inputs, outputs, request): def test_plotting_docstring(): common_formats = ".png .dot .jpg .jpeg .pdf .svg".split() for ext in common_formats: - assert ext in network.plot_graph.__doc__ + assert ext in plot.plot_graph.__doc__ + assert ext in base.NetworkOperation.plot.__doc__ + assert ext in network.Network.plot.__doc__ def test_plot_formats(pipeline, input_names, outputs, solution, tmp_path): From b08a3631cfd600e6b9efa0df389cacba1f72c2c7 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 5 Oct 2019 21:38:04 +0300 Subject: [PATCH 31/33] DROP PY3.4 - add PY3.6, PY3.7... ...pytest has problems with 3.4. --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4af0a1c1..cbd8cf82 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,9 @@ language: python python: - "2.7" - - "3.4" - "3.5" + - "3.6" + - "3.7" addons: apt: From 3a879592e9d2650db4fffc198222f42f0b3da258 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 5 Oct 2019 21:46:16 +0300 Subject: [PATCH 32/33] refact(plot): separate graphviz building from IO --- graphkit/base.py | 2 +- graphkit/network.py | 2 +- graphkit/plot.py | 173 +++++++++++++++++++++++--------------------- test/test_plot.py | 4 +- 4 files changed, 94 insertions(+), 87 deletions(-) diff --git a/graphkit/base.py b/graphkit/base.py index 140b7a97..212de939 100644 --- a/graphkit/base.py +++ b/graphkit/base.py @@ -177,7 +177,7 @@ def plot(self, filename=None, show=False, jupyter=None, :param str filename: Write diagram into a file. Common extensions are ``.png .dot .jpg .jpeg .pdf .svg`` - call :func:`network.supported_plot_formats()` for more. + call :func:`plot.supported_plot_formats()` for more. :param show: If it evaluates to true, opens the diagram in a matplotlib window. If it equals `-1`, it plots but does not open the Window. diff --git a/graphkit/network.py b/graphkit/network.py index fa94f822..582d0c0a 100644 --- a/graphkit/network.py +++ b/graphkit/network.py @@ -382,7 +382,7 @@ def plot(self, filename=None, show=False, jupyter=None, :param str filename: Write diagram into a file. Common extensions are ``.png .dot .jpg .jpeg .pdf .svg`` - call :func:`network.supported_plot_formats()` for more. + call :func:`plot.supported_plot_formats()` for more. :param show: If it evaluates to true, opens the diagram in a matplotlib window. If it equals `-1``, it plots but does not open the Window. diff --git a/graphkit/plot.py b/graphkit/plot.py index 65d7d4a2..bed110d8 100644 --- a/graphkit/plot.py +++ b/graphkit/plot.py @@ -15,6 +15,90 @@ def supported_plot_formats(): return [".%s" % f for f in pydot.Dot().formats] +def build_pydot(graph, steps=None, inputs=None, outputs=None, solution=None): + """ Build a Graphviz graph """ + import pydot + + assert graph is not None + + def get_node_name(a): + if isinstance(a, Operation): + return a.name + return a + + dot = pydot.Dot(graph_type="digraph") + + # draw nodes + for nx_node in graph.nodes: + kw = {} + if isinstance(nx_node, str): + # Only DeleteInstructions data in steps. + if nx_node in steps: + kw = {"color": "red", "penwidth": 2} + + # SHAPE change if in inputs/outputs. + # tip: https://graphviz.gitlab.io/_pages/doc/info/shapes.html + shape = "rect" + if inputs and outputs and nx_node in inputs and nx_node in outputs: + shape = "hexagon" + else: + if inputs and nx_node in inputs: + shape = "invhouse" + if outputs and nx_node in outputs: + shape = "house" + + # LABEL change from solution. + if solution and nx_node in solution: + kw["style"] = "filled" + kw["fillcolor"] = "gray" + # kw["tooltip"] = nx_node, solution.get(nx_node) + node = pydot.Node(name=nx_node, shape=shape, URL="fdgfdf", **kw) + else: # Operation + kw = {} + shape = "oval" if isinstance(nx_node, NetworkOperation) else "circle" + if nx_node in steps: + kw["style"] = "bold" + node = pydot.Node(name=nx_node.name, shape=shape, **kw) + + dot.add_node(node) + + # draw edges + for src, dst in graph.edges: + src_name = get_node_name(src) + dst_name = get_node_name(dst) + kw = {} + if isinstance(dst, Operation) and any( + n == src and isinstance(n, optional) for n in dst.needs + ): + kw["style"] = "dashed" + edge = pydot.Edge(src=src_name, dst=dst_name, **kw) + dot.add_edge(edge) + + # draw steps sequence + if steps and len(steps) > 1: + it1 = iter(steps) + it2 = iter(steps) + next(it2) + for i, (src, dst) in enumerate(zip(it1, it2), 1): + src_name = get_node_name(src) + dst_name = get_node_name(dst) + edge = pydot.Edge( + src=src_name, + dst=dst_name, + label=str(i), + style="dotted", + color="green", + fontcolor="green", + fontname="bold", + fontsize=18, + penwidth=3, + arrowhead="vee", + ) + dot.add_edge(edge) + + return dot + + def plot_graph( graph, filename=None, @@ -55,7 +139,7 @@ def plot_graph( :param str filename: Write diagram into a file. Common extensions are ``.png .dot .jpg .jpeg .pdf .svg`` - call :func:`network.supported_plot_formats()` for more. + call :func:`plot.supported_plot_formats()` for more. :param show: If it evaluates to true, opens the diagram in a matplotlib window. If it equals `-1``, it plots but does not open the Window. @@ -93,84 +177,7 @@ def plot_graph( >>> pipeline.plot('plot.svg', inputs=inputs, solution=solution, outputs=['asked', 'b1']); """ - import pydot - - assert graph is not None - - def get_node_name(a): - if isinstance(a, Operation): - return a.name - return a - - g = pydot.Dot(graph_type="digraph") - - # draw nodes - for nx_node in graph.nodes: - kw = {} - if isinstance(nx_node, str): - # Only DeleteInstructions data in steps. - if nx_node in steps: - kw = {"color": "red", "penwidth": 2} - - # SHAPE change if in inputs/outputs. - # tip: https://graphviz.gitlab.io/_pages/doc/info/shapes.html - shape = "rect" - if inputs and outputs and nx_node in inputs and nx_node in outputs: - shape = "hexagon" - else: - if inputs and nx_node in inputs: - shape = "invhouse" - if outputs and nx_node in outputs: - shape = "house" - - # LABEL change from solution. - if solution and nx_node in solution: - kw["style"] = "filled" - kw["fillcolor"] = "gray" - # kw["tooltip"] = nx_node, solution.get(nx_node) - node = pydot.Node(name=nx_node, shape=shape, URL="fdgfdf", **kw) - else: # Operation - kw = {} - shape = "oval" if isinstance(nx_node, NetworkOperation) else "circle" - if nx_node in steps: - kw["style"] = "bold" - node = pydot.Node(name=nx_node.name, shape=shape, **kw) - - g.add_node(node) - - # draw edges - for src, dst in graph.edges: - src_name = get_node_name(src) - dst_name = get_node_name(dst) - kw = {} - if isinstance(dst, Operation) and any( - n == src and isinstance(n, optional) for n in dst.needs - ): - kw["style"] = "dashed" - edge = pydot.Edge(src=src_name, dst=dst_name, **kw) - g.add_edge(edge) - - # draw steps sequence - if steps and len(steps) > 1: - it1 = iter(steps) - it2 = iter(steps) - next(it2) - for i, (src, dst) in enumerate(zip(it1, it2), 1): - src_name = get_node_name(src) - dst_name = get_node_name(dst) - edge = pydot.Edge( - src=src_name, - dst=dst_name, - label=str(i), - style="dotted", - color="green", - fontcolor="green", - fontname="bold", - fontsize=18, - penwidth=3, - arrowhead="vee", - ) - g.add_edge(edge) + dot = build_pydot(graph, steps, inputs, outputs, solution) # Save plot # @@ -183,14 +190,14 @@ def get_node_name(a): " File extensions must be one of: %s" % (ext, " ".join(formats)) ) - g.write(filename, format=ext.lower()[1:]) + dot.write(filename, format=ext.lower()[1:]) ## Return an SVG renderable in jupyter. # if jupyter: from IPython.display import SVG - g = SVG(data=g.create_svg()) + dot = SVG(data=dot.create_svg()) ## Display graph via matplotlib # @@ -198,11 +205,11 @@ def get_node_name(a): import matplotlib.pyplot as plt import matplotlib.image as mpimg - png = g.create_png() + png = dot.create_png() sio = io.BytesIO(png) img = mpimg.imread(sio) plt.imshow(img, aspect="equal") if show != -1: plt.show() - return g + return dot diff --git a/test/test_plot.py b/test/test_plot.py index 1ba2cc0b..39ad039f 100644 --- a/test/test_plot.py +++ b/test/test_plot.py @@ -59,7 +59,7 @@ def test_plot_formats(pipeline, input_names, outputs, solution, tmp_path): # ...these are not working on my PC, or travis. forbidden_formats = ".dia .hpgl .mif .mp .pcl .pic .vtx .xlib".split() prev_dot = None - for ext in network.supported_plot_formats(): + for ext in plot.supported_plot_formats(): if ext not in forbidden_formats: dot = pipeline.plot(inputs=input_names, outputs=outputs, solution=solution) assert dot @@ -72,7 +72,7 @@ def test_plot_bad_format(pipeline, tmp_path): pipeline.plot(filename="bad.format") ## Check help msg lists all siupported formats - for ext in network.supported_plot_formats(): + for ext in plot.supported_plot_formats(): assert exinfo.match(ext) From 4aca0138c4a6607df785efc21a8173386838c17d Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sun, 6 Oct 2019 10:56:45 +0300 Subject: [PATCH 33/33] FIX(plot): failing if steps not a list/ is none... backported from #29 --- graphkit/plot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/graphkit/plot.py b/graphkit/plot.py index bed110d8..c9db1fe4 100644 --- a/graphkit/plot.py +++ b/graphkit/plot.py @@ -33,7 +33,7 @@ def get_node_name(a): kw = {} if isinstance(nx_node, str): # Only DeleteInstructions data in steps. - if nx_node in steps: + if steps and nx_node in steps: kw = {"color": "red", "penwidth": 2} # SHAPE change if in inputs/outputs. @@ -129,7 +129,7 @@ def plot_graph( ARROWS - - **solid black arrows**: dependencies (source-data are``need``\ed + - **solid black arrows**: dependencies (source-data are``need``-ed by target-operations, sources-operations ``provide`` target-data) - **dashed black arrows**: optional needs - **green-dotted arrows**: execution steps labeled in succession