Network Viewer

* #361: Network Display * #381: Network Analysis sub-page
man-group · Jan 4, 2021 · c462f66 · c462f66
1 parent 06db702
commit c462f66
Show file tree

Hide file tree

Showing 37 changed files with 1,545 additions and 163 deletions.
diff --git a/dtale/static/css/main.css b/dtale/static/css/main.css
@@ -10764,6 +10764,13 @@ div.container-fluid.describe > div#popup-content > div.modal-body {
 div.container-fluid.describe div.describe-dtypes-grid-col {
     height: calc(100vh - 112px);
 }
+div.container-fluid.network {
+  height: 100%;
+}
+div.container-fluid.network > #content {
+  height: 100%;
+  padding-top: 0.5em;
+}
 div.modal-dialog div.modal-body.describe-body {
   height: 500px;
   overflow-y: auto;

diff --git a/dtale/templates/dtale/base.html b/dtale/templates/dtale/base.html
@@ -1,5 +1,5 @@
 <!doctype html>
-<html>
+<html style="{{ 'height: 100%' if network else '' }}">
     <head>
 	<meta charset="utf-8">
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
@@ -22,7 +22,7 @@
     #}
     <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='css/main.css') }}" />
     </head>
-    <body class="{{ theme }}-mode">
+    <body class="{{ theme }}-mode" style="{{ 'height: 100%' if network else '' }}">
         {% if github_fork %}
         <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='css/github_fork.css') }}" />
         <span id="forkongithub">

diff --git a/dtale/templates/dtale/network.html b/dtale/templates/dtale/network.html
@@ -0,0 +1,13 @@
+{% extends "dtale/base.html" %}
+
+{% set network = True %}
+{% block full_content %}
+<div class="container-fluid network">
+    <div id="content">
+    </div>
+</div>
+{% endblock %}
+
+{% block js %}
+<script type="text/javascript" src="{{ url_for('static', filename='dist/network_bundle.js') }}"></script>
+{% endblock %}
diff --git a/dtale/views.py b/dtale/views.py
@@ -10,6 +10,7 @@
 
 from flask import current_app, json, make_response, redirect, render_template, request
 
+import networkx as nx
 import numpy as np
 import pandas as pd
 import platform
@@ -1043,6 +1044,23 @@ def view_calculation(calc_type="skew"):
     return render_template("dtale/{}.html".format(calc_type))
 
 
+@dtale.route("/network")
+@dtale.route("/network/<data_id>")
+def view_network(data_id=None):
+    """
+    :class:`flask:flask.Flask` route which serves up base jinja template housing JS files
+
+    :param data_id: integer string identifier for a D-Tale process's data
+    :type data_id: str
+    :return: HTML
+    """
+    if data_id is None or data_id not in global_state.get_data().keys():
+        return redirect("/dtale/network/{}".format(head_endpoint()))
+    return base_render_template(
+        "dtale/network.html", data_id, title="Network Viewer", iframe=False
+    )
+
+
 @dtale.route("/code-popup")
 def view_code_popup():
     """
@@ -3148,3 +3166,106 @@ def build_row_text(data_id):
         end = int(end)
         data = data.iloc[(start - 1) : end, :]
     return data[columns].to_csv(index=False, sep="\t", header=False)
+
+
+@dtale.route("/network-data/<data_id>")
+@exception_decorator
+def network_data(data_id):
+    df = global_state.get_data(data_id)
+    to_col = get_str_arg(request, "to")
+    from_col = get_str_arg(request, "from")
+    group = get_str_arg(request, "group", "")
+    weight = get_str_arg(request, "weight")
+
+    nodes = list(df[to_col].unique())
+    nodes += list(df[~df[from_col].isin(nodes)][from_col].unique())
+    nodes = sorted(nodes)
+    nodes = {node: node_id for node_id, node in enumerate(nodes, 1)}
+
+    edge_cols = [to_col, from_col]
+    if weight:
+        edge_cols.append(weight)
+
+    edges = df[[to_col, from_col]].applymap(nodes.get)
+    edges.columns = ["to", "from"]
+    if weight:
+        edges.loc[:, "value"] = df[weight]
+    edges = edges.to_dict(orient="records")
+
+    if group:
+        group = df[[from_col, group]].set_index(from_col)[group].astype("str").to_dict()
+    else:
+        group = {}
+
+    groups = {}
+
+    def build_group(node, node_id):
+        group_val = group.get(node, "N/A")
+        groups[group_val] = node_id
+        return group_val
+
+    nodes = [
+        dict(id=node_id, label=node, group=build_group(node, node_id))
+        for node, node_id in nodes.items()
+    ]
+    return jsonify(dict(nodes=nodes, edges=edges, groups=groups, success=True))
+
+
+@dtale.route("/network-analysis/<data_id>")
+@exception_decorator
+def network_analysis(data_id):
+    df = global_state.get_data(data_id)
+    to_col = get_str_arg(request, "to")
+    from_col = get_str_arg(request, "from")
+    weight = get_str_arg(request, "weight")
+
+    G = nx.Graph()
+    max_edge, min_edge, avg_weight = (None, None, None)
+    if weight:
+        G.add_weighted_edges_from(
+            [tuple(x) for x in df[[to_col, from_col, weight]].values]
+        )
+        sorted_edges = sorted(
+            G.edges(data=True), key=lambda x: x[2]["weight"], reverse=True
+        )
+        max_edge = sorted_edges[0]
+        min_edge = sorted_edges[-1]
+        avg_weight = df[weight].mean()
+    else:
+        G.add_edges_from([tuple(x) for x in df[[to_col, from_col]].values])
+
+    most_connected_node = max(dict(G.degree()).items(), key=lambda x: x[1])
+    return_data = {
+        "node_ct": len(G),
+        "triangle_ct": int(sum(nx.triangles(G).values()) / 3),
+        "most_connected_node": "{} (Connections: {})".format(*most_connected_node),
+        "leaf_ct": sum((1 for edge, degree in dict(G.degree()).items() if degree == 1)),
+        "edge_ct": sum(dict(G.degree()).values()),
+        "max_edge": None
+        if max_edge is None
+        else "{} (source: {}, target: {})".format(
+            max_edge[-1]["weight"], max_edge[0], max_edge[1]
+        ),
+        "min_edge": None
+        if min_edge is None
+        else "{} (source: {}, target: {})".format(
+            min_edge[-1]["weight"], min_edge[0], min_edge[1]
+        ),
+        "avg_weight": json_float(avg_weight),
+    }
+    return jsonify(dict(data=return_data, success=True))
+
+
+@dtale.route("/shortest-path/<data_id>")
+@exception_decorator
+def shortest_path(data_id):
+    df = global_state.get_data(data_id)
+    to_col = get_str_arg(request, "to")
+    from_col = get_str_arg(request, "from")
+    start_val = get_str_arg(request, "start")
+    end_val = get_str_arg(request, "end")
+
+    G = nx.Graph()
+    G.add_edges_from([tuple(x) for x in df[[to_col, from_col]].values])
+    shortest_path = nx.shortest_path(G, source=start_val, target=end_val)
+    return jsonify(dict(data=shortest_path, success=True))
diff --git a/package.json b/package.json
@@ -159,9 +159,10 @@
     "serialize-javascript": "5.0.1",
     "string.prototype.startswith": "1.0.0",
     "styled-components": "5.2.1",
-    "uuid": "8.3.2"
+    "uuid": "8.3.2",
+    "vis-network": "8.5.6"
   },
   "resolutions": {
     "node-notifier": "8.0.1"
   }
- }
+}
diff --git a/setup.py b/setup.py
@@ -86,6 +86,8 @@ def run_tests(self):
         "future >= 0.14.0",
         "itsdangerous",
         "kaleido",
+        "networkx == 2.2; python_version < '3.0'",
+        "networkx; python_version >= '3.0'",
         "pandas",
         "plotly>=4.9.0",
         "ppscore; python_version >= '3.6'",

diff --git a/static/ButtonToggle.jsx b/static/ButtonToggle.jsx
@@ -22,6 +22,9 @@ class ButtonToggle extends React.Component {
           const buttonProps = { className: "btn" };
           if (value === this.state.active) {
             buttonProps.className += " btn-primary active";
+            if (this.props.allowDeselect) {
+              buttonProps.onClick = () => this.setState({ active: null }, () => update(null));
+            }
           } else {
             buttonProps.className += " btn-primary inactive";
             buttonProps.onClick = () => this.setState({ active: value }, () => update(value));
@@ -41,6 +44,8 @@ ButtonToggle.propTypes = {
   options: PropTypes.array,
   update: PropTypes.func,
   defaultValue: PropTypes.string,
+  allowDeselect: PropTypes.bool,
 };
+ButtonToggle.defaultProps = { allowDeselect: false };
 
 export default ButtonToggle;
diff --git a/static/Collapsible.jsx b/static/Collapsible.jsx
@@ -0,0 +1,39 @@
+import _ from "lodash";
+import PropTypes from "prop-types";
+import React from "react";
+
+require("./Collapsible.scss");
+
+class Collapsible extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = { isOpen: false };
+  }
+
+  render() {
+    const { title, content } = this.props;
+    if (!content) {
+      return null;
+    }
+    const { isOpen } = this.state;
+    const onClick = () => this.setState({ isOpen: !this.state.isOpen }, this.props.onExpand ?? _.noop);
+    return (
+      <dl className="accordion pt-3">
+        <dt className={`accordion-title${isOpen ? " is-expanded" : ""} pointer pl-3`} onClick={onClick}>
+          {title}
+        </dt>
+        <dd className={`accordion-content${isOpen ? " is-expanded" : ""}`} onClick={onClick}>
+          {content}
+        </dd>
+      </dl>
+    );
+  }
+}
+Collapsible.displayName = "Collapsible";
+Collapsible.propTypes = {
+  title: PropTypes.oneOfType([PropTypes.string, PropTypes.node]),
+  content: PropTypes.node,
+  onExpand: PropTypes.func,
+};
+
+export default Collapsible;
diff --git a/...c/popups/correlations/PPSCollapsible.scss → static/Collapsible.scss b/...c/popups/correlations/PPSCollapsible.scss → static/Collapsible.scss
diff --git a/static/__tests__/dtale/DataViewer-base-test.jsx b/static/__tests__/dtale/DataViewer-base-test.jsx
@@ -77,7 +77,7 @@ describe("DataViewer tests", () => {
     ).toEqual(
       _.concat(
         ["Convert To XArray", "Describe", "Custom Filter", "Build Column", "Summarize Data", "Duplicates"],
-        ["Correlations", "Predictive Power Score", "Charts", "Heat Map", "Highlight Dtypes"],
+        ["Correlations", "Predictive Power Score", "Charts", "Network Viewer", "Heat Map", "Highlight Dtypes"],
         ["Highlight Missing", "Highlight Outliers", "Highlight Range", "Low Variance Flag", "Instances 1"],
         ["Code Export", "Export", "Load Data", "Refresh Widths", "About", "Theme", "Reload Data", "Shutdown"]
       )

diff --git a/static/__tests__/iframe/DataViewer-base-test.jsx b/static/__tests__/iframe/DataViewer-base-test.jsx
@@ -115,7 +115,7 @@ describe("DataViewer iframe tests", () => {
     ).toEqual(
       _.concat(
         ["XArray Dimensions", "Describe", "Custom Filter", "Build Column", "Summarize Data", "Duplicates"],
-        ["Correlations", "Predictive Power Score", "Charts", "Heat Map", "Highlight Dtypes"],
+        ["Correlations", "Predictive Power Score", "Charts", "Network Viewer", "Heat Map", "Highlight Dtypes"],
         ["Highlight Missing", "Highlight Outliers", "Highlight Range", "Low Variance Flag", "Instances 1"],
         ["Code Export", "Export", "Load Data", "Refresh Widths", "About", "Theme", "Reload Data", "Open In New Tab"],
         ["Shutdown"]
@@ -217,6 +217,8 @@ describe("DataViewer iframe tests", () => {
     expect(window.open.mock.calls[window.open.mock.calls.length - 1][0]).toBe("/dtale/popup/correlations/1");
     clickMainMenuButton(result, "Charts");
     expect(window.open.mock.calls[window.open.mock.calls.length - 1][0]).toBe("/dtale/charts/1");
+    clickMainMenuButton(result, "Network Viewer");
+    expect(window.open.mock.calls[window.open.mock.calls.length - 1][0]).toBe("/dtale/network/1");
     clickMainMenuButton(result, "Instances 1");
     expect(window.open.mock.calls[window.open.mock.calls.length - 1][0]).toBe("/dtale/popup/instances/1");
     const exports = findMainMenuButton(result, "CSV", "div.btn-group");

diff --git a/static/__tests__/iframe/DataViewer-within-iframe-test.jsx b/static/__tests__/iframe/DataViewer-within-iframe-test.jsx
@@ -88,7 +88,7 @@ describe("DataViewer within iframe tests", () => {
     ).toEqual(
       _.concat(
         ["Convert To XArray", "Describe", "Custom Filter", "Build Column", "Summarize Data", "Duplicates"],
-        ["Correlations", "Predictive Power Score", "Charts", "Heat Map", "Highlight Dtypes"],
+        ["Correlations", "Predictive Power Score", "Charts", "Network Viewer", "Heat Map", "Highlight Dtypes"],
         ["Highlight Missing", "Highlight Outliers", "Highlight Range", "Low Variance Flag", "Instances 1"],
         ["Code Export", "Export", "Load Data", "Refresh Widths", "About", "Theme", "Reload Data", "Open In New Tab"],
         ["Shutdown"]

diff --git a/static/__tests__/main-test.jsx b/static/__tests__/main-test.jsx
@@ -43,20 +43,22 @@ describe("main tests", () => {
     window.opener = opener;
   });
 
-  const testMain = (mainName, search = "") => {
+  const testMain = (mainName, search = "", fname = "main") => {
     window.location = { pathname: `/dtale/${mainName}/1`, search };
     buildInnerHTML();
     const mockReactDOM = { renderStatus: false };
     mockReactDOM.render = () => {
       mockReactDOM.renderStatus = true;
     };
     withGlobalJquery(() => jest.mock("react-dom", () => mockReactDOM));
-    require(`../main`);
+    require(`../${fname}`);
     expect(mockReactDOM.renderStatus).toBe(true);
   };
 
   it("main rendering", () => testMain("main"));
 
+  it("network main rendering", () => testMain("network/1", "", "network/main"));
+
   it("base_styles.js loading", () => {
     require("../base_styles");
     return;