From 5220dfe6b5985015cc6c95ea0fdebab211ea63aa Mon Sep 17 00:00:00 2001 From: jarusified Date: Tue, 11 Aug 2020 01:44:55 -0700 Subject: [PATCH 01/13] Basic notebook magic --- callflow/__init__.py | 12 +++++ callflow/notebook.py | 107 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 callflow/notebook.py diff --git a/callflow/__init__.py b/callflow/__init__.py index 75d688cd..3cfd2007 100644 --- a/callflow/__init__.py +++ b/callflow/__init__.py @@ -15,7 +15,9 @@ from .datastructures.ensemblegraph import EnsembleGraph from .callflow import CallFlow +from .notebook import _load_ipython_extension +# CallFlow's public API. __all__ = [ "init_logger", "get_logger", @@ -23,4 +25,14 @@ "SuperGraph", "EnsembleGraph", "CallFlow", + "notebook", ] + + +def load_ipython_extension(ipython): + """IPython API entry point. + Only intended to be called by the IPython runtime. + See: + https://ipython.readthedocs.io/en/stable/config/extensions/index.html + """ + _load_ipython_extension(ipython) diff --git a/callflow/notebook.py b/callflow/notebook.py new file mode 100644 index 00000000..17af2fe9 --- /dev/null +++ b/callflow/notebook.py @@ -0,0 +1,107 @@ +# Copyright 2017-2020 Lawrence Livermore National Security, LLC and other +# CallFlow Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: MIT +import shlex +import json + + +def _load_ipython_extension(ipython): + """Load the TensorBoard notebook extension. + Intended to be called from `%load_ext callflow`. Do not invoke this + directly. + Args: + ipython: An `IPython.InteractiveShell` instance. + """ + _register_magics(ipython) + + +def _register_magics(ipython): + """Register IPython line/cell magics. + Args: + ipython: An `InteractiveShell` instance. + """ + ipython.register_magic_function( + _start_magic, magic_kind="line", magic_name="callflow", + ) + + +def _start_magic(line): + """Implementation of the `%callflow` line magic.""" + return start(line) + + +def start(args_string): + """Launch and display a TensorBoard instance as if at the command line. + Args: + args_string: Command-line arguments to TensorBoard, to be + interpreted by `shlex.split`: e.g., "--logdir ./logs --port 0". + Shell metacharacters are not supported: e.g., "--logdir 2>&1" will + point the logdir at the literal directory named "2>&1". + """ + try: + import IPython + import IPython.display + except ImportError: + IPython = None + + handle = IPython.display.display( + IPython.display.Pretty("Launching CallFlow..."), display_id=True, + ) + + def print_or_update(message): + if handle is None: + print(message) + else: + handle.update(IPython.display.Pretty(message)) + + parsed_args = shlex.split(args_string, comments=True, posix=True) + print(parsed_args) + + +def _display_ipython(port, height, display_handle): + import IPython.display + + frame_id = "callflow-frame-{:08x}".format(random.getrandbits(64)) + shell = """ + + + """ + proxy_url = os.environ.get("TENSORBOARD_PROXY_URL") + if proxy_url is not None: + # Allow %PORT% in $TENSORBOARD_PROXY_URL + proxy_url = proxy_url.replace("%PORT%", "%d" % port) + replacements = [ + ("%HTML_ID%", html_escape(frame_id, quote=True)), + ("%JSON_ID%", json.dumps(frame_id)), + ("%HEIGHT%", "%d" % height), + ("%PORT%", "0"), + ("%URL%", json.dumps(proxy_url)), + ] + else: + replacements = [ + ("%HTML_ID%", html_escape(frame_id, quote=True)), + ("%JSON_ID%", json.dumps(frame_id)), + ("%HEIGHT%", "%d" % height), + ("%PORT%", "%d" % port), + ("%URL%", json.dumps("/")), + ] + + for (k, v) in replacements: + shell = shell.replace(k, v) + iframe = IPython.display.HTML(shell) + if display_handle: + display_handle.update(iframe) + else: + IPython.display.display(iframe) From 2fc01b339453556dbf57bd08dee61bad5e9a43da Mon Sep 17 00:00:00 2001 From: jarusified Date: Wed, 12 Aug 2020 16:00:28 -0700 Subject: [PATCH 02/13] Add a sample notebook and some documentation --- callflow/manager.py | 132 +++++++++++++++++++++++++ callflow/notebook.py | 25 ++++- examples/%callflow-ipython-magic.ipynb | 78 +++++++++++++++ 3 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 callflow/manager.py create mode 100644 examples/%callflow-ipython-magic.ipynb diff --git a/callflow/manager.py b/callflow/manager.py new file mode 100644 index 00000000..517fc5c6 --- /dev/null +++ b/callflow/manager.py @@ -0,0 +1,132 @@ +# Copyright 2017-2020 Lawrence Livermore National Security, LLC and other +# CallFlow Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: MIT + +# NOTE: The manager.py adopts Tensorboard's philosophy of launching applications +# through the IPython interface. +# The code can be found at https://github.com/tensorflow/tensorboard/blob/master/tensorboard/manager.py + +import os +import time +import tempfile +import subprocess +import collections +import datetime +import errno + +# The following five types enumerate the possible return values of the +# `start` function. +# Indicates that a call to `start` was compatible with an existing +# CallFlow process, which can be reused according to the provided +# info. +StartReused = collections.namedtuple("StartReused", ("info",)) + +# Indicates that a call to `start` successfully launched a new +# CallFlow process, which is available with the provided info. +StartLaunched = collections.namedtuple("StartLaunched", ("info",)) + +# Indicates that a call to `start` tried to launch a new CallFlow +# instance, but the subprocess exited with the given exit code and +# output streams. +StartFailed = collections.namedtuple( + "StartFailed", + ( + "exit_code", # int, as `Popen.returncode` (negative for signal) + "stdout", # str, or `None` if the stream could not be read + "stderr", # str, or `None` if the stream could not be read + ), +) + +# Indicates that a call to `start` failed to invoke the subprocess. +StartExecFailed = collections.namedtuple( + "StartExecFailed", ("os_error",), # `OSError` due to `Popen` invocation +) + +# Indicates that a call to `start` launched a CallFlow process, but +# that process neither exited nor wrote its info file within the allowed +# timeout period. The process may still be running under the included +# PID. +StartTimedOut = collections.namedtuple("StartTimedOut", ("pid",)) + + +def _exec(cmd): + """ + cmd is expected to be something like "cd [place]" + """ + cmd = cmd + " && pwd" + p = subprocess.Popen( + cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + + out = p.stdout.read() + err = p.stderr.read() + + if out != "": + os.chdir(out[0 : len(out) - 1]) + if err != "": + print(err) + return + + +def start(args, timeout=datetime.timedelta(seconds=60)): + """ + Start a CallFlow (server and client) as a subprocess in the background. + TODO: Improve logic to check if there is a callflow process already. + TODO: Fix the path not found error. + """ + (stdout_fd, stdout_path) = tempfile.mkstemp(prefix=".callflow-stdout-") + (stderr_fd, stderr_path) = tempfile.mkstemp(prefix=".callflow-stderr-") + start_time_seconds = time.time() + try: + p = subprocess.Popen( + "python3 server/main.py " + " ".join(args), + stdout=stdout_fd, + stderr=stderr_fd, + ) + + from subprocess import check_output + + out = check_output(["ls"]) + except OSError as e: + return StartExecFailed(os_error=e) + finally: + os.close(stdout_fd) + os.close(stderr_fd) + + poll_interval_seconds = 0.5 + end_time_seconds = start_time_seconds + timeout.total_seconds() + while time.time() < end_time_seconds: + time.sleep(poll_interval_seconds) + subprocess_result = p.poll() + if subprocess_result is not None: + return StartFailed( + exit_code=subprocess_result, + stdout=_maybe_read_file(stdout_path), + stderr=_maybe_read_file(stderr_path), + ) + # for info in get_all(): + # if info.pid == p.pid and info.start_time >= start_time_seconds: + return StartLaunched(info=out) + else: + return StartTimedOut(pid=p.pid) + + +def _maybe_read_file(filename): + """Read the given file, if it exists. + Args: + filename: A path to a file. + Returns: + A string containing the file contents, or `None` if the file does + not exist. + """ + try: + with open(filename) as infile: + return infile.read() + except IOError as e: + if e.errno == errno.ENOENT: + return None diff --git a/callflow/notebook.py b/callflow/notebook.py index 17af2fe9..4aaf31a5 100644 --- a/callflow/notebook.py +++ b/callflow/notebook.py @@ -2,8 +2,27 @@ # CallFlow Project Developers. See the top-level LICENSE file for details. # # SPDX-License-Identifier: MIT + +# NOTE: The manager.py adopts Tensorboard's philosophy of launching applications +# through the IPython interface. +# The code can be found at https://github.com/tensorflow/tensorboard/blob/master/tensorboard/notebook.py + import shlex import json +import random + +try: + import html + + html_escape = html.escape + del html +except ImportError: + import cgi + + html_escape = cgi.escape + del cgi + +from callflow import manager def _load_ipython_extension(ipython): @@ -56,7 +75,11 @@ def print_or_update(message): handle.update(IPython.display.Pretty(message)) parsed_args = shlex.split(args_string, comments=True, posix=True) - print(parsed_args) + start_result = manager.start(parsed_args) + IPython.display.display( + IPython.display.Pretty(start_result), display_id=True, + ) + print(start_result) def _display_ipython(port, height, display_handle): diff --git a/examples/%callflow-ipython-magic.ipynb b/examples/%callflow-ipython-magic.ipynb new file mode 100644 index 00000000..d92123b7 --- /dev/null +++ b/examples/%callflow-ipython-magic.ipynb @@ -0,0 +1,78 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import callflow" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext callflow" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Launching CallFlow..." + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "ename": "TypeError", + "evalue": "Pretty expects text, not StartLaunched(info=b'AMM-15.ipynb\\nBland-altman.ipynb\\ncallflow_cct.dot\\ncallflow_cct.json\\ncallflow_cct.pdf\\nCallFlow integration.ipynb\\nCallFlow-interface-demo-with-ensemble.ipynb\\nCallFlow-python-interface-demo.ipynb\\ncccpa.ipynb\\nCCT using plotly.ipynb\\nCompare-two-gfs.ipynb\\nConfig File Dumper.ipynb\\ncreate_calc_pi_datasets.ipynb\\nFilter.ipynb\\nGromov-wasserstein distance.ipynb\\nGroup graph-Copy1.ipynb\\nGroup graph.ipynb\\nHierarchy.ipynb\\nInformation.ipynb\\nJupyter-pre-squash-and-post-squash-demo.ipynb\\nKripke-Multiple.ipynb\\nKripke multiple module mapping.ipynb\\nKripke-Scaling.ipynb\\nKripke-Scaling Module mapping.ipynb\\nLulesh-8-runs-Copy2.ipynb\\nLulesh-8-runs.ipynb\\nLulesh Module aggregation.ipynb\\nMake graph equivalent.ipynb\\nnetworkx_example.ipynb\\nnxGraph.ipynb\\nOSU_bcast - 10 ranks.ipynb\\nOSU_bcast - 18 ranks.ipynb\\nOSU_bcast - all.ipynb\\nOSU Bcast Module mapping.ipynb\\nosu_bcast_projection_view.ipynb\\npre-squash and post-squash result.ipynb\\n__pycache__\\nQuantiles.ipynb\\nReduced entire_df.ipynb\\nReveal paths.ipynb\\nSimilarity.ipynb\\ntest_create_graphframe.ipynb\\ntest-lulesh-cali.ipynb\\ntest processing pipeline.ipynb\\ntest rendering pipeline.ipynb\\ntest-vue.ipynb\\nUnion.ipynb\\nVerify_hierarchy.ipynb\\n')", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_line_magic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'callflow'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'--config /home/suraj/Work/llnl/CallFlow/data/lulesh-8-runs/callflow.config.js'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mrun_line_magic\u001b[0;34m(self, magic_name, line, _stack_depth)\u001b[0m\n\u001b[1;32m 2305\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'local_ns'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getframe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstack_depth\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf_locals\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2306\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2307\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2308\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2309\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Work/llnl/CallFlow/callflow/notebook.py\u001b[0m in \u001b[0;36m_start_magic\u001b[0;34m(line)\u001b[0m\n\u001b[1;32m 43\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_start_magic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 44\u001b[0m \u001b[0;34m\"\"\"Implementation of the `%callflow` line magic.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 46\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Work/llnl/CallFlow/callflow/notebook.py\u001b[0m in \u001b[0;36mstart\u001b[0;34m(args_string)\u001b[0m\n\u001b[1;32m 73\u001b[0m \u001b[0mstart_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmanager\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparsed_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 74\u001b[0m IPython.display.display(\n\u001b[0;32m---> 75\u001b[0;31m \u001b[0mIPython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdisplay\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPretty\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart_result\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdisplay_id\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 76\u001b[0m )\n\u001b[1;32m 77\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart_result\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/IPython/core/display.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, data, url, filename, metadata)\u001b[0m\n\u001b[1;32m 626\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 627\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 628\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 629\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 630\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__repr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/IPython/core/display.py\u001b[0m in \u001b[0;36m_check_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 675\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_check_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 676\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 677\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"%s expects text, not %r\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 678\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 679\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mPretty\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTextDisplayObject\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mTypeError\u001b[0m: Pretty expects text, not StartLaunched(info=b'AMM-15.ipynb\\nBland-altman.ipynb\\ncallflow_cct.dot\\ncallflow_cct.json\\ncallflow_cct.pdf\\nCallFlow integration.ipynb\\nCallFlow-interface-demo-with-ensemble.ipynb\\nCallFlow-python-interface-demo.ipynb\\ncccpa.ipynb\\nCCT using plotly.ipynb\\nCompare-two-gfs.ipynb\\nConfig File Dumper.ipynb\\ncreate_calc_pi_datasets.ipynb\\nFilter.ipynb\\nGromov-wasserstein distance.ipynb\\nGroup graph-Copy1.ipynb\\nGroup graph.ipynb\\nHierarchy.ipynb\\nInformation.ipynb\\nJupyter-pre-squash-and-post-squash-demo.ipynb\\nKripke-Multiple.ipynb\\nKripke multiple module mapping.ipynb\\nKripke-Scaling.ipynb\\nKripke-Scaling Module mapping.ipynb\\nLulesh-8-runs-Copy2.ipynb\\nLulesh-8-runs.ipynb\\nLulesh Module aggregation.ipynb\\nMake graph equivalent.ipynb\\nnetworkx_example.ipynb\\nnxGraph.ipynb\\nOSU_bcast - 10 ranks.ipynb\\nOSU_bcast - 18 ranks.ipynb\\nOSU_bcast - all.ipynb\\nOSU Bcast Module mapping.ipynb\\nosu_bcast_projection_view.ipynb\\npre-squash and post-squash result.ipynb\\n__pycache__\\nQuantiles.ipynb\\nReduced entire_df.ipynb\\nReveal paths.ipynb\\nSimilarity.ipynb\\ntest_create_graphframe.ipynb\\ntest-lulesh-cali.ipynb\\ntest processing pipeline.ipynb\\ntest rendering pipeline.ipynb\\ntest-vue.ipynb\\nUnion.ipynb\\nVerify_hierarchy.ipynb\\n')" + ] + } + ], + "source": [ + "%callflow --config /home/suraj/Work/llnl/CallFlow/data/lulesh-8-runs/callflow.config.js" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.7.4 64-bit (conda)", + "language": "python", + "name": "python37464bitconda681e547a9b06483ebedfd5d049542d16" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 43c19d0c7a13f2f6e361135c4d211bb541c10be0 Mon Sep 17 00:00:00 2001 From: jarusified Date: Sat, 15 Aug 2020 23:25:15 -0700 Subject: [PATCH 03/13] Fix the subprocess error --- callflow/manager.py | 9 ++++----- callflow/notebook.py | 4 +--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/callflow/manager.py b/callflow/manager.py index 517fc5c6..ba430ea6 100644 --- a/callflow/manager.py +++ b/callflow/manager.py @@ -81,17 +81,16 @@ def start(args, timeout=datetime.timedelta(seconds=60)): """ (stdout_fd, stdout_path) = tempfile.mkstemp(prefix=".callflow-stdout-") (stderr_fd, stderr_path) = tempfile.mkstemp(prefix=".callflow-stderr-") + cwd = os.getcwd().split('CallFlow')[0] + "CallFlow/server/main.py" + cmd = ["python3", cwd] + args start_time_seconds = time.time() try: p = subprocess.Popen( - "python3 server/main.py " + " ".join(args), + cmd, stdout=stdout_fd, stderr=stderr_fd, ) - from subprocess import check_output - - out = check_output(["ls"]) except OSError as e: return StartExecFailed(os_error=e) finally: @@ -111,7 +110,7 @@ def start(args, timeout=datetime.timedelta(seconds=60)): ) # for info in get_all(): # if info.pid == p.pid and info.start_time >= start_time_seconds: - return StartLaunched(info=out) + return StartLaunched(info="Started" + stdout_path) else: return StartTimedOut(pid=p.pid) diff --git a/callflow/notebook.py b/callflow/notebook.py index 4aaf31a5..13aedff2 100644 --- a/callflow/notebook.py +++ b/callflow/notebook.py @@ -76,9 +76,7 @@ def print_or_update(message): parsed_args = shlex.split(args_string, comments=True, posix=True) start_result = manager.start(parsed_args) - IPython.display.display( - IPython.display.Pretty(start_result), display_id=True, - ) + print(start_result) From 9a4967f1435fcbfa4037ff789f592b0421591f06 Mon Sep 17 00:00:00 2001 From: jarusified Date: Tue, 18 Aug 2020 16:41:32 -0700 Subject: [PATCH 04/13] Add the sample notebook --- examples/%callflow-ipython-magic.ipynb | 134 +++++++++++++++++++++---- 1 file changed, 116 insertions(+), 18 deletions(-) diff --git a/examples/%callflow-ipython-magic.ipynb b/examples/%callflow-ipython-magic.ipynb index d92123b7..62b62e9b 100644 --- a/examples/%callflow-ipython-magic.ipynb +++ b/examples/%callflow-ipython-magic.ipynb @@ -20,7 +20,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -33,32 +33,130 @@ "output_type": "display_data" }, { - "ename": "TypeError", - "evalue": "Pretty expects text, not StartLaunched(info=b'AMM-15.ipynb\\nBland-altman.ipynb\\ncallflow_cct.dot\\ncallflow_cct.json\\ncallflow_cct.pdf\\nCallFlow integration.ipynb\\nCallFlow-interface-demo-with-ensemble.ipynb\\nCallFlow-python-interface-demo.ipynb\\ncccpa.ipynb\\nCCT using plotly.ipynb\\nCompare-two-gfs.ipynb\\nConfig File Dumper.ipynb\\ncreate_calc_pi_datasets.ipynb\\nFilter.ipynb\\nGromov-wasserstein distance.ipynb\\nGroup graph-Copy1.ipynb\\nGroup graph.ipynb\\nHierarchy.ipynb\\nInformation.ipynb\\nJupyter-pre-squash-and-post-squash-demo.ipynb\\nKripke-Multiple.ipynb\\nKripke multiple module mapping.ipynb\\nKripke-Scaling.ipynb\\nKripke-Scaling Module mapping.ipynb\\nLulesh-8-runs-Copy2.ipynb\\nLulesh-8-runs.ipynb\\nLulesh Module aggregation.ipynb\\nMake graph equivalent.ipynb\\nnetworkx_example.ipynb\\nnxGraph.ipynb\\nOSU_bcast - 10 ranks.ipynb\\nOSU_bcast - 18 ranks.ipynb\\nOSU_bcast - all.ipynb\\nOSU Bcast Module mapping.ipynb\\nosu_bcast_projection_view.ipynb\\npre-squash and post-squash result.ipynb\\n__pycache__\\nQuantiles.ipynb\\nReduced entire_df.ipynb\\nReveal paths.ipynb\\nSimilarity.ipynb\\ntest_create_graphframe.ipynb\\ntest-lulesh-cali.ipynb\\ntest processing pipeline.ipynb\\ntest rendering pipeline.ipynb\\ntest-vue.ipynb\\nUnion.ipynb\\nVerify_hierarchy.ipynb\\n')", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_line_magic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'callflow'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'--config /home/suraj/Work/llnl/CallFlow/data/lulesh-8-runs/callflow.config.js'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mrun_line_magic\u001b[0;34m(self, magic_name, line, _stack_depth)\u001b[0m\n\u001b[1;32m 2305\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'local_ns'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getframe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstack_depth\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf_locals\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2306\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2307\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2308\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2309\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Work/llnl/CallFlow/callflow/notebook.py\u001b[0m in \u001b[0;36m_start_magic\u001b[0;34m(line)\u001b[0m\n\u001b[1;32m 43\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_start_magic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 44\u001b[0m \u001b[0;34m\"\"\"Implementation of the `%callflow` line magic.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 46\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Work/llnl/CallFlow/callflow/notebook.py\u001b[0m in \u001b[0;36mstart\u001b[0;34m(args_string)\u001b[0m\n\u001b[1;32m 73\u001b[0m \u001b[0mstart_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmanager\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparsed_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 74\u001b[0m IPython.display.display(\n\u001b[0;32m---> 75\u001b[0;31m \u001b[0mIPython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdisplay\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPretty\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart_result\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdisplay_id\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 76\u001b[0m )\n\u001b[1;32m 77\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart_result\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/IPython/core/display.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, data, url, filename, metadata)\u001b[0m\n\u001b[1;32m 626\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 627\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 628\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 629\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 630\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__repr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/IPython/core/display.py\u001b[0m in \u001b[0;36m_check_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 675\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_check_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 676\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 677\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"%s expects text, not %r\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 678\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 679\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mPretty\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTextDisplayObject\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mTypeError\u001b[0m: Pretty expects text, not StartLaunched(info=b'AMM-15.ipynb\\nBland-altman.ipynb\\ncallflow_cct.dot\\ncallflow_cct.json\\ncallflow_cct.pdf\\nCallFlow integration.ipynb\\nCallFlow-interface-demo-with-ensemble.ipynb\\nCallFlow-python-interface-demo.ipynb\\ncccpa.ipynb\\nCCT using plotly.ipynb\\nCompare-two-gfs.ipynb\\nConfig File Dumper.ipynb\\ncreate_calc_pi_datasets.ipynb\\nFilter.ipynb\\nGromov-wasserstein distance.ipynb\\nGroup graph-Copy1.ipynb\\nGroup graph.ipynb\\nHierarchy.ipynb\\nInformation.ipynb\\nJupyter-pre-squash-and-post-squash-demo.ipynb\\nKripke-Multiple.ipynb\\nKripke multiple module mapping.ipynb\\nKripke-Scaling.ipynb\\nKripke-Scaling Module mapping.ipynb\\nLulesh-8-runs-Copy2.ipynb\\nLulesh-8-runs.ipynb\\nLulesh Module aggregation.ipynb\\nMake graph equivalent.ipynb\\nnetworkx_example.ipynb\\nnxGraph.ipynb\\nOSU_bcast - 10 ranks.ipynb\\nOSU_bcast - 18 ranks.ipynb\\nOSU_bcast - all.ipynb\\nOSU Bcast Module mapping.ipynb\\nosu_bcast_projection_view.ipynb\\npre-squash and post-squash result.ipynb\\n__pycache__\\nQuantiles.ipynb\\nReduced entire_df.ipynb\\nReveal paths.ipynb\\nSimilarity.ipynb\\ntest_create_graphframe.ipynb\\ntest-lulesh-cali.ipynb\\ntest processing pipeline.ipynb\\ntest rendering pipeline.ipynb\\ntest-vue.ipynb\\nUnion.ipynb\\nVerify_hierarchy.ipynb\\n')" + "name": "stdout", + "output_type": "stream", + "text": [ + "StartLaunched(info='Started/var/folders/rv/jkb6mxgd0mbgz87qvq9356zw0000gn/T/.callflow-stdout-tk0r4y7r')\n" ] } ], "source": [ - "%callflow --config /home/suraj/Work/llnl/CallFlow/data/lulesh-8-runs/callflow.config.js" + "%callflow --config /Users/jarus/Work/llnl/CallFlow/data/lulesh-8-runs/callflow.config.js --process" ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "import tempfile\n", + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/Users/jarus/Work/llnl/\n" + ] + } + ], + "source": [ + "cwd = os.getcwd().split('CallFlow')[0]\n", + "print(cwd)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "(stdout_fd, stdout_path) = tempfile.mkstemp(prefix=\".callflow-stdout-\")\n", + "(stderr_fd, stderr_path) = tempfile.mkstemp(prefix=\".callflow-stderr-\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/var/folders/rv/jkb6mxgd0mbgz87qvq9356zw0000gn/T/.callflow-stdout-kwa0ff2_\n", + "/var/folders/rv/jkb6mxgd0mbgz87qvq9356zw0000gn/T/.callflow-stderr-9nhidwq0\n" + ] + } + ], + "source": [ + "print(stdout_path)\n", + "print(stderr_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "p = subprocess.Popen(\n", + " [\"python3\", cwd, \"/server/main.py\", \"--config\", \"/Work/llnl/CallFlow/data/lulesh-8-runs/callflow.config.json\"],\n", + " stdout=stdout_fd,\n", + " stderr=stderr_fd,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "None\n" + ] + } + ], + "source": [ + "subprocess_result = p.poll()\n", + "print(subprocess_result)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { + "finalized": { + "timestamp": 1597556178700, + "trusted": false + }, "kernelspec": { - "display_name": "Python 3.7.4 64-bit (conda)", + "display_name": "Python 3.7.6 64-bit", "language": "python", - "name": "python37464bitconda681e547a9b06483ebedfd5d049542d16" + "name": "python37664bit3a5637fa2c7f4443bca7a2894d18d23d" }, "language_info": { "codemirror_mode": { @@ -70,7 +168,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.4" + "version": "3.7.7" } }, "nbformat": 4, From a04bcac3f60e69ad4aba8288879ab5376649837e Mon Sep 17 00:00:00 2001 From: jarusified Date: Tue, 18 Aug 2020 18:48:11 -0700 Subject: [PATCH 05/13] Working callflow app on the notebook. Clean up left --- callflow/manager.py | 104 +++++++++++++++---- callflow/notebook.py | 37 +++---- examples/%callflow-ipython-magic.ipynb | 135 +++++-------------------- 3 files changed, 131 insertions(+), 145 deletions(-) diff --git a/callflow/manager.py b/callflow/manager.py index ba430ea6..82f0f6d6 100644 --- a/callflow/manager.py +++ b/callflow/manager.py @@ -14,6 +14,7 @@ import collections import datetime import errno +import json # The following five types enumerate the possible return values of the # `start` function. @@ -73,31 +74,97 @@ def _exec(cmd): return -def start(args, timeout=datetime.timedelta(seconds=60)): +def _get_info_dir(): + """Get path to directory in which to store info files. + The directory returned by this function is "owned" by this module. If + the contents of the directory are modified other than via the public + functions of this module, subsequent behavior is undefined. + The directory will be created if it does not exist. + """ + path = os.path.join(tempfile.gettempdir(), ".callflow-info") + try: + os.makedirs(path) + except OSError as e: + if e.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + else: + os.chmod(path, 0o777) + return path + + +def get_launch_information(): + info_dir = _get_info_dir() + results = [] + for filename in os.listdir(info_dir): + filepath = os.path.join(info_dir, filename) + try: + with open(filepath) as infile: + contents = infile.read() + except IOError as e: + if e.errno == errno.EACCES: + # May have been written by this module in a process whose + # `umask` includes some bits of 0o444. + continue + else: + raise + try: + info = contents + except ValueError: + # Ignore unrecognized files, logging at debug only. + print("invalid info file: %r", filepath) + else: + results.append(info) + return results + + +def start(args): """ Start a CallFlow (server and client) as a subprocess in the background. TODO: Improve logic to check if there is a callflow process already. TODO: Fix the path not found error. """ - (stdout_fd, stdout_path) = tempfile.mkstemp(prefix=".callflow-stdout-") - (stderr_fd, stderr_path) = tempfile.mkstemp(prefix=".callflow-stderr-") - cwd = os.getcwd().split('CallFlow')[0] + "CallFlow/server/main.py" - cmd = ["python3", cwd] + args + + """ + Launch python server. + """ + print("Launching Server") + cwd = os.getcwd().split("CallFlow")[0] + "CallFlow/server/main.py" + server_cmd = ["python3", cwd] + args + launch_cmd(server_cmd, alias="server") + + """ + Launch callflow app server. + """ + print("Launching client") + cwd = os.getcwd().split("CallFlow")[0] + "CallFlow/app" + prefix_string = ["--silent", "--prefix=" + cwd] + client_cmd = ["npm", "run", "dev"] + prefix_string + print(client_cmd) + launch_cmd(client_cmd, alias="client") + + return StartLaunched(info="Started") + + +def launch_cmd(cmd, timeout=datetime.timedelta(seconds=60), alias=""): + """ + Launch a cmd. + """ + stdprefix_path = "/tmp/.callflow-info/" + alias + "-" + (stdout_fd, stdout_path) = tempfile.mkstemp(prefix=stdprefix_path + "stdout-") + (stderr_fd, stderr_path) = tempfile.mkstemp(prefix=stdprefix_path + "stderr-") + start_time_seconds = time.time() try: - p = subprocess.Popen( - cmd, - stdout=stdout_fd, - stderr=stderr_fd, - ) - + p = subprocess.Popen(cmd, stdout=stdout_fd, stderr=stderr_fd,) except OSError as e: return StartExecFailed(os_error=e) finally: os.close(stdout_fd) os.close(stderr_fd) - poll_interval_seconds = 0.5 + poll_interval_seconds = 2 end_time_seconds = start_time_seconds + timeout.total_seconds() while time.time() < end_time_seconds: time.sleep(poll_interval_seconds) @@ -108,11 +175,14 @@ def start(args, timeout=datetime.timedelta(seconds=60)): stdout=_maybe_read_file(stdout_path), stderr=_maybe_read_file(stderr_path), ) - # for info in get_all(): - # if info.pid == p.pid and info.start_time >= start_time_seconds: - return StartLaunched(info="Started" + stdout_path) - else: - return StartTimedOut(pid=p.pid) + # for info in get_launch_information(): + # if info.pid == p.pid and info.start_time >= start_time_seconds: + info = get_launch_information() + return StartLaunched( + info={"out_path": stdout_path, "err_path": stderr_path, "pid": p.pid,} + ) + else: + return StartTimedOut(pid=p.pid) def _maybe_read_file(filename): diff --git a/callflow/notebook.py b/callflow/notebook.py index 13aedff2..51accc00 100644 --- a/callflow/notebook.py +++ b/callflow/notebook.py @@ -76,8 +76,13 @@ def print_or_update(message): parsed_args = shlex.split(args_string, comments=True, posix=True) start_result = manager.start(parsed_args) - - print(start_result) + + print("aaaa", start_result) + + if isinstance(start_result, manager.StartLaunched): + _display_ipython( + port=1024, height=500, display_handle=handle, + ) def _display_ipython(port, height, display_handle): @@ -99,29 +104,19 @@ def _display_ipython(port, height, display_handle): })(); """ - proxy_url = os.environ.get("TENSORBOARD_PROXY_URL") - if proxy_url is not None: - # Allow %PORT% in $TENSORBOARD_PROXY_URL - proxy_url = proxy_url.replace("%PORT%", "%d" % port) - replacements = [ - ("%HTML_ID%", html_escape(frame_id, quote=True)), - ("%JSON_ID%", json.dumps(frame_id)), - ("%HEIGHT%", "%d" % height), - ("%PORT%", "0"), - ("%URL%", json.dumps(proxy_url)), - ] - else: - replacements = [ - ("%HTML_ID%", html_escape(frame_id, quote=True)), - ("%JSON_ID%", json.dumps(frame_id)), - ("%HEIGHT%", "%d" % height), - ("%PORT%", "%d" % port), - ("%URL%", json.dumps("/")), - ] + + replacements = [ + ("%HTML_ID%", html_escape(frame_id, quote=True)), + ("%JSON_ID%", json.dumps(frame_id)), + ("%HEIGHT%", "%d" % height), + ("%PORT%", "%d" % port), + ("%URL%", json.dumps("/")), + ] for (k, v) in replacements: shell = shell.replace(k, v) iframe = IPython.display.HTML(shell) + print(iframe) if display_handle: display_handle.update(iframe) else: diff --git a/examples/%callflow-ipython-magic.ipynb b/examples/%callflow-ipython-magic.ipynb index 62b62e9b..413a8e77 100644 --- a/examples/%callflow-ipython-magic.ipynb +++ b/examples/%callflow-ipython-magic.ipynb @@ -20,13 +20,30 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 3, "metadata": {}, "outputs": [ { "data": { + "text/html": [ + "\n", + " \n", + " \n", + " " + ], "text/plain": [ - "Launching CallFlow..." + "" ] }, "metadata": {}, @@ -36,108 +53,16 @@ "name": "stdout", "output_type": "stream", "text": [ - "StartLaunched(info='Started/var/folders/rv/jkb6mxgd0mbgz87qvq9356zw0000gn/T/.callflow-stdout-tk0r4y7r')\n" + "Launching Server\n", + "Launching client\n", + "['npm', 'run', 'dev', '--silent', '--prefix=/home/suraj/Work/llnl/CallFlow/app']\n", + "aaaa StartLaunched(info='Started')\n", + "\n" ] } ], "source": [ - "%callflow --config /Users/jarus/Work/llnl/CallFlow/data/lulesh-8-runs/callflow.config.js --process" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "import subprocess\n", - "import tempfile\n", - "import os" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/Users/jarus/Work/llnl/\n" - ] - } - ], - "source": [ - "cwd = os.getcwd().split('CallFlow')[0]\n", - "print(cwd)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "(stdout_fd, stdout_path) = tempfile.mkstemp(prefix=\".callflow-stdout-\")\n", - "(stderr_fd, stderr_path) = tempfile.mkstemp(prefix=\".callflow-stderr-\")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/var/folders/rv/jkb6mxgd0mbgz87qvq9356zw0000gn/T/.callflow-stdout-kwa0ff2_\n", - "/var/folders/rv/jkb6mxgd0mbgz87qvq9356zw0000gn/T/.callflow-stderr-9nhidwq0\n" - ] - } - ], - "source": [ - "print(stdout_path)\n", - "print(stderr_path)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "p = subprocess.Popen(\n", - " [\"python3\", cwd, \"/server/main.py\", \"--config\", \"/Work/llnl/CallFlow/data/lulesh-8-runs/callflow.config.json\"],\n", - " stdout=stdout_fd,\n", - " stderr=stderr_fd,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "None\n" - ] - } - ], - "source": [ - "subprocess_result = p.poll()\n", - "print(subprocess_result)" + "%callflow --config /home/suraj/Work/llnl/CallFlow/data/lulesh-8-runs/callflow.config.json" ] }, { @@ -149,14 +74,10 @@ } ], "metadata": { - "finalized": { - "timestamp": 1597556178700, - "trusted": false - }, "kernelspec": { - "display_name": "Python 3.7.6 64-bit", + "display_name": "Python 3.7.4 64-bit (conda)", "language": "python", - "name": "python37664bit3a5637fa2c7f4443bca7a2894d18d23d" + "name": "python37464bitconda681e547a9b06483ebedfd5d049542d16" }, "language_info": { "codemirror_mode": { @@ -168,7 +89,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.7" + "version": "3.7.4" } }, "nbformat": 4, From 240d165cddbe68440b9af1eb7881bd3b8c608f23 Mon Sep 17 00:00:00 2001 From: jarusified Date: Wed, 19 Aug 2020 16:33:58 -0700 Subject: [PATCH 06/13] Dumping info file into a tmp directory --- callflow/manager.py | 149 ++++++++++++++++++++----- callflow/notebook.py | 48 ++++++-- examples/%callflow-ipython-magic.ipynb | 17 ++- server/main.py | 36 +++++- 4 files changed, 201 insertions(+), 49 deletions(-) diff --git a/callflow/manager.py b/callflow/manager.py index 82f0f6d6..293e712b 100644 --- a/callflow/manager.py +++ b/callflow/manager.py @@ -15,6 +15,8 @@ import datetime import errno import json +import base64 +import argparse # The following five types enumerate the possible return values of the # `start` function. @@ -50,6 +52,80 @@ # PID. StartTimedOut = collections.namedtuple("StartTimedOut", ("pid",)) +# Information about a running CallFlow's info. +_CALLFLOW_INFO_FIELDS = collections.OrderedDict( + ( + ("version", str), + ("start_time", int), # seconds since epoch + ("pid", int), + ("port", int), + ("config_path", str), # may be empty + ("cache_key", str), # opaque, as given by `cache_key` below + ) +) + +CallFlowLaunchInfo = collections.namedtuple( + "CallFlowLaunchInfo", _CALLFLOW_INFO_FIELDS, +) + + +def cache_key(working_directory, arguments): + """ + @working_directory (str) - Current working directory + @arguments (argparse.Namespace) - Arguments passed during launch. + + Returns a cache_key that encodes the input arguments + Used for comparing instances after launch. + """ + if not isinstance(arguments, dict): + raise TypeError( + "'arguments' should be a list of arguments, but found: %r " + "(use `shlex.split` if given a string)" % (arguments,) + ) + + datum = { + "working_directory": working_directory, + "arguments": arguments, + } + raw = base64.b64encode( + json.dumps(datum, sort_keys=True, separators=(",", ":")).encode("utf-8") + ) + # `raw` is of type `bytes`, even though it only contains ASCII + # characters; we want it to be `str` in both Python 2 and 3. + return str(raw.decode("ascii")) + + +def _info_to_string(info): + """ + Convert the callflow's launch info to json. + """ + json_value = {k: getattr(info, k) for k in _CALLFLOW_INFO_FIELDS} + return json.dumps(json_value, sort_keys=True, indent=4) + + +def _get_info_file_path(): + """Get path to info file for the current process. + As with `_get_info_dir`, the info directory will be created if it + does not exist. + """ + return os.path.join(_get_info_dir(), "pid-%d.info" % os.getpid()) + + +def write_info_file(info): + """Write CallFlowInfo to the current process's info file. + This should be called by `main` once the server is ready. When the + server shuts down, `remove_info_file` should be called. + Args: + info: A valid `CallFlowLaunchInfo` object. + Raises: + ValueError: If any field on `info` is not of the correct type. + """ + payload = "%s\n" % _info_to_string(info) + path = _get_info_file_path() + print("CallFlow config for this instance is dumped into", path) + with open(path, "w") as outfile: + outfile.write(payload) + def _exec(cmd): """ @@ -98,40 +174,46 @@ def get_launch_information(): info_dir = _get_info_dir() results = [] for filename in os.listdir(info_dir): - filepath = os.path.join(info_dir, filename) - try: - with open(filepath) as infile: - contents = infile.read() - except IOError as e: - if e.errno == errno.EACCES: - # May have been written by this module in a process whose - # `umask` includes some bits of 0o444. - continue - else: - raise - try: - info = contents - except ValueError: - # Ignore unrecognized files, logging at debug only. - print("invalid info file: %r", filepath) - else: + if filename.split("-")[0] == "pid": + filepath = os.path.join(info_dir, filename) + try: + with open(filepath) as infile: + contents = infile.read() + except IOError as e: + if e.errno == errno.EACCES: + # May have been written by this module in a process whose + # `umask` includes some bits of 0o444. + continue + else: + raise + try: + info = json.loads(contents) + except ValueError: + # Ignore unrecognized files, logging at debug only. + print("invalid info file: %r", filepath) results.append(info) return results -def start(args): +def start(args, args_string): """ Start a CallFlow (server and client) as a subprocess in the background. TODO: Improve logic to check if there is a callflow process already. - TODO: Fix the path not found error. """ + match = _find_matching_instance( + cache_key(working_directory=os.getcwd(), arguments=vars(args),), + ) + print(match) + if match: + return StartReused(info=match) + """ Launch python server. """ print("Launching Server") cwd = os.getcwd().split("CallFlow")[0] + "CallFlow/server/main.py" - server_cmd = ["python3", cwd] + args + server_cmd = ["python3", cwd] + args_string launch_cmd(server_cmd, alias="server") """ @@ -141,12 +223,25 @@ def start(args): cwd = os.getcwd().split("CallFlow")[0] + "CallFlow/app" prefix_string = ["--silent", "--prefix=" + cwd] client_cmd = ["npm", "run", "dev"] + prefix_string - print(client_cmd) launch_cmd(client_cmd, alias="client") return StartLaunched(info="Started") +def _find_matching_instance(cache_key): + """Find a running CallFlow instance compatible with the cache key. + Returns: + A `CalLFlowInfo` object, or `None` if none matches the cache key. + """ + infos = get_launch_information() + candidates = [info for info in infos if info["cache_key"] == cache_key] + print(candidates) + for candidate in sorted(candidates, key=lambda x: x.port): + print(candidate) + return candidate + return None + + def launch_cmd(cmd, timeout=datetime.timedelta(seconds=60), alias=""): """ Launch a cmd. @@ -155,6 +250,8 @@ def launch_cmd(cmd, timeout=datetime.timedelta(seconds=60), alias=""): (stdout_fd, stdout_path) = tempfile.mkstemp(prefix=stdprefix_path + "stdout-") (stderr_fd, stderr_path) = tempfile.mkstemp(prefix=stdprefix_path + "stderr-") + print(stdout_path, stderr_path) + start_time_seconds = time.time() try: p = subprocess.Popen(cmd, stdout=stdout_fd, stderr=stderr_fd,) @@ -175,12 +272,10 @@ def launch_cmd(cmd, timeout=datetime.timedelta(seconds=60), alias=""): stdout=_maybe_read_file(stdout_path), stderr=_maybe_read_file(stderr_path), ) - # for info in get_launch_information(): - # if info.pid == p.pid and info.start_time >= start_time_seconds: - info = get_launch_information() - return StartLaunched( - info={"out_path": stdout_path, "err_path": stderr_path, "pid": p.pid,} - ) + for info in get_launch_information(): + if info.pid == p.pid and info.start_time >= start_time_seconds: + info = get_launch_information() + return StartLaunched(info=info) else: return StartTimedOut(pid=p.pid) diff --git a/callflow/notebook.py b/callflow/notebook.py index 51accc00..5952ca37 100644 --- a/callflow/notebook.py +++ b/callflow/notebook.py @@ -10,6 +10,7 @@ import shlex import json import random +import argparse try: import html @@ -26,7 +27,7 @@ def _load_ipython_extension(ipython): - """Load the TensorBoard notebook extension. + """Load the CallFLow notebook extension. Intended to be called from `%load_ext callflow`. Do not invoke this directly. Args: @@ -50,13 +51,22 @@ def _start_magic(line): return start(line) +def _jupyter_args_to_argparse(args_string): + """ + Converts jupyter launch command to argparse.Namespace. + """ + parser = argparse.ArgumentParser(prefix_chars="--") + parser.add_argument("--verbose", action="store_true") + parser.add_argument("--process", action="store_true") + parser.add_argument("--config", nargs="*") + + return parser.parse_args(shlex.split(args_string)) + + def start(args_string): - """Launch and display a TensorBoard instance as if at the command line. + """Launch and display a CallFlow instance as if at the command line. Args: - args_string: Command-line arguments to TensorBoard, to be - interpreted by `shlex.split`: e.g., "--logdir ./logs --port 0". - Shell metacharacters are not supported: e.g., "--logdir 2>&1" will - point the logdir at the literal directory named "2>&1". + args_string: Command-line arguments to CallFlow. """ try: import IPython @@ -74,14 +84,31 @@ def print_or_update(message): else: handle.update(IPython.display.Pretty(message)) - parsed_args = shlex.split(args_string, comments=True, posix=True) - start_result = manager.start(parsed_args) + parsed_args = _jupyter_args_to_argparse(args_string) + start_result = manager.start( + parsed_args, shlex.split(args_string, comments=True, posix=True) + ) - print("aaaa", start_result) + print(start_result) if isinstance(start_result, manager.StartLaunched): _display_ipython( - port=1024, height=500, display_handle=handle, + port=1024, height=800, display_handle=handle, + ) + + elif isinstance(start_result, manager.StartReused): + template = ( + "Reusing CallFlow on port {port} (pid {pid}), started {delta} ago. " + "(Use '!kill {pid}' to kill it.)" + ) + message = template.format( + port=start_result.info.port, + pid=start_result.info.pid, + delta=_time_delta_from_info(start_result.info), + ) + print_or_update(message) + _display_ipython( + port=start_result.info.port, print_message=False, display_handle=None, ) @@ -116,7 +143,6 @@ def _display_ipython(port, height, display_handle): for (k, v) in replacements: shell = shell.replace(k, v) iframe = IPython.display.HTML(shell) - print(iframe) if display_handle: display_handle.update(iframe) else: diff --git a/examples/%callflow-ipython-magic.ipynb b/examples/%callflow-ipython-magic.ipynb index 413a8e77..6de734f8 100644 --- a/examples/%callflow-ipython-magic.ipynb +++ b/examples/%callflow-ipython-magic.ipynb @@ -21,17 +21,19 @@ { "cell_type": "code", "execution_count": 3, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [ { "data": { "text/html": [ "\n", - " \n", " """ - replacements = [ ("%HTML_ID%", html_escape(frame_id, quote=True)), ("%JSON_ID%", json.dumps(frame_id)), diff --git a/callflow/operations/argparser.py b/callflow/operations/argparser.py index 6d753494..1ab19a85 100644 --- a/callflow/operations/argparser.py +++ b/callflow/operations/argparser.py @@ -6,6 +6,7 @@ import os import jsonschema import argparse +import shlex import callflow @@ -29,7 +30,7 @@ class ArgParser: Config file contains the information to process datasets accrodingly. """ - def __init__(self): + def __init__(self, args_string=[]): _READ_MODES = { "config": ArgParser._read_config, @@ -38,39 +39,40 @@ def __init__(self): } # Parse the arguments passed. - args = self._create_parser() + args = self._create_parser(args_string) # Verify if only valid things are passed. # Read mode determines how arguments will be consumed by CallFlow. read_mode = self._verify_parser(args) LOGGER.debug(f"Read mode: {read_mode}") + # Check if read mode is one of the keys of _READ_MODES. assert read_mode in _READ_MODES.keys() - self.arguments = _READ_MODES[read_mode](args) + self.config = _READ_MODES[read_mode](args) # Add read_mode to arguments. - self.arguments["read_mode"] = read_mode + self.read_mode = read_mode # Add process to arguments - self.arguments["process"] = args.process + self.process = args.process # validate the json. - jsonschema.validate(instance=self.arguments, schema=schema) + jsonschema.validate(instance=self.config, schema=schema) - LOGGER.debug(f"Arguments: {self.arguments}") + LOGGER.debug(f"CallFlow instantiation configuration: {self.config}") def get_arguments(self): return self.arguments # Private methods. @staticmethod - def _create_parser(): + def _create_parser(args_string): """ Parse the input arguments. """ - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(prefix_chars="--") parser.add_argument( "--verbose", action="store_true", help="Display debug points" ) @@ -95,12 +97,13 @@ def _create_parser(): help="Set group by (e.g., grouping by 'name' column gets call graph, and grouping by 'module' produces a super graph", ) parser.add_argument("--save_path", help="Save path for the processed files") - parser.add_argument( - "--read_parameter", help="Enable parameter analysis", action="store_true" - ) + parser.add_argument("--read_parameter", help="Enable parameter analysis", action="store_true") + parser.add_argument("--gfs", help="Enter graphframes") - args = parser.parse_args() - return args + if args_string: + return parser.parse_args(shlex.split(args_string)) + else: + return parser.parse_args() @staticmethod def _verify_parser(args: argparse.Namespace): @@ -120,7 +123,7 @@ def _verify_parser(args: argparse.Namespace): process_mode: 'config' or 'directory' or 'gfs' Process mode with which CallFlow will process the data. """ - if not args.config and not args.data_dir and args.gfs: + if not args.config and not args.data_dir and not args.gfs: s = "Please provide a config file (or) directory (or) pass in the graphframes. To see options, use --help" raise Exception(s) @@ -280,7 +283,7 @@ def _scheme_dataset_map_caliper_json(data_path: str): scheme = {} scheme["runs"] = [] scheme["paths"] = {} - scheme["format"] = {} + scheme["profile_format"] = {} list_json_paths = [ f.path for f in os.scandir(data_path) @@ -293,7 +296,7 @@ def _scheme_dataset_map_caliper_json(data_path: str): filename = name.split(".")[0] scheme["runs"].append(filename) scheme["paths"][filename] = subfolder_path - scheme["format"][filename] = "caliper-json" + scheme["profile_format"][filename] = "caliper_json" return scheme diff --git a/server/main.py b/server/main.py index 5075e680..3f1b67d8 100644 --- a/server/main.py +++ b/server/main.py @@ -8,10 +8,13 @@ import json from networkx.readwrite import json_graph import time +import os # ------------------------------------------------------------------------------ # CallFlow imports. import callflow +from callflow.operations import ConfigFileReader +from callflow import manager from callflow.operations import ArgParser LOGGER = callflow.get_logger(__name__) @@ -29,15 +32,20 @@ class CallFlowServer: """ def __init__(self): - self.args = ArgParser().get_arguments() + self.args = ArgParser() + + # Set cache key to store the current instance's arguments. + self.cache_key = manager.cache_key( + working_directory=os.getcwd(), arguments=vars(self.args) + ) self.debug = True self.production = False - self.process = self.args["process"] + self.process = self.args.process - ndatasets = len(self.args["properties"]["runs"]) + ndatasets = len(self.args.config["properties"]["runs"]) assert ndatasets > 0 - self.callflow = callflow.CallFlow(config=self.args, ensemble=ndatasets > 1) + self.callflow = callflow.CallFlow(config=self.args.config, ensemble=ndatasets > 1) if self.process: self.callflow.process() @@ -61,23 +69,12 @@ def _create_server(self): # Socket request handlers self._request_handler_general() - if len(self.args["properties"]["runs"]) == 1: + if len(self.args.config["properties"]["runs"]) == 1: self._request_handler_single() else: self._request_handler_single() self._request_handler_ensemble() - info = manager.CallFlowLaunchInfo( - version=__version__, - start_time=int(time.time()), - port=_CALLFLOW_SERVER_PORT, - pid=os.getpid(), - config_path=self.config.json, - cache_key=self.cache_key, - ) - - manager.write_info_file(info) - # Start the server. if self.production: sockets.run( From 9a7a44c96c1b3a627e10bbaeca15b8191e4eed60 Mon Sep 17 00:00:00 2001 From: jarusified Date: Fri, 25 Sep 2020 11:38:19 -0700 Subject: [PATCH 09/13] Fix the reuse of existing processes --- callflow/manager.py | 2 +- callflow/notebook.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/callflow/manager.py b/callflow/manager.py index 36db6cbe..ae25eddf 100644 --- a/callflow/manager.py +++ b/callflow/manager.py @@ -70,7 +70,7 @@ ) _CALLFLOW_DEFAULT_SERVER_PORT = 5000 -_CALLFLOW_DEFAULT_CLIENT_PORT = 8000 +_CALLFLOW_DEFAULT_CLIENT_PORT = 1024 def cache_key(working_directory, arguments): """ diff --git a/callflow/notebook.py b/callflow/notebook.py index d8a2e0e9..864559c7 100644 --- a/callflow/notebook.py +++ b/callflow/notebook.py @@ -75,8 +75,6 @@ def print_or_update(message): else: handle.update(IPython.display.Pretty(message)) - args = ArgParser(args_string) - args = ArgParser(args_string) start_result = manager.start(args, shlex.split(args_string, comments=True, posix=True)) @@ -98,7 +96,7 @@ def print_or_update(message): ) print_or_update(message) _display_ipython( - port=start_result.info["server_port"], display_handle=None, height=800 + port=start_result.info["client_port"], display_handle=None, height=800 ) def _time_delta_from_info(info): From 27f28dcd98e5e7abd97f73dabff6d4759e88fd24 Mon Sep 17 00:00:00 2001 From: jarusified Date: Fri, 25 Sep 2020 11:44:38 -0700 Subject: [PATCH 10/13] Add the notebook --- examples/%callflow-ipython-magic.ipynb | 48 ++++++++++---------------- 1 file changed, 18 insertions(+), 30 deletions(-) diff --git a/examples/%callflow-ipython-magic.ipynb b/examples/%callflow-ipython-magic.ipynb index 3466c2b2..8180b34c 100644 --- a/examples/%callflow-ipython-magic.ipynb +++ b/examples/%callflow-ipython-magic.ipynb @@ -21,17 +21,28 @@ { "cell_type": "code", "execution_count": 3, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [ + { + "data": { + "text/plain": [ + "Reusing CallFlow's server is on port 5000 and client is on 1024 (pid 7827), started 0:19:01 ago. (Use '!kill 7827' to kill it.)" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, { "data": { "text/html": [ "\n", - " \n", "