diff --git a/README.md b/README.md index 2ee18e46..5cda690e 100644 --- a/README.md +++ b/README.md @@ -12,10 +12,10 @@ CallFlow is structured as three components: #### Installing CallFlow -The `callflow` (python package) requires [python](https://realpython.com/installing-python/) (>= 3.6) and [pip](https://pip.pypa.io/en/stable/news/) (>= 20.1.1). Other dependencies are checked/installed during the installation of `callflow` using `pip`. +The `callflow` (python package) requires [python](https://realpython.com/installing-python/) (>= 3.6) and [pip](https://pip.pypa.io/en/stable/news/) (>= 20.1.1). Other dependencies are checked/installed during the installation of `callflow` using `setup.py`. ``` -pip install . +python3 setup.py install ``` #### Installing Visualization Client diff --git a/callflow/__init__.py b/callflow/__init__.py index 3cfd2007..e6cb9bc3 100644 --- a/callflow/__init__.py +++ b/callflow/__init__.py @@ -3,9 +3,6 @@ # # SPDX-License-Identifier: MIT -__version_info__ = ("1", "1", "0") -__version__ = ".".join(__version_info__) - # callflow.__init__.py from .logger import init_logger from logging import getLogger as get_logger @@ -15,7 +12,6 @@ from .datastructures.ensemblegraph import EnsembleGraph from .callflow import CallFlow -from .notebook import _load_ipython_extension # CallFlow's public API. __all__ = [ @@ -25,14 +21,4 @@ "SuperGraph", "EnsembleGraph", "CallFlow", - "notebook", ] - - -def load_ipython_extension(ipython): - """IPython API entry point. - Only intended to be called by the IPython runtime. - See: - https://ipython.readthedocs.io/en/stable/config/extensions/index.html - """ - _load_ipython_extension(ipython) diff --git a/callflow/operations/argparser.py b/callflow/argparser.py similarity index 100% rename from callflow/operations/argparser.py rename to callflow/argparser.py diff --git a/callflow/operations/__init__.py b/callflow/operations/__init__.py index cd46aa2c..a4faee18 100644 --- a/callflow/operations/__init__.py +++ b/callflow/operations/__init__.py @@ -7,6 +7,5 @@ from .group import Group from .filter import Filter from .read_config import ConfigFileReader -from .argparser import ArgParser -__all__ = ["Process", "Group", "Filter", "ConfigFileReader", "ArgParser"] +__all__ = ["Process", "Group", "Filter", "ConfigFileReader"] diff --git a/callflow/version.py b/callflow/version.py new file mode 100644 index 00000000..5fc6ad3a --- /dev/null +++ b/callflow/version.py @@ -0,0 +1,7 @@ +# Copyright 2017-2020 Lawrence Livermore National Security, LLC and other +# CallFlow Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: MIT + +__version_info__ = ("1", "1", "0") +__version__ = ".".join(__version_info__) diff --git a/docs/getting_started.rst b/docs/getting_started.rst index bd3252e3..627ef099 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -41,11 +41,11 @@ You can get CallFlow from its `GitHub `_ using Install callflow python package ------------------------------- -To install callflow python package, run the following command using pip. +To install callflow python package, run the following command using `setup.py`. .. code-block:: console - $ pip install . + $ python3 setup.py install To install in the dev mode, diff --git a/requirements.txt b/requirements.txt index b956a2e3..6f57c063 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ -pandas -numpy -flask -flask_socketio +colorlog==4.1.0 +statsmodels==0.10.1 +hatchet==1.0.1 +matplotlib==3.1.1 +jsonschema==3.0.2 networkx==2.2 -colorlog -statsmodels -matplotlib -sklearn -jsonschema -eventlet -hatchet +scipy==1.3.1 +numpy==1.17.2 +pandas==0.23.0 +Flask_SocketIO==4.2.1 +ipython==7.18.1 +scikit_learn==0.23.2 diff --git a/server/__init__.py b/server/__init__.py new file mode 100644 index 00000000..6b32491d --- /dev/null +++ b/server/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2017-2020 Lawrence Livermore National Security, LLC and other +# CallFlow Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: MIT + +from .notebook_server import _load_ipython_extension + + +def load_ipython_extension(ipython): + """IPython API entry point. + Only intended to be called by the IPython runtime. + See: + https://ipython.readthedocs.io/en/stable/config/extensions/index.html + """ + _load_ipython_extension(ipython) diff --git a/server/main.py b/server/callflow_server.py similarity index 99% rename from server/main.py rename to server/callflow_server.py index 87bfb88b..39f45627 100644 --- a/server/main.py +++ b/server/callflow_server.py @@ -3,17 +3,19 @@ # # SPDX-License-Identifier: MIT # Library imports -from flask import Flask -from flask_socketio import SocketIO, emit +import os import json from networkx.readwrite import json_graph -import os + +from flask import Flask +from flask_socketio import SocketIO, emit # ------------------------------------------------------------------------------ # CallFlow imports. import callflow -from callflow import manager -from callflow.operations import ArgParser +from callflow.argparser import ArgParser +from . import manager + LOGGER = callflow.get_logger(__name__) _CALLFLOW_SERVER_PORT = 5000 @@ -434,8 +436,12 @@ def split_mpi_rank(data): emit("split_mpi_distribution", result, json=True) -if __name__ == "__main__": +def main(): # if verbose, level = 1 # else, level = 2 callflow.init_logger(level=2) CallFlowServer() + + +if __name__ == "__main__": + main() diff --git a/callflow/manager.py b/server/manager.py similarity index 95% rename from callflow/manager.py rename to server/manager.py index 561c3cfe..b784ee32 100644 --- a/callflow/manager.py +++ b/server/manager.py @@ -16,7 +16,16 @@ import errno import json import base64 -from callflow import __version__ + +from codecs import open + +version = {} +vfile = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "callflow", "version.py" +) +with open(vfile) as fp: + exec(fp.read(), version) +__version__ = version["__version__"] # The following five types enumerate the possible return values of the # `start` function. @@ -231,16 +240,19 @@ def start(args, args_string): Launch python server. """ print("Launching Server") + """ cwd = os.getcwd().split("CallFlow")[0] + "CallFlow/server/main.py" server_cmd = ["python3", cwd] + args_string + """ + server_cmd = ["callflow_server"] + args_string launch_cmd(server_cmd, alias="server") """ Launch callflow app server. """ print("Launching client") - cwd = os.getcwd().split("CallFlow")[0] + "CallFlow/app" - prefix_string = ["--silent", "--prefix=" + cwd] + app_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "app") + prefix_string = ["--silent", "--prefix=" + app_path] client_cmd = ["npm", "run", "dev"] + prefix_string launch_cmd(client_cmd, alias="client") diff --git a/callflow/notebook.py b/server/notebook_server.py similarity index 98% rename from callflow/notebook.py rename to server/notebook_server.py index aec0bc88..538ad178 100644 --- a/callflow/notebook.py +++ b/server/notebook_server.py @@ -7,12 +7,11 @@ # through the IPython interface. # The code can be found at https://github.com/tensorflow/tensorboard/blob/master/tensorboard/notebook.py -import shlex import json -import random import time import datetime -from callflow.operations import ArgParser +import shlex +import random try: import html @@ -25,7 +24,8 @@ html_escape = cgi.escape del cgi -from callflow import manager +from callflow.argparser import ArgParser +from . import manager def _load_ipython_extension(ipython): diff --git a/setup.py b/setup.py index 0115c2b0..258b6444 100644 --- a/setup.py +++ b/setup.py @@ -1,18 +1,91 @@ # Copyright 2017-2020 Lawrence Livermore National Security, LLC and other -# Hatchet Project Developers. See the top-level LICENSE file for details. +# CallFlow Project Developers. See the top-level LICENSE file for details. # # SPDX-License-Identifier: MIT +import os from setuptools import setup, find_packages -from callflow import __version__ +# ------------------------------------------------------------------------------ +# get the version safely! +from codecs import open + +version = {} +vfile = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "callflow", "version.py" +) +with open(vfile) as fp: + exec(fp.read(), version) +version = version["__version__"] + + +# ------------------------------------------------------------------------------ +# Only allow folders in https://github.com/LLNL/CallFlow/tree/develop/data to be added. +# For now we are hard-coding this. +# TODO: Find a more automated solution. +_GITHUB_DATA_FOLDERS = [ + "caliper-cali", + "caliper-lulesh-json", + "hpctoolkit-cpi-database", +] + +# Only allow jupyter notebooks in https://github.com/LLNL/CallFlow/tree/develop/example to be added. +_GITHUB_EXAMPLE_FILES = [ + "%callflow-ipython-magic.ipynb", + "CallFlow-python-interface-demo.ipynb", +] + + +# gather the data to be copied +def list_files(directory, whitelist_files=[], whitelist_folders=[]): + paths = [] + if len(whitelist_folders) > 0: + for item in os.listdir(directory): + if item in whitelist_folders: + for (path, directories, filenames) in os.walk( + os.path.join(directory, item) + ): + if ".callflow" not in path.split("/"): + paths.append((path, [os.path.join(path, f) for f in filenames])) + + if len(whitelist_files) > 0: + for (path, directories, filenames) in os.walk(directory): + paths.append( + ( + path, + [os.path.join(path, f) for f in filenames if f in whitelist_files], + ) + ) + return paths + + +data_files = list_files("data", whitelist_folders=_GITHUB_DATA_FOLDERS) +example_files = list_files("examples", whitelist_files=_GITHUB_EXAMPLE_FILES) + + +deps = [ + "flask_socketio", + "ipython", + "colorlog", + "jsonschema", + "numpy", + "scipy", + "pandas", + "scikit_learn", + "statsmodels", + "hatchet", + "networkx", + "matplotlib", +] +# ------------------------------------------------------------------------------ +# now set up setup( name="CallFlow", - version=__version__, + version=version, license="MIT", description="", url="https://github.com/LLNL/CallFlow", - author="Suraj Kesavan, Huu Tan Nguyen", + author="Suraj Kesavan", author_email="spkesavan@ucdavis.edu", classifiers=[ "Development Status :: 3 - Alpha", @@ -20,15 +93,9 @@ ], keywords="", packages=find_packages(), - install_requires=[ - "numpy", - "pandas", - "tables", - "flask", - "flask_socketio", - "sklearn", - "statsmodels", - "networkx == 2.2", - "hatchet", - ], + data_files=data_files + example_files, + entry_points={"console_scripts": ["callflow_server = server.callflow_server:main"]}, + install_requires=deps, ) + +# ------------------------------------------------------------------------------