diff --git a/Makefile b/Makefile index 04d7b81c..2b0177ef 100644 --- a/Makefile +++ b/Makefile @@ -44,8 +44,8 @@ lint-cpp: echo "C++ linting disabled for now" lint-docs: - python -m mdformat --check docs/wiki/ README.md examples/README.md - python -m codespell_lib docs/wiki/ README.md examples/README.md + python -m mdformat --check docs/wiki/ README.md examples/ + python -m codespell_lib docs/wiki/ README.md examples/ --skip "*.cpp,*.h" # lint: lint-py lint-cpp ## run lints lint: lint-py lint-docs ## run lints @@ -62,8 +62,8 @@ fix-cpp: echo "C++ autoformatting disabled for now" fix-docs: - python -m mdformat docs/wiki/ README.md examples/README.md - python -m codespell_lib --write docs/wiki/ README.md examples/README.md + python -m mdformat docs/wiki/ README.md examples/ + python -m codespell_lib --write docs/wiki/ README.md examples/ --skip "*.cpp,*.h" fix: fix-py fix-cpp fix-docs ## run autofixers diff --git a/conda/dev-environment-unix.yml b/conda/dev-environment-unix.yml index 5faf83b4..b520d640 100644 --- a/conda/dev-environment-unix.yml +++ b/conda/dev-environment-unix.yml @@ -5,6 +5,7 @@ channels: dependencies: - bison - brotli + - build - bump2version>=1 - cmake - codespell @@ -12,10 +13,11 @@ dependencies: - cyrus-sasl - exprtk - flex + - graphviz - python-graphviz - gtest - - httpx - - isort + - httpx>=0.20,<1 + - isort>=5,<6 - libarrow=15 - librdkafka - libboost-headers @@ -24,16 +26,18 @@ dependencies: - mdformat - ninja - numpy + - pillow - psutil - pyarrow=15 - pandas - pillow - polars + - psutil - pytz - pytest + - pytest-asyncio - pytest-cov - pytest-sugar - - pytest-asyncio - python<3.12 - python-rapidjson - rapidjson @@ -44,6 +48,7 @@ dependencies: - slack-sdk - sqlalchemy - tar + - threadpoolctl - tornado - twine - unzip diff --git a/csp/tests/test_examples.py b/csp/tests/test_examples.py index f0694c6d..2d2c03f6 100644 --- a/csp/tests/test_examples.py +++ b/csp/tests/test_examples.py @@ -17,10 +17,11 @@ def _get_module(folder, filename): return None -def _get_modules_to_test(folder): +def _get_modules_to_test(*folders): + folder = ".".join(folders) if len(folders) > 0 else folders[0] return [ (file, _get_module(folder, file)) - for file in os.listdir(os.path.join(EXAMPLES_ROOT, folder)) + for file in os.listdir(os.path.join(EXAMPLES_ROOT, *folders)) if file.endswith(".py") ] @@ -31,12 +32,37 @@ def _no_examples_folder_or_running_sdist_tests(): @pytest.mark.skipif(_no_examples_folder_or_running_sdist_tests(), reason="no examples present or manually skipping") class TestExamples: - @pytest.mark.parametrize("filename,module", _get_modules_to_test("1_basics")) + @pytest.mark.parametrize("filename,module", _get_modules_to_test("01_basics")) def test_1_basics(self, filename, module): assert module.main module.main() - @pytest.mark.parametrize("filename,module", _get_modules_to_test("2_intermediate")) + @pytest.mark.parametrize("filename,module", _get_modules_to_test("02_intermediate")) def test_2_intermediate(self, filename, module): assert module.main module.main() + + @pytest.mark.parametrize("filename,module", _get_modules_to_test("03_using_adapters", "parquet")) + def test_3_adapters_parquet(self, filename, module): + assert module.main + module.main() + + @pytest.mark.parametrize("filename,module", _get_modules_to_test("04_writing_adapters")) + def test_4_writing_adapters(self, filename, module): + assert module.main + module.main() + + @pytest.mark.parametrize("filename,module", _get_modules_to_test("06_advanced")) + def test_6_advanced(self, filename, module): + assert module.main + module.main() + + @pytest.mark.parametrize("filename,module", _get_modules_to_test("98_just_for_fun")) + def test_98_just_for_fun(self, filename, module): + assert module.main + module.main() + + @pytest.mark.parametrize("filename,module", _get_modules_to_test("99_developer_tools")) + def test_99_developer_tools(self, filename, module): + assert module.main + module.main() diff --git a/examples/01_basics/README.md b/examples/01_basics/README.md new file mode 100644 index 00000000..72ebc94f --- /dev/null +++ b/examples/01_basics/README.md @@ -0,0 +1,6 @@ +# Basics + +- [Simplest Possible Graph](./e1_basic.py) +- [Ticking Graphs](./e2_ticking.py) +- [Complete Example (Trading)](./e3_trade_pnl.py) +- [Visualizing a Graph](./e4_show_graph.py) diff --git a/examples/01_basics/e1_basic.py b/examples/01_basics/e1_basic.py new file mode 100644 index 00000000..e318f3fb --- /dev/null +++ b/examples/01_basics/e1_basic.py @@ -0,0 +1,29 @@ +from datetime import datetime + +import csp +from csp import ts + + +@csp.node +def add(x: ts[int], y: ts[int]) -> ts[int]: + return x + y + + +@csp.graph +def my_graph(): + x = csp.const(1) + y = csp.const(2) + + sum = add(x, y) + + csp.print("x", x) + csp.print("y", y) + csp.print("sum", sum) + + +def main(): + csp.run(my_graph, starttime=datetime.now()) + + +if __name__ == "__main__": + main() diff --git a/examples/01_basics/e2_ticking.py b/examples/01_basics/e2_ticking.py new file mode 100644 index 00000000..5377027f --- /dev/null +++ b/examples/01_basics/e2_ticking.py @@ -0,0 +1,49 @@ +from datetime import datetime, timedelta + +import csp +from csp import ts + + +@csp.node +def add(x: ts[int], y: ts[int]) -> ts[int]: + return x + y + + +@csp.node +def accum(val: ts[int]) -> ts[int]: + with csp.state(): + s_sum = 0 + if csp.ticked(val): + s_sum += val + return val + + +@csp.graph +def my_graph(): + st = datetime(2020, 1, 1) + + # Dummy x values + x = csp.curve(int, [(st + timedelta(1), 1), (st + timedelta(2), 2), (st + timedelta(3), 3)]) + + # Dummy y values + y = csp.curve(int, [(st + timedelta(1), -1), (st + timedelta(3), -1), (st + timedelta(4), -1)]) + + # Add the time series + sum = add(x, y) + + # Accumulate the result + acc = accum(sum) + + csp.print("x", x) + csp.print("y", y) + csp.print("sum", sum) + csp.print("accum", acc) + + +def main(): + start = datetime(2020, 1, 1) + csp.run(my_graph, starttime=start) + + +if __name__ == "__main__": + main() diff --git a/examples/1_basics/e_02_show_graph.py b/examples/01_basics/e3_show_graph.py similarity index 100% rename from examples/1_basics/e_02_show_graph.py rename to examples/01_basics/e3_show_graph.py diff --git a/examples/1_basics/e_01_trade_pnl.py b/examples/01_basics/e4_trade_pnl.py similarity index 100% rename from examples/1_basics/e_01_trade_pnl.py rename to examples/01_basics/e4_trade_pnl.py diff --git a/examples/02_intermediate/README.md b/examples/02_intermediate/README.md new file mode 100644 index 00000000..0dc03c12 --- /dev/null +++ b/examples/02_intermediate/README.md @@ -0,0 +1,6 @@ +# Intermediate + +- [Graph Loops (`csp.feedback`)](./e1_feedback.py) +- [Statistics Nodes](./e2_stats.py) +- [Statistics Nodes with Numpy](./e3_numpy_stats.py) +- [Expression Nodes with `exprtk`](./e4_exprtk.py) diff --git a/examples/2_intermediate/e_13_feedback.py b/examples/02_intermediate/e1_feedback.py similarity index 98% rename from examples/2_intermediate/e_13_feedback.py rename to examples/02_intermediate/e1_feedback.py index d1ad9a61..3491bfb5 100644 --- a/examples/2_intermediate/e_13_feedback.py +++ b/examples/02_intermediate/e1_feedback.py @@ -80,7 +80,7 @@ def main(): if show_graph: csp.showgraph.show_graph(my_graph) else: - csp.run(my_graph, starttime=datetime.utcnow(), endtime=timedelta(seconds=60), realtime=True) + csp.run(my_graph, starttime=datetime.utcnow(), endtime=timedelta(seconds=5), realtime=False) if __name__ == "__main__": diff --git a/examples/2_intermediate/e_20_stats.py b/examples/02_intermediate/e2_stats.py similarity index 100% rename from examples/2_intermediate/e_20_stats.py rename to examples/02_intermediate/e2_stats.py diff --git a/examples/2_intermediate/e_21_numpy_stats.py b/examples/02_intermediate/e3_numpy_stats.py similarity index 98% rename from examples/2_intermediate/e_21_numpy_stats.py rename to examples/02_intermediate/e3_numpy_stats.py index 57ba659f..0debaf21 100644 --- a/examples/2_intermediate/e_21_numpy_stats.py +++ b/examples/02_intermediate/e3_numpy_stats.py @@ -56,7 +56,7 @@ def numpy_stats_graph(): def main(): - results = csp.run(numpy_stats_graph, starttime=st, endtime=st + timedelta(minutes=10)) + results = csp.run(numpy_stats_graph, starttime=st, endtime=st + timedelta(minutes=10), realtime=False) print("Price Averages\n") for i in range(10): diff --git a/examples/2_intermediate/e_19_exprtk_example.py b/examples/02_intermediate/e4_exprtk.py similarity index 100% rename from examples/2_intermediate/e_19_exprtk_example.py rename to examples/02_intermediate/e4_exprtk.py diff --git a/examples/03_using_adapters/kafka/README.md b/examples/03_using_adapters/kafka/README.md new file mode 100644 index 00000000..907ff323 --- /dev/null +++ b/examples/03_using_adapters/kafka/README.md @@ -0,0 +1,3 @@ +# Kafka Adapter + +- [Kafka Example](./e1_kafka.py) diff --git a/examples/3_using_adapters/e_08_kafka.py b/examples/03_using_adapters/kafka/e1_kafka.py similarity index 100% rename from examples/3_using_adapters/e_08_kafka.py rename to examples/03_using_adapters/kafka/e1_kafka.py diff --git a/examples/03_using_adapters/parquet/README.md b/examples/03_using_adapters/parquet/README.md new file mode 100644 index 00000000..f50da3b3 --- /dev/null +++ b/examples/03_using_adapters/parquet/README.md @@ -0,0 +1,3 @@ +# Parquet Adapter + +- [Parquet Writer](./e1_parquet_writer.py) diff --git a/examples/3_using_adapters/e_06_parquet_writer_example.py b/examples/03_using_adapters/parquet/e1_parquet_writer.py similarity index 98% rename from examples/3_using_adapters/e_06_parquet_writer_example.py rename to examples/03_using_adapters/parquet/e1_parquet_writer.py index 386449ce..21c5ddc7 100644 --- a/examples/3_using_adapters/e_06_parquet_writer_example.py +++ b/examples/03_using_adapters/parquet/e1_parquet_writer.py @@ -48,12 +48,12 @@ def my_graph(struct_file_name: str, series_file_name: str): write_series(series_file_name) -if __name__ == "__main__": +def main(): with tempfile.NamedTemporaryFile(suffix=".parquet") as struct_file: struct_file.file.close() with tempfile.NamedTemporaryFile(suffix=".parquet") as series_file: series_file.file.close() - g = csp.run( + csp.run( my_graph, struct_file.name, series_file.name, @@ -72,3 +72,7 @@ def my_graph(struct_file_name: str, series_file_name: str): print(f"Struct data:\n{struct_df}") series_df = pandas.read_parquet(series_file.name) print(f"Series data:\n{series_df}") + + +if __name__ == "__main__": + main() diff --git a/examples/03_using_adapters/slack/README.me b/examples/03_using_adapters/slack/README.me new file mode 100644 index 00000000..be657ba7 --- /dev/null +++ b/examples/03_using_adapters/slack/README.me @@ -0,0 +1 @@ +# Slack Adapter diff --git a/examples/03_using_adapters/websocket/README.md b/examples/03_using_adapters/websocket/README.md new file mode 100644 index 00000000..3cec2eff --- /dev/null +++ b/examples/03_using_adapters/websocket/README.md @@ -0,0 +1,3 @@ +# Websocket Adapter + +- [Websocket Output](./e1_websocket_output.py) diff --git a/examples/3_using_adapters/e_11_websocket_output.py b/examples/03_using_adapters/websocket/e1_websocket_output.py similarity index 89% rename from examples/3_using_adapters/e_11_websocket_output.py rename to examples/03_using_adapters/websocket/e1_websocket_output.py index 7ef9b6d0..917ab727 100755 --- a/examples/3_using_adapters/e_11_websocket_output.py +++ b/examples/03_using_adapters/websocket/e1_websocket_output.py @@ -29,7 +29,7 @@ def times(timer: ts[bool]) -> ts[datetime]: @csp.graph -def main(port: int, num_keys: int): +def my_graph(port: int, num_keys: int): snap = csp.timer(timedelta(seconds=0.25)) angle = csp.count(snap) @@ -57,7 +57,10 @@ def main(port: int, num_keys: int): port = 7677 num_keys = 10 -csp.run(main, port, num_keys, starttime=datetime.utcnow(), endtime=timedelta(seconds=360), realtime=True) + +def main(): + csp.run(my_graph, port, num_keys, starttime=datetime.utcnow(), endtime=timedelta(seconds=360), realtime=True) + """ Sample html to view the data. Note to put your machine name on the websocket line below @@ -84,3 +87,6 @@ def main(port: int, num_keys: int): """ + +if __name__ == "__main__": + main() diff --git a/examples/04_writing_adapters/README.md b/examples/04_writing_adapters/README.md new file mode 100644 index 00000000..04192af2 --- /dev/null +++ b/examples/04_writing_adapters/README.md @@ -0,0 +1,9 @@ +# Writing Adapters + +- [Generic Push Adapter](./e1_generic_push_adapter.py) +- [Pull Input Adapter](./e2_pullinput.py) +- [Pull Input Adapter with Adapter Manager](./e3_adaptermanager_pullinput.py) +- [Push Input Adapter](./e4_pushinput.py) +- [Push Input Adapter with Adapter Manager](./e5_adaptermanager_pushinput.py) +- [Output Adapter](./e6_outputadapter.py) +- [Complete Input/Output Adapter with Adapter Manager](./e7_adaptermanager_inputoutput.py) diff --git a/examples/4_writing_adapters/e_14_generic_push_adapter.py b/examples/04_writing_adapters/e1_generic_push_adapter.py similarity index 88% rename from examples/4_writing_adapters/e_14_generic_push_adapter.py rename to examples/04_writing_adapters/e1_generic_push_adapter.py index 687dd2be..6ab3dc53 100644 --- a/examples/4_writing_adapters/e_14_generic_push_adapter.py +++ b/examples/04_writing_adapters/e1_generic_push_adapter.py @@ -32,7 +32,7 @@ def _run(self): counter = 0 # Optionally, we can wait for the adapter to start before proceeding # Alternatively we can start pushing data, but push_tick may fail and return False if - # the csp engine isnt ready yet + # the csp engine isn't ready yet self._adapter.wait_for_start() while self._active and not self._adapter.stopped(): @@ -52,4 +52,9 @@ def my_graph(): csp.print("data", adapter.out()) -csp.run(my_graph, realtime=True, starttime=datetime.utcnow(), endtime=timedelta(seconds=10)) +def main(): + csp.run(my_graph, realtime=True, starttime=datetime.utcnow(), endtime=timedelta(seconds=2)) + + +if __name__ == "__main__": + main() diff --git a/examples/4_writing_adapters/e_14_user_adapters_01_pullinput.py b/examples/04_writing_adapters/e2_pullinput.py similarity index 80% rename from examples/4_writing_adapters/e_14_user_adapters_01_pullinput.py rename to examples/04_writing_adapters/e2_pullinput.py index 3e35239d..7fb274ec 100644 --- a/examples/4_writing_adapters/e_14_user_adapters_01_pullinput.py +++ b/examples/04_writing_adapters/e2_pullinput.py @@ -1,9 +1,9 @@ """ -PullInputAdapter is the simplest form of an input adapter for historical data. One instance is created -to provide data on a single timeseries. There are use cases for this construct, though they are limited. -This is useful when feeding a single source of historical data into a single timeseries. In most cases however, -you will likely have a single source that is processed and used to provide data to multiple inputs. For that construct -see e_14_user_adapters_02_adaptermanager_siminput.py +PullInputAdapter is the simplest form of an input adapter for historical data. One instance is created +to provide data on a single timeseries. There are use cases for this construct, though they are limited. +This is useful when feeding a single source of historical data into a single timeseries. In most cases however, +you will likely have a single source that is processed and used to provide data to multiple inputs. For that construct +see e3_adaptermanager_pullinput.py """ from datetime import datetime, timedelta @@ -45,7 +45,7 @@ def next(self): return None -# MyPullAdapter is the graph-building time construct. This is simply a representation of what the +# MyPullAdapter is the graph-building time construct. This is simply a representation of what the # input adapter is and how to create it, including the Impl to use and arguments to pass into it upon construction MyPullAdapter = py_pull_adapter_def("MyPullAdapter", MyPullAdapterImpl, ts[int], interval=timedelta, num_ticks=int) @@ -58,4 +58,9 @@ def my_graph(): print("End of graph building") -csp.run(my_graph, starttime=datetime(2020, 12, 28)) +def main(): + csp.run(my_graph, starttime=datetime(2020, 12, 28)) + + +if __name__ == "__main__": + main() diff --git a/examples/4_writing_adapters/e_14_user_adapters_02_adaptermanager_siminput.py b/examples/04_writing_adapters/e3_adaptermanager_pullinput.py similarity index 88% rename from examples/4_writing_adapters/e_14_user_adapters_02_adaptermanager_siminput.py rename to examples/04_writing_adapters/e3_adaptermanager_pullinput.py index f52e7de7..c5d2e48b 100644 --- a/examples/4_writing_adapters/e_14_user_adapters_02_adaptermanager_siminput.py +++ b/examples/04_writing_adapters/e3_adaptermanager_pullinput.py @@ -1,7 +1,7 @@ """ -This example introduces the concept of an AdapterManager. AdapterManagers are constructs that are used -when you have a shared input or output resources ( ie single CSV / Parquet file, some pub/sub session, etc ) -that you want to connect to once, but provide data to / from many input / output adapters ( aka time series ) +This example introduces the concept of an AdapterManager. AdapterManagers are constructs that are used +when you have a shared input or output resources (ie single CSV / Parquet file, some pub/sub session, etc) +that you want to connect to once, but provide data to/from many input/output adapters (aka time series) """ import random @@ -18,7 +18,7 @@ class MyData(csp.Struct): value: int -# This object represents our AdapterManager at graph time. It describes the manager's properties +# This object represents our AdapterManager at graph time. It describes the manager's properties # and will be used to create the actual impl when its time to build the engine class MyAdapterManager: def __init__(self, interval: timedelta): @@ -78,9 +78,9 @@ def register_input_adapter(self, symbol, adapter): def process_next_sim_timeslice(self, now): """After start is called, process_next_sim_timeslice will be called repeatedly - to process the next available timestamp from the data source. Every call to this method + to process the next available timestamp from the data source. Every call to this method should process all "rows" for the given timestamp. - For every tick that aplies to an input, we push the tick into the adapter. + For every tick that applies to an input, we push the tick into the adapter. This method should return the datetime of the next even in the data, or None if there is no data left. First call will be for "starttime" """ @@ -104,7 +104,7 @@ def process_next_sim_timeslice(self, now): # The Impl object is created at runtime when the graph is converted into the runtime engine -# it does not exist at graph building time. a managed sim adapter impl will get the +# it does not exist at graph building time. a managed sim adapter impl will get the # adapter manager runtime impl as its first argument class MyManagedSimAdapterImpl(ManagedSimInputAdapter): def __init__(self, manager_impl, symbol): @@ -146,4 +146,9 @@ def my_graph(): print("End of graph building") -csp.run(my_graph, starttime=datetime(2020, 12, 28), endtime=timedelta(seconds=10)) +def main(): + csp.run(my_graph, starttime=datetime(2020, 12, 28), endtime=timedelta(seconds=10)) + + +if __name__ == "__main__": + main() diff --git a/examples/4_writing_adapters/e_14_user_adapters_03_pushinput.py b/examples/04_writing_adapters/e4_pushinput.py similarity index 79% rename from examples/4_writing_adapters/e_14_user_adapters_03_pushinput.py rename to examples/04_writing_adapters/e4_pushinput.py index 838da4e3..88589f0c 100644 --- a/examples/4_writing_adapters/e_14_user_adapters_03_pushinput.py +++ b/examples/04_writing_adapters/e4_pushinput.py @@ -1,9 +1,9 @@ """ -PushInputAdapter is the simplest form of an input adapter for real-time data. One instance is created -to provide data on a single timeseries. There are use cases for this construct, though they are limited. -This is useful when feeding a single source of data into a single timeseries. In most cases however, -you will likely have a single source that is processed and used to provide data to multiple inputs. For that construct -see e_14_user_adapters_04_adaptermanager_pushinput.py +PushInputAdapter is the simplest form of an input adapter for real-time data. One instance is created +to provide data on a single timeseries. There are use cases for this construct, though they are limited. +This is useful when feeding a single source of data into a single timeseries. In most cases however, +you will likely have a single source that is processed and used to provide data to multiple inputs. For that construct +see e5_adaptermanager_pushinput.py """ import threading @@ -27,7 +27,7 @@ def __init__(self, interval): def start(self, starttime, endtime): """start will get called at the start of the engine, at which point the push - input adapter should start its thread that will push the data onto the adapter. Note + input adapter should start its thread that will push the data onto the adapter. Note that push adapters will ALWAYS have a separate thread driving ticks into the csp engine thread """ print("MyPushAdapterImpl::start") @@ -52,14 +52,14 @@ def _run(self): time.sleep(self._interval.total_seconds()) -# MyPushAdapter is the graph-building time construct. This is simply a representation of what the +# MyPushAdapter is the graph-building time construct. This is simply a representation of what the # input adapter is and how to create it, including the Impl to create and arguments to pass into it MyPushAdapter = py_push_adapter_def("MyPushAdapter", MyPushAdapterImpl, ts[int], interval=timedelta) @csp.graph def my_graph(): - # At this point we create the graph-time representation of the input adapter. This will be converted + # At this point we create the graph-time representation of the input adapter. This will be converted # into the impl once the graph is done constructing and the engine is created in order to run print("Start of graph building") data = MyPushAdapter(timedelta(seconds=1)) @@ -67,4 +67,9 @@ def my_graph(): print("End of graph building") -csp.run(my_graph, realtime=True, starttime=datetime.utcnow()) +def main(): + csp.run(my_graph, realtime=True, starttime=datetime.utcnow(), endtime=timedelta(seconds=2)) + + +if __name__ == "__main__": + main() diff --git a/examples/4_writing_adapters/e_14_user_adapters_04_adaptermanager_pushinput.py b/examples/04_writing_adapters/e5_adaptermanager_pushinput.py similarity index 89% rename from examples/4_writing_adapters/e_14_user_adapters_04_adaptermanager_pushinput.py rename to examples/04_writing_adapters/e5_adaptermanager_pushinput.py index a3bd64f2..aacdb864 100644 --- a/examples/4_writing_adapters/e_14_user_adapters_04_adaptermanager_pushinput.py +++ b/examples/04_writing_adapters/e5_adaptermanager_pushinput.py @@ -1,7 +1,7 @@ """ -This example introduces the concept of an AdapterManager for realtime data. AdapterManagers are constructs that are used -when you have a shared input or output resources ( ie single CSV / Parquet file, some pub/sub session, etc ) -that you want to connect to once, but provide data to / from many input / output adapters ( aka time series ) +This example introduces the concept of an AdapterManager for realtime data. AdapterManagers are constructs that are used +when you have a shared input or output resources (ie single CSV / Parquet file, some pub/sub session, etc) +that you want to connect to once, but provide data to/from many input/output adapters (aka time series) """ import random @@ -21,7 +21,7 @@ class MyData(csp.Struct): value: int -# This object represents our AdapterManager at graph time. It describes the manager's properties +# This object represents our AdapterManager at graph time. It describes the manager's properties # and will be used to create the actual impl when its time to build the engine class MyAdapterManager: def __init__(self, interval: timedelta): @@ -66,7 +66,7 @@ def __init__(self, engine, interval): self._thread = None def start(self, starttime, endtime): - """start wil get called at the start of the engine run. At this point + """start will get called at the start of the engine run. At this point one would start up the realtime data source / spawn the driving thread(s) and subscribe to the needed data""" print("MyAdapterManagerImpl::start") @@ -113,7 +113,7 @@ def _run(self): # The Impl object is created at runtime when the graph is converted into the runtime engine -# it does not exist at graph building time. a managed sim adapter impl will get the +# it does not exist at graph building time. a managed sim adapter impl will get the # adapter manager runtime impl as its first argument class MyPushAdapterImpl(PushInputAdapter): def __init__(self, manager_impl, symbol): @@ -151,4 +151,9 @@ def my_graph(): print("End of graph building") -csp.run(my_graph, starttime=datetime.utcnow(), endtime=timedelta(seconds=10), realtime=True) +def main(): + csp.run(my_graph, starttime=datetime.utcnow(), endtime=timedelta(seconds=2), realtime=True) + + +if __name__ == "__main__": + main() diff --git a/examples/4_writing_adapters/e_14_user_adapters_05_outputadapter.py b/examples/04_writing_adapters/e6_outputadapter.py similarity index 96% rename from examples/4_writing_adapters/e_14_user_adapters_05_outputadapter.py rename to examples/04_writing_adapters/e6_outputadapter.py index cc0480c0..eafb83fb 100644 --- a/examples/4_writing_adapters/e_14_user_adapters_05_outputadapter.py +++ b/examples/04_writing_adapters/e6_outputadapter.py @@ -59,11 +59,15 @@ def my_graph(): MyBufferWriterAdapter(curve, output_buffer=output_buffer) -if __name__ == "__main__": +def main(): csp.run( my_graph, starttime=datetime.utcnow(), - endtime=timedelta(seconds=3), + endtime=timedelta(seconds=2), realtime=True, ) print("output buffer: {}".format(output_buffer)) + + +if __name__ == "__main__": + main() diff --git a/examples/4_writing_adapters/e_14_user_adapters_06_adaptermanager_inputoutput.py b/examples/04_writing_adapters/e7_adaptermanager_inputoutput.py similarity index 97% rename from examples/4_writing_adapters/e_14_user_adapters_06_adaptermanager_inputoutput.py rename to examples/04_writing_adapters/e7_adaptermanager_inputoutput.py index c2a789f6..d05b1534 100644 --- a/examples/4_writing_adapters/e_14_user_adapters_06_adaptermanager_inputoutput.py +++ b/examples/04_writing_adapters/e7_adaptermanager_inputoutput.py @@ -124,9 +124,7 @@ def __init__(self, manager, symbol): class MyOutputAdapterImpl(OutputAdapter): - """Similarly, our output adpter is simple as well, defering - its functionality to the manager - """ + """Similarly, our output adapter is simple as well, deferring its functionality to the manager""" def __init__(self, manager, symbol): manager.register_publication(symbol) @@ -168,5 +166,9 @@ def my_graph(): adapter_manager.publish(data_3, "data_3") +def main(): + csp.run(my_graph, starttime=datetime.utcnow(), endtime=timedelta(seconds=2), realtime=True) + + if __name__ == "__main__": - csp.run(my_graph, starttime=datetime.utcnow(), endtime=timedelta(seconds=10), realtime=True) + main() diff --git a/examples/5_cpp/1_cpp_node/CMakeLists.txt b/examples/05_cpp/1_cpp_node/CMakeLists.txt similarity index 100% rename from examples/5_cpp/1_cpp_node/CMakeLists.txt rename to examples/05_cpp/1_cpp_node/CMakeLists.txt diff --git a/examples/5_cpp/1_cpp_node/README.md b/examples/05_cpp/1_cpp_node/README.md similarity index 99% rename from examples/5_cpp/1_cpp_node/README.md rename to examples/05_cpp/1_cpp_node/README.md index 32c9823e..3c30ef50 100644 --- a/examples/5_cpp/1_cpp_node/README.md +++ b/examples/05_cpp/1_cpp_node/README.md @@ -1,17 +1,21 @@ # Custom C++ Node + This is a small example to create a custom C++ node. Compile: + ```bash python setup.py build build_ext --inplace ``` Run: + ```bash python -m piglatin ``` Output: + ```raw 2020-01-01 00:00:00.500000 input:pig 2020-01-01 00:00:00.500000 output:IGPAY diff --git a/examples/5_cpp/1_cpp_node/piglatin.cpp b/examples/05_cpp/1_cpp_node/piglatin.cpp similarity index 100% rename from examples/5_cpp/1_cpp_node/piglatin.cpp rename to examples/05_cpp/1_cpp_node/piglatin.cpp diff --git a/examples/5_cpp/1_cpp_node/piglatin/__init__.py b/examples/05_cpp/1_cpp_node/piglatin/__init__.py similarity index 100% rename from examples/5_cpp/1_cpp_node/piglatin/__init__.py rename to examples/05_cpp/1_cpp_node/piglatin/__init__.py diff --git a/examples/5_cpp/1_cpp_node/piglatin/__main__.py b/examples/05_cpp/1_cpp_node/piglatin/__main__.py similarity index 100% rename from examples/5_cpp/1_cpp_node/piglatin/__main__.py rename to examples/05_cpp/1_cpp_node/piglatin/__main__.py diff --git a/examples/5_cpp/1_cpp_node/pyproject.toml b/examples/05_cpp/1_cpp_node/pyproject.toml similarity index 100% rename from examples/5_cpp/1_cpp_node/pyproject.toml rename to examples/05_cpp/1_cpp_node/pyproject.toml diff --git a/examples/5_cpp/1_cpp_node/setup.py b/examples/05_cpp/1_cpp_node/setup.py similarity index 100% rename from examples/5_cpp/1_cpp_node/setup.py rename to examples/05_cpp/1_cpp_node/setup.py diff --git a/examples/5_cpp/2_cpp_node_with_struct/.gitignore b/examples/05_cpp/2_cpp_node_with_struct/.gitignore similarity index 100% rename from examples/5_cpp/2_cpp_node_with_struct/.gitignore rename to examples/05_cpp/2_cpp_node_with_struct/.gitignore diff --git a/examples/5_cpp/2_cpp_node_with_struct/CMakeLists.txt b/examples/05_cpp/2_cpp_node_with_struct/CMakeLists.txt similarity index 100% rename from examples/5_cpp/2_cpp_node_with_struct/CMakeLists.txt rename to examples/05_cpp/2_cpp_node_with_struct/CMakeLists.txt diff --git a/examples/5_cpp/2_cpp_node_with_struct/README.md b/examples/05_cpp/2_cpp_node_with_struct/README.md similarity index 98% rename from examples/5_cpp/2_cpp_node_with_struct/README.md rename to examples/05_cpp/2_cpp_node_with_struct/README.md index edf8cd94..03415b65 100644 --- a/examples/5_cpp/2_cpp_node_with_struct/README.md +++ b/examples/05_cpp/2_cpp_node_with_struct/README.md @@ -1,12 +1,15 @@ # Custom C++ Node w/ Struct + This is a small example to create a custom C++ node, interfacing with `csp.Struct` instances from C++. Compile: + ```bash python setup.py build build_ext --inplace ``` Run: + ```bash python -m struct ``` diff --git a/examples/5_cpp/2_cpp_node_with_struct/mystruct/__init__.py b/examples/05_cpp/2_cpp_node_with_struct/mystruct/__init__.py similarity index 100% rename from examples/5_cpp/2_cpp_node_with_struct/mystruct/__init__.py rename to examples/05_cpp/2_cpp_node_with_struct/mystruct/__init__.py diff --git a/examples/5_cpp/2_cpp_node_with_struct/mystruct/__main__.py b/examples/05_cpp/2_cpp_node_with_struct/mystruct/__main__.py similarity index 100% rename from examples/5_cpp/2_cpp_node_with_struct/mystruct/__main__.py rename to examples/05_cpp/2_cpp_node_with_struct/mystruct/__main__.py diff --git a/examples/5_cpp/2_cpp_node_with_struct/mystruct/node.py b/examples/05_cpp/2_cpp_node_with_struct/mystruct/node.py similarity index 100% rename from examples/5_cpp/2_cpp_node_with_struct/mystruct/node.py rename to examples/05_cpp/2_cpp_node_with_struct/mystruct/node.py diff --git a/examples/5_cpp/2_cpp_node_with_struct/mystruct/struct.py b/examples/05_cpp/2_cpp_node_with_struct/mystruct/struct.py similarity index 100% rename from examples/5_cpp/2_cpp_node_with_struct/mystruct/struct.py rename to examples/05_cpp/2_cpp_node_with_struct/mystruct/struct.py diff --git a/examples/5_cpp/2_cpp_node_with_struct/pyproject.toml b/examples/05_cpp/2_cpp_node_with_struct/pyproject.toml similarity index 100% rename from examples/5_cpp/2_cpp_node_with_struct/pyproject.toml rename to examples/05_cpp/2_cpp_node_with_struct/pyproject.toml diff --git a/examples/5_cpp/2_cpp_node_with_struct/setup.py b/examples/05_cpp/2_cpp_node_with_struct/setup.py similarity index 100% rename from examples/5_cpp/2_cpp_node_with_struct/setup.py rename to examples/05_cpp/2_cpp_node_with_struct/setup.py diff --git a/examples/5_cpp/2_cpp_node_with_struct/struct.cpp b/examples/05_cpp/2_cpp_node_with_struct/struct.cpp similarity index 100% rename from examples/5_cpp/2_cpp_node_with_struct/struct.cpp rename to examples/05_cpp/2_cpp_node_with_struct/struct.cpp diff --git a/examples/05_cpp/README.md b/examples/05_cpp/README.md new file mode 100644 index 00000000..8aa75d44 --- /dev/null +++ b/examples/05_cpp/README.md @@ -0,0 +1,4 @@ +# C++ Nodes and Adapters + +- [C++ Node](./e1_cpp_node/) +- [C++ Node w/ `csp.Struct`](./e2_cpp_node_with_struct/) diff --git a/examples/06_advanced/README.md b/examples/06_advanced/README.md new file mode 100644 index 00000000..01e82f7d --- /dev/null +++ b/examples/06_advanced/README.md @@ -0,0 +1,4 @@ +# Advanced + +- [Dynamic Graphs](./e1_dynamic.py) +- [Pandas Extension](./e2_pandas_extension_example.py) diff --git a/examples/2_intermediate/e_15_dynamic.py b/examples/06_advanced/e1_dynamic.py similarity index 96% rename from examples/2_intermediate/e_15_dynamic.py rename to examples/06_advanced/e1_dynamic.py index fc823e89..66d45946 100644 --- a/examples/2_intermediate/e_15_dynamic.py +++ b/examples/06_advanced/e1_dynamic.py @@ -3,7 +3,7 @@ import csp from csp import ts -# This example demonstrates the advanced concept of dynamic graphs. Dynamic graphs provide the ability to extend the shape of the graph during runtime, +# This example demonstrates the advanced concept of dynamic graphs. Dynamic graphs provide the ability to extend the shape of the graph during runtime, # which is useful when you may not necessarily know what you will be processing at start diff --git a/examples/2_intermediate/e_18_pandas_extension_example.py b/examples/06_advanced/e2_pandas_extension.py similarity index 100% rename from examples/2_intermediate/e_18_pandas_extension_example.py rename to examples/06_advanced/e2_pandas_extension.py diff --git a/examples/6_advanced/e_12_caching_example.py b/examples/6_advanced/e_12_caching_example.py deleted file mode 100644 index e9a13afd..00000000 --- a/examples/6_advanced/e_12_caching_example.py +++ /dev/null @@ -1,252 +0,0 @@ -import math -import os -import pyarrow -from datetime import datetime, timedelta -from random import randint, random - -import csp -from csp import Config, ts -from csp.cache_support import CacheConfig, CacheConfigResolver, GraphCacheOptions -from csp.impl.config import CacheCategoryConfig - - -class Book(csp.Struct): - exchange_timestamp: datetime - bid: float - bid_size: int - ask: float - ask_size: float - - -@csp.node -def book(ticker: str) -> ts[Book]: - with csp.alarms(): - a = csp.alarm(bool) - - with csp.state(): - mid = float - spread = float - mid = hash(ticker) % 100 - - with csp.start(): - csp.schedule_alarm(a, timedelta(minutes=1), False) - - if csp.ticked(a): - csp.schedule_alarm(a, timedelta(minutes=1), False) - mid = mid + random() - spread = random() - bid = max(0, mid - spread / 2) - ask = mid + spread / 2 - bid_size = randint(100, 10000) - ask_size = randint(100, 10000) - return Book( - exchange_timestamp=csp.now() - timedelta(milliseconds=200), - bid=bid, - ask=ask, - bid_size=bid_size, - ask_size=ask_size, - ) - - -@csp.graph -def mid(ticker: str) -> csp.ts[float]: - @csp.node - def _calc(book: csp.ts[Book]) -> csp.ts[float]: - if csp.ticked(book): - return (book.ask + book.bid) / 2 - - return _calc(book(ticker)) - - -@csp.node -def average_spread(book: ts[Book], decay_factor: float = 1) -> ts[float]: - with csp.state(): - last_time = float - cur_average = float - last_time = None - cur_average = None - - if csp.ticked(book): - spread = book.ask - book.bid - cur_time = csp.now() - if last_time is None: - cur_average = spread - else: - q = math.exp(-decay_factor * (cur_time - last_time).total_seconds()) - cur_average = cur_average * q + (1 - q) * spread - - last_time = cur_time - return cur_average - - -@csp.graph(cache=True) -def get_book_with_average_spread(ticker: str) -> csp.Outputs(average_spread=csp.ts[float], book_ts=ts[Book]): - book_ts = book(ticker) - return csp.output(average_spread=average_spread(book_ts), book_ts=book_ts) - - -@csp.node -def output_book_times_aux(book: ts[Book]) -> csp.ts[Book]: - with csp.alarms(): - a_start = csp.alarm(bool) - a_end = csp.alarm(bool) - - with csp.start(): - csp.schedule_alarm(a_start, timedelta(), True) - csp.schedule_alarm(a_end, csp.engine_end_time(), True) - - if csp.ticked(book): - if csp.ticked(a_end): - csp.schedule_alarm(a_end, timedelta(), True) - return book - elif csp.ticked(a_start) or csp.ticked(a_end): - return Book(exchange_timestamp=csp.now()) - - -# This is a special graph, it's for advanced usage. Here we override the timestamp with which the data is written to cache. -# It's written using the exchange_timestamp from book_ts.exchange_timestamp. This kind of graphs must always be written in a separate -# engine run and consumed using get_book_with_average_spread_exchange_timestamp.cached -@csp.graph(cache=True, cache_options=GraphCacheOptions(data_timestamp_column_name="book_ts.exchange_timestamp")) -def get_book_with_average_spread_exchange_timestamp(ticker: str) -> csp.Outputs( - average_spread=csp.ts[float], book_ts=ts[Book] -): - return csp.output( - average_spread=get_book_with_average_spread(ticker).average_spread, - book_ts=output_book_times_aux(get_book_with_average_spread(ticker).book_ts), - ) - - -@csp.graph(cache=True, cache_options=GraphCacheOptions(category=["forecasts", "researched", "mean_reversion_based"])) -def dummy_mean_reversion_forecast(ticker: str) -> csp.Outputs(dummy_fc=csp.ts[float]): - @csp.node - def _calc(mid_price: csp.ts[float]) -> csp.ts[float]: - """Compute a dummy mean reversion forecast - :param sampled_prices: - :return: - """ - with csp.state(): - look_back_timedelta = timedelta(minutes=5) - - with csp.start(): - csp.set_buffering_policy(mid_price, tick_history=look_back_timedelta) - - prev_price = csp.value_at(mid_price, -look_back_timedelta, default=math.nan) - - if not math.isnan(prev_price): - return prev_price - mid_price - - return csp.output(dummy_fc=_calc(mid(ticker))) - - -@csp.graph -def main_graph(): - csp.print("AAPL spread", get_book_with_average_spread("AAPL").average_spread) - get_book_with_average_spread("IBM") - csp.print("IBM spread", get_book_with_average_spread("IBM").average_spread) - # The following line would fail, if uncommented. - # graphs with custom data_timestamp_column_name must be first written and then consumed using .cached property - # csp.print('IBM spread', get_book_with_average_spread_exchange_timestamp('AAPL').average_spread) - # We can write them though without consuming - get_book_with_average_spread_exchange_timestamp("AAPL") - get_book_with_average_spread_exchange_timestamp("IBM") - dummy_mean_reversion_forecast("AAPL") - dummy_mean_reversion_forecast("IBM") - - -@csp.graph -def main_graph_cached(): - csp.print("AAPL spread", get_book_with_average_spread.cached("AAPL").average_spread) - csp.print( - "AAPL spread exchange timestamp", get_book_with_average_spread_exchange_timestamp.cached("AAPL").average_spread - ) - # We can also read some specific columns from the struct, only bid will be read from file in this case: - csp.print("AAPL bid", get_book_with_average_spread.cached("AAPL").book_ts.bid) - - -if __name__ == "__main__": - output_folder = os.path.expanduser("~/csp_example_cache") - output_folder_forecasts = os.path.join(output_folder, "overridden_forecast_output_folder") - starttime = datetime(2020, 1, 1, 9, 29) - endtime = starttime + timedelta(minutes=20) - cache_config = CacheConfig( - data_folder=output_folder, - category_overrides=[ - CacheCategoryConfig(category=["forecasts", "researched"], data_folder=output_folder_forecasts) - ], - ) - csp.run(main_graph, starttime=starttime, endtime=endtime, config=Config(cache_config=cache_config)) - print("-" * 30 + " FINISHED MAIN GRAPH RUN" + "-" * 30) - # After the cache was generated, we can run the graph that accesses cached only - csp.run( - main_graph_cached, - starttime=starttime, - endtime=endtime, - config=Config(cache_config=CacheConfig(data_folder=output_folder)), - ) - print("-" * 30 + " FINISHED CACHED GRAPH RUN" + "-" * 30) - - # We can now look into the dataset on disk: - cached_data = get_book_with_average_spread.using().cached_data(output_folder)("AAPL") - print("Files are:", cached_data.get_data_files_for_period(starttime, endtime)) - # We can also load the data as dataframe: - print("AAPL data:\n", cached_data.get_data_df_for_period(starttime, endtime)) - # We can alternative load just some columns - print( - "AAPL data few columns:\n", - cached_data.get_data_df_for_period(starttime, endtime, column_list=["book_ts.ask", "book_ts.bid"]), - ) - # The following will raise an exception since we don't have the data for the given period in cache: - try: - print( - "AAPL data few columns larger period:\n", - cached_data.get_data_df_for_period( - starttime - timedelta(seconds=1), endtime, column_list=["book_ts.ask", "book_ts.bid"] - ), - ) - except RuntimeError as e: - print(f"Missing data error: {str(e)}") - # The following will work, since we provide missing_range_handler that ignores the missing ranges - print( - "AAPL data few columns larger period:\n", - cached_data.get_data_df_for_period( - starttime - timedelta(seconds=1), - endtime, - missing_range_handler=lambda start, end: True, - column_list=["book_ts.ask", "book_ts.bid"], - ), - ) - - fc_cached_data = dummy_mean_reversion_forecast.cached_data(output_folder_forecasts)("AAPL") - - # alternative more convenient way to get cached data if cache config available is to use CacheDataPathResolver: - data_path_resolver = CacheConfigResolver(cache_config) - fc_cached_data2 = dummy_mean_reversion_forecast.cached_data(data_path_resolver)("AAPL") - - print( - f"AAPL mean reversion forecast files:{list(fc_cached_data.get_data_files_for_period(starttime, endtime).values())}" - ) - print(f"AAPL mean reversion forecast:\n{fc_cached_data.get_data_df_for_period(starttime, endtime)}") - fc_cached_data.get_data_df_for_period(starttime, endtime) == fc_cached_data2.get_data_df_for_period( - starttime, endtime - ) - # The loaded data from both ways of accessing the cache should be the same - assert ( - ( - fc_cached_data.get_data_df_for_period(starttime, endtime) - == fc_cached_data2.get_data_df_for_period(starttime, endtime) - ) - .all() - .all() - ) - - # we can also read the forecasts directly as parquet table: - appl_fc_arrow_table = fc_cached_data.get_data_df_for_period( - starttime, - endtime, - data_loader_function=lambda starttime, - endtime, - data_file, - column_list, - basket_column_list: pyarrow.parquet.read_table(data_file, column_list), - )[0] - print(f"AAPL forecast using pyarrow read: {appl_fc_arrow_table['dummy_fc']}") diff --git a/examples/98_just_for_fun/README.md b/examples/98_just_for_fun/README.md new file mode 100644 index 00000000..988abb74 --- /dev/null +++ b/examples/98_just_for_fun/README.md @@ -0,0 +1,3 @@ +# Just for fun! + +- [NAND Computer](./e1_csp_nand_computer.py) diff --git a/examples/98_just_for_fun/e_99_csp_nand_computer.py b/examples/98_just_for_fun/e1_csp_nand_computer.py similarity index 80% rename from examples/98_just_for_fun/e_99_csp_nand_computer.py rename to examples/98_just_for_fun/e1_csp_nand_computer.py index 1b34fa18..378663f9 100644 --- a/examples/98_just_for_fun/e_99_csp_nand_computer.py +++ b/examples/98_just_for_fun/e1_csp_nand_computer.py @@ -1,7 +1,10 @@ -"""The purpose of this example is to help demonstrate the difference between csp.node and csp.graph concepts ( and because its awesome ). -We define a single node, which is a simple NAND gate. We then wire up the NAND gate nodes in more and more complex constructs under csp.graph definitions +""" +The purpose of this example is to help demonstrate the difference between csp.node and csp.graph concepts (and because its awesome). +We define a single node, which is a simple NAND gate. We then wire up the NAND gate nodes in more and more complex constructs under csp.graph definitions to build up our "computer". -csp.graph calls only define the wiring of the NAND gate nodes. At runtime, the only things that exist and execute are the NAND nodes. + +csp.graph calls only define the wiring of the NAND gate nodes. At runtime, the only things that exist and execute are the NAND nodes. + The wiring done here has been taken from the awesome book https://www.amazon.com/Elements-Computing-Systems-Building-Principles/dp/0262640686 """ @@ -88,7 +91,7 @@ def basket_to_bitstring(x: [ts[bool]]) -> ts[str]: @csp.graph -def main(bits: int = 16): +def my_graph(bits: int = 16): x = number_to_basket(42001, bits) y = number_to_basket(136, bits) @@ -101,6 +104,11 @@ def main(bits: int = 16): csp.print("x+y_bits", basket_to_bitstring(add)) -# Show graph with 4-bit ints to limit size -csp.showgraph.show_graph(main, 4) -csp.run(main, starttime=datetime(2022, 6, 24)) +def main(): + # Show graph with 4-bit ints to limit size + csp.showgraph.show_graph(my_graph, 4) + csp.run(my_graph, starttime=datetime(2022, 6, 24)) + + +if __name__ == "__main__": + main() diff --git a/examples/99_developer_tools/README.md b/examples/99_developer_tools/README.md new file mode 100644 index 00000000..872776ea --- /dev/null +++ b/examples/99_developer_tools/README.md @@ -0,0 +1,3 @@ +# Developer Tools + +- [Profiling `csp` code](./e1_profiling.py) diff --git a/examples/99_developer_tools/e1_profiling.py b/examples/99_developer_tools/e1_profiling.py new file mode 100644 index 00000000..66259d5a --- /dev/null +++ b/examples/99_developer_tools/e1_profiling.py @@ -0,0 +1,39 @@ +from datetime import datetime, timedelta + +import csp +from csp import profiler + +st = datetime(2020, 1, 1) + + +@csp.graph +def graph1(): + x = csp.curve(int, [(st + timedelta(seconds=i), i) for i in range(100)]) # 1,2,3...100 + y = x**2 + + z = x + y + w = csp.merge(y, z) + p = csp.merge(x, w) + o = csp.merge(w, p) + + csp.add_graph_output("o", o) + + +def main(): + # Example 1: view a graph's static attributes using graph_info + info = profiler.graph_info(graph1) # noqa: F841 + + # Uncomment line below to print only the static graph info for graph1 + # info.print_info() + + # Example 2: profile a graph in runtime + + with profiler.Profiler() as p: + csp.run(graph1, starttime=st, endtime=st + timedelta(seconds=100)) + + prof = p.results() + prof.print_stats() + + +if __name__ == "__main__": + main() diff --git a/examples/99_developer_tools/e_22_profiling.py b/examples/99_developer_tools/e_22_profiling.py deleted file mode 100644 index 6d5f0150..00000000 --- a/examples/99_developer_tools/e_22_profiling.py +++ /dev/null @@ -1,34 +0,0 @@ -from datetime import datetime, timedelta - -import csp -from csp import profiler - -st = datetime(2020, 1, 1) - - -@csp.graph -def graph1(): - x = csp.curve(int, [(st + timedelta(seconds=i), i) for i in range(100)]) # 1,2,3...100 - y = x**2 - - z = x + y - w = csp.merge(y, z) - p = csp.merge(x, w) - o = csp.merge(w, p) - - csp.add_graph_output("o", o) - - -# Example 1: view a graph's static attributes using graph_info - -info = profiler.graph_info(graph1) -# Uncomment line below to print only the static graph info for graph1 -# info.print_info() - -# Example 2: profile a graph in runtime - -with profiler.Profiler() as p: - results = csp.run(graph1, starttime=st, endtime=st + timedelta(seconds=100)) - -prof = p.results() -prof.print_stats() diff --git a/examples/README.md b/examples/README.md index fe6abc0f..72e59330 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1 +1,10 @@ # `csp` examples + +- [Basics](./01_basics/) +- [Intermediate](./02_intermediate/) +- [Adapters](./03_using_adapters/) +- [Writing Adapters](./04_writing_adapters/) +- [Writing C++ Nodes and Adapters](./05_cpp/) +- [Advanced](./06_advanced/) +- [Just for fun!](./98_just_for_fun/) +- [Developer Tools](./99_developer_tools/) diff --git a/pyproject.toml b/pyproject.toml index de78a26c..dafa366c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,31 +52,55 @@ Tracker = "https://github.com/point72/csp/issues" [project.optional-dependencies] develop = [ + # build/dist "bump2version>=1.0.0", "build", - "graphviz", - "httpx>=0.20,<1", - "isort>=5,<6", "ruamel.yaml", - "ruff>=0.3,<0.4", - "mdformat", - "codespell", "scikit-build", "twine", "wheel", - # Test dependencies - "polars", - "psutil", + # lint + "codespell", + "isort>=5,<6", + "mdformat", + "ruff>=0.3,<0.4", + # test + "pytest", + "pytest-asyncio", + "pytest-cov", + "pytest-sugar", + # showgraph + "graphviz", + "pillow", + # adapters + "httpx>=0.20,<1", # kafka + "polars", # parquet + "psutil", # test_engine/test_history + "slack-sdk>=3", # slack + "sqlalchemy", # db + "requests", # symphony + "threadpoolctl", # test_random + "tornado", # profiler, perspective, websocket +] +showgraph = [ + "graphviz", + "pillow", +] +test = [ + "graphviz", + "pillow", "pytest", "pytest-asyncio", "pytest-cov", "pytest-sugar", + "httpx>=0.20,<1", + "polars", + "psutil", + "requests", + "slack-sdk>=3", "sqlalchemy", "threadpoolctl", "tornado", - # Extras - "requests", - "slack-sdk>=3", ] symphony = [ "requests",