diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 207f62f..027d682 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -67,12 +67,12 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel + pip install setuptools wheel build - name: Build and publish run: | - python setup.py sdist bdist_wheel + python -m build --sdist --wheel --outdir dist/ . - name: Publish package - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.PYPI_DEPLOYMENT_TOKEN }} diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..2095729 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,34 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.10" + # You can also specify other tool versions: + # nodejs: "19" + # rust: "1.64" + # golang: "1.19" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/source/conf.py + +# Optionally build your docs in additional formats such as PDF and ePub +formats: + - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: docs/requirements.txt + - method: pip + path: . diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..3f6d54f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,40 @@ +# Changelog +## 5.30.2 +### What's Changed +* Update readthedocs to new yaml for testing. by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/40 +* Converting pandas index takes very long, add in arrow_table. by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/41 + + +**Full Changelog**: https://github.com/PingThingsIO/btrdb-python/compare/v5.30.1...v5.30.2 + +## 5.30.1 +### What's Changed +* Small version bump for pypi release + + +**Full Changelog**: https://github.com/PingThingsIO/btrdb-python/compare/v5.30.0...v5.30.1 + + +## 5.30.0 +### What's Changed +* Merge Arrow support into Main for Release by @youngale-pingthings in https://github.com/PingThingsIO/btrdb-python/pull/37 + * This PR contains many changes that support the commercial Arrow data fetches and inserts + * `arrow_` prefixed methods for `Stream` Objects: + * `insert, aligned_windows, windows, values` + * `arrow_` prefixed methods for StreamSet` objects: + * `insert, values, to_dataframe, to_polars, to_arrow_table, to_numpy, to_dict, to_series` +* Justin gilmer patch 1 by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/39 + + +**Full Changelog**: https://github.com/PingThingsIO/btrdb-python/compare/v5.28.1...v5.30.0 + + +## 5.28.1 +### What's Changed +* Upgrade ray versions by @jleifnf in https://github.com/PingThingsIO/btrdb-python/pull/15 +* Release v5.28.1 and Update Python by @youngale-pingthings in https://github.com/PingThingsIO/btrdb-python/pull/17 + +### New Contributors +* @jleifnf made their first contribution in https://github.com/PingThingsIO/btrdb-python/pull/15 + +**Full Changelog**: https://github.com/PingThingsIO/btrdb-python/compare/v5.15.1...v5.28.1 diff --git a/btrdb/conn.py b/btrdb/conn.py index 09f8721..a77eced 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -61,7 +61,10 @@ def __init__(self, addrportstr, apikey=None): """ addrport = addrportstr.split(":", 2) - chan_ops = [] + # 100MB size limit ~ 2500 streams for 5000 points with each point being 64bit + # 500MB size limit ~ 13K streams for 5000 points + # -1 size limit = no limit of size to send + chan_ops = [("grpc.max_receive_message_length", -1)] if len(addrport) != 2: raise ValueError("expecting address:port") @@ -313,6 +316,7 @@ def streams(self, *identifiers, versions=None, is_collection_prefix=False): raise ValueError( f"Could not identify stream based on `{ident}`. Identifier must be UUID or collection/name." ) + obj = StreamSet(streams) if versions: diff --git a/btrdb/stream.py b/btrdb/stream.py index a7a840a..7a02ac2 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -918,7 +918,7 @@ def arrow_values( end : int or datetime like object The end time in nanoseconds for the range to be queried. (see :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) - version: int + version: int, default: 0 The version of the stream to be queried auto_retry: bool, default: False Whether to retry this request in the event of an error @@ -932,6 +932,7 @@ def arrow_values( Exponential factor by which the backoff increases between retries. Will be ignored if auto_retry is False + Returns ------ pyarrow.Table @@ -1050,6 +1051,7 @@ def arrow_aligned_windows( start: int, end: int, pointwidth: int, + sort_time: bool = False, version: int = 0, auto_retry=False, retries=5, @@ -1082,7 +1084,9 @@ def arrow_aligned_windows( :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) pointwidth : int, required Specify the number of ns between data points (2**pointwidth) - version : int + sort_time : bool, default: False + Should the table be sorted on the 'time' column? + version : int, default: 0 Version of the stream to query auto_retry: bool, default: False Whether to retry this request in the event of an error @@ -1124,7 +1128,10 @@ def arrow_aligned_windows( ) if len(tables) > 0: tabs, ver = zip(*tables) - return pa.concat_tables(tabs) + if sort_time: + return pa.concat_tables(tabs).sort_by("time") + else: + return pa.concat_tables(tabs) else: schema = pa.schema( [ @@ -1218,6 +1225,7 @@ def arrow_windows( start: int, end: int, width: int, + sort_time: bool = False, version: int = 0, auto_retry=False, retries=5, @@ -1236,6 +1244,8 @@ def arrow_windows( :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) width : int, required The number of nanoseconds in each window. + sort_time : bool, default: False + Should the table be sorted on the 'time' column. version : int, default=0, optional The version of the stream to query. auto_retry: bool, default: False @@ -1286,7 +1296,10 @@ def arrow_windows( ) if len(tables) > 0: tabs, ver = zip(*tables) - return pa.concat_tables(tabs) + if sort_time: + return pa.concat_tables(tabs).sort_by("time") + else: + return pa.concat_tables(tabs) else: schema = pa.schema( [ @@ -2101,6 +2114,8 @@ def arrow_values( ): """Return a pyarrow table of stream values based on the streamset parameters. + This data will be sorted by the 'time' column. + Notes ----- This method is available for commercial customers with arrow-enabled servers. @@ -2145,6 +2160,7 @@ def arrow_values( data = tablex else: data = tablex + data = data.sort_by("time") elif self.width is not None and self.depth is not None: # create list of stream.windows data (the windows method should @@ -2175,6 +2191,7 @@ def arrow_values( data = tablex else: data = tablex + data = data.sort_by("time") else: sampling_freq = params.pop("sampling_frequency", 0) period_ns = 0 diff --git a/btrdb/transformers.py b/btrdb/transformers.py index 4939215..9bcf7b5 100644 --- a/btrdb/transformers.py +++ b/btrdb/transformers.py @@ -224,10 +224,7 @@ def arrow_to_dataframe( tmp = tmp_table.select(["time", *usable_cols]) else: tmp = tmp_table - df = tmp.to_pandas(date_as_object=False, types_mapper=pd.ArrowDtype) - df = df.set_index("time") - df.index = pd.DatetimeIndex(df.index, tz="UTC") - return df + return tmp.to_pandas(date_as_object=False, types_mapper=pd.ArrowDtype) def to_dataframe(streamset, columns=None, agg="mean", name_callable=None): @@ -668,5 +665,7 @@ class StreamSetTransformer(object): to_polars = to_polars arrow_to_polars = arrow_to_polars + arrow_to_arrow_table = arrow_to_arrow_table + to_csv = to_csv to_table = to_table diff --git a/btrdb/version.py b/btrdb/version.py index 492de74..e98a93b 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## -__version_info__ = {"major": 5, "minor": 28, "micro": 1, "releaselevel": "final"} +__version_info__ = {"major": 5, "minor": 30, "micro": 2, "releaselevel": "final"} ########################################################################## ## Helper Functions diff --git a/docs/requirements.txt b/docs/requirements.txt index a3bb1f5..49e71d8 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,5 @@ alabaster>=0.7.12 Sphinx>=1.7 sphinx-rtd-theme +numpydoc +pydata-sphinx-theme diff --git a/docs/source/api-index.rst b/docs/source/api-index.rst new file mode 100644 index 0000000..72b478f --- /dev/null +++ b/docs/source/api-index.rst @@ -0,0 +1,16 @@ + +.. _API REF: + +API Reference +------------- + +.. toctree:: + :maxdepth: 2 + + api/package + api/conn + api/streams + api/points + api/exceptions + api/transformers + api/utils-timez diff --git a/docs/source/api/conn.rst b/docs/source/api/conn.rst index c577eaa..62ba065 100644 --- a/docs/source/api/conn.rst +++ b/docs/source/api/conn.rst @@ -1,6 +1,7 @@ btrdb.conn ============= +.. _Conn info: .. automodule:: btrdb.conn .. autoclass:: BTrDB diff --git a/docs/source/api/streams.rst b/docs/source/api/streams.rst index 6c55473..7137d36 100644 --- a/docs/source/api/streams.rst +++ b/docs/source/api/streams.rst @@ -1,10 +1,12 @@ btrdb.stream ============== +.. _StreamGeneralDocs: .. automodule:: btrdb.stream .. autoclass:: Stream :members: +.. _StreamSet API: .. autoclass:: StreamSetBase :members: diff --git a/docs/source/api/transformers.rst b/docs/source/api/transformers.rst index facbbc4..830e0b5 100644 --- a/docs/source/api/transformers.rst +++ b/docs/source/api/transformers.rst @@ -1,3 +1,4 @@ +.. _TransformersDocs: btrdb.transformers ========================= diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst new file mode 100644 index 0000000..8a6881d --- /dev/null +++ b/docs/source/changelog.rst @@ -0,0 +1,80 @@ +.. _changelog: +Changelog +========= + +5.30.2 +------ +What’s Changed +^^^^^^^^^^^^^^ +- Update readthedocs to new yaml for testing. by @justinGilmer in + https://github.com/PingThingsIO/btrdb-python/pull/40 +- Converting pandas index takes very long, fix this in arrow_table. by + @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/41 + +**Full Changelog**: +`5.30.2 `_ + +.. _section-1: + +5.30.1 +------ +.. _whats-changed-1: + +What’s Changed +^^^^^^^^^^^^^^ + +- Small version bump for pypi release + +**Full Changelog**: +`5.30.1 `_ + +.. _section-2: + +5.30.0 +------ +.. _whats-changed-2: + +What’s Changed +^^^^^^^^^^^^^^ + +- Merge Arrow support into Main for Release by @youngale-pingthings in + https://github.com/PingThingsIO/btrdb-python/pull/37 + + - This PR contains many changes that support the commercial Arrow + data fetches and inserts + - ``arrow_`` prefixed methods for ``Stream`` Objects: + + - ``insert, aligned_windows, windows, values`` + + - ``arrow_`` prefixed methods for StreamSet\` objects: + + - ``insert, values, to_dataframe, to_polars, to_arrow_table, to_numpy, to_dict, to_series`` + +- Justin gilmer patch 1 by @justinGilmer in + https://github.com/PingThingsIO/btrdb-python/pull/39 + +**Full Changelog**: +`5.30.0 `_ + +.. _section-3: + +5.28.1 +------ +.. _whats-changed-3: + +What’s Changed +^^^^^^^^^^^^^^ + +- Upgrade ray versions by @jleifnf in + https://github.com/PingThingsIO/btrdb-python/pull/15 +- Release v5.28.1 and Update Python by @youngale-pingthings in + https://github.com/PingThingsIO/btrdb-python/pull/17 + +New Contributors +^^^^^^^^^^^^^^^^ + +- @jleifnf made their first contribution in + https://github.com/PingThingsIO/btrdb-python/pull/15 + +**Full Changelog**: +`5.28.1 `_ diff --git a/docs/source/concepts.rst b/docs/source/concepts.rst index 09c6675..4685556 100644 --- a/docs/source/concepts.rst +++ b/docs/source/concepts.rst @@ -22,6 +22,7 @@ BTrDB focuses on univariate data which opens a host of benefits and is one of the reasons BTrDB is able to process incredibly large amounts of data quickly and easily. +.. _Points Described: Points ------------ Points of data within a time series make up the smallest objects you will be @@ -40,10 +41,10 @@ value within the stream. # view time and value of a single point in the stream point.time - >> 1547241923338098176 + >>> 1547241923338098176 point.value - >> 120.5 + >>> 120.5 StatPoint ^^^^^^^^^^^^ @@ -63,30 +64,37 @@ not need to read the underlying points to return these statistics! # view aggregate values for points in a stream point.time - >> 1547241923338098176 + >>> 1547241923338098176 point.min - >> 42.1 + >>> 42.1 point.mean - >> 78.477 + >>> 78.477 point.max - >> 122.4 + >>> 122.4 point.count - >> 18600 + >>> 18600 point.stddev - >> 3.4 + >>> 3.4 + + +Tabular Data +------------ +In addition to working with the :code:`RawPoint` or :code:`StatPoint` objects, newer versions of the platform now natively support some tabular data formats as well. +This is enabled for commercial customers and are available using the :code:`stream.arrow_` or :code:`streamset.arrow_` methods. +Refer to the :ref:`arrow enabled queries page ` and the :ref:`API docs ` Streams ------------ Streams represent a single series of time/value pairs. As such, the database can hold an almost unlimited amount of individual streams. Each stream has a -`collection` which is similar to a "path" or grouping for multiple streams. Each -steam will also have a `name` as well as a `uuid` which is guaranteed to be unique +:code:`collection` which is similar to a "path" or grouping for multiple streams. Each +steam will also have a :code:`name` as well as a :code:`uuid` which is guaranteed to be unique across streams. BTrDB data is versioned such that changes to a given stream (time series) will @@ -109,26 +117,26 @@ metadata. # retrieve stream's UUID stream.uuid - >> UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + >>> UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") # retrieve stream's current version stream.version() - >> 244 + >>> 244 # retrieve stream tags stream.tags() - >> {'name': 'L1MAG', 'unit': 'volts', 'ingress': ''} + >>> {'name': 'L1MAG', 'unit': 'volts', 'ingress': ''} # retrieve stream annotations stream.annotations() - >> {'poc': 'Salvatore McFesterson', 'region': 'northwest', 'state': 'WA'} + >>> {'poc': 'Salvatore McFesterson', 'region': 'northwest', 'state': 'WA'} # loop through points in the stream for point, _ in stream.values(end=1547241923338098176, version=133): print(point) - >> RawPoint(1500000000100000000, 2.4) - >> RawPoint(1500000000200000000, 2.8) - >> RawPoint(1500000000300000000, 3.6) + >>> RawPoint(1500000000100000000, 2.4) + >>> RawPoint(1500000000200000000, 2.8) + >>> RawPoint(1500000000300000000, 3.6) ... @@ -166,3 +174,9 @@ our API docs to see the full list of features provided to you. 7 1500000000700000000 8.0 NaN 8 1500000000800000000 NaN 9.0 9 1500000000900000000 10.0 NaN + + +Apache-Arrow Accelerated Methods +-------------------------------- + +* Refer to :ref:`Arrow Page` diff --git a/docs/source/conf.py b/docs/source/conf.py index a2ce391..1cc1ebf 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -27,7 +27,7 @@ # -- Project information ----------------------------------------------------- project = "btrdb" -copyright = "2021, Ping Things, Inc." +copyright = "2023, Ping Things, Inc." author = "PingThingsIO" # The short X.Y version @@ -88,7 +88,8 @@ # a list of builtin themes. # # html_theme = "alabaster" -html_theme = "sphinx_rtd_theme" +# html_theme = "sphinx_rtd_theme" +html_theme = "pydata_sphinx_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -103,7 +104,7 @@ "show_related": False, "note_bg": "#FFF59C", "description": "A midweight library to converse with the BTrDB database.", - "extra_nav_links": {"btrdb": "http://btrdb.io"}, + "extra_nav_links": {"btrdb": "http://btrdb-python.readthedocs.io"}, "show_relbars": True, } diff --git a/docs/source/explained.rst b/docs/source/explained.rst index 6ace50e..68bdfe7 100644 --- a/docs/source/explained.rst +++ b/docs/source/explained.rst @@ -126,6 +126,5 @@ The original version of this page can be found at: This page is written based on the following sources: -- `Homepage `_ - `Whitepaper `_ - `Code `_ diff --git a/docs/source/index.rst b/docs/source/index.rst index da1b71e..3f1076b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -42,7 +42,7 @@ your appetite. import btrdb from btrdb.utils.timez import to_nanoseconds - # establish connection to server + # establish connection to a server conn = btrdb.connect("192.168.1.101:4410") # search for streams and view metadata @@ -75,27 +75,8 @@ in Github! concepts working explained - - -API Reference -------------- - -.. toctree:: - :maxdepth: 2 - - api/package - api/conn - api/streams - api/points - api/exceptions - api/transformers - api/utils-timez - - -Maintainers ------------ - -* :doc:`maintainers/anaconda` + api-index + changelog diff --git a/docs/source/installing.rst b/docs/source/installing.rst index 5b82210..71cab8e 100644 --- a/docs/source/installing.rst +++ b/docs/source/installing.rst @@ -13,13 +13,24 @@ We recommend using pip to install btrdb-python on all platforms: $ pip install btrdb +With :code:`btrdb>=5.30.2`, there are now extra dependencies that can be installed with ``pip``. +We recommend installing the :code:`data` extra dependencies (the second option in the code block below). + + +.. code-block:: shell-session + + $ pip install "btrdb>=5.30.2" # standard btrdb + $ pip install "btrdb[data]>=5.30.2" # btrdb with data science packages included (recommended) + $ pip install "btrdb[all]>=5.30.2" # btrdb with testing, data science and all other optional packages + + To get a specific version of btrdb-python supply the version number. The major -version of this library is pegged to the major version of the BTrDB database as -in the 4.x bindings are best used to speak to a 4.x BTrDB database. +version of this library is tied to the major version of the BTrDB database as +in the 4.X bindings are best used to speak to a 4.X BTrDB database, the 5.X bindings for 5.X platform.. .. code-block:: bash - $ pip install btrdb==4.11.2 + $ pip install "btrdb[data]==5.30.2" To upgrade using pip: @@ -32,12 +43,4 @@ To upgrade using pip: Installing with Anaconda ------------------------ -If you'd like to use Anaconda, you'll need to download the library from the pingthings -channel as shown below. - -Note however that only the version 5 bindings are available in Anaconda Cloud. If you'd -like to install the version 4 bindings you will need to use `pip` as shown above. - -.. code-block:: bash - - $ conda install -c pingthings btrdb +We recommend installing using ``pip``. diff --git a/docs/source/quick-start.rst b/docs/source/quick-start.rst index bc4af5f..8901177 100644 --- a/docs/source/quick-start.rst +++ b/docs/source/quick-start.rst @@ -20,6 +20,14 @@ Connecting to a server is easy with the supplied :code:`connect` function from t # connect with API key conn = btrdb.connect("192.168.1.101:4411", apikey="123456789123456789") +Get Platform Information +^^^^^^^^^^^^^^^^^^^^^^^^ +.. code-block:: python + + conn.info() + +Refer to :ref:`the connection API documentation page. ` + Retrieving a Stream ---------------------- diff --git a/docs/source/working.rst b/docs/source/working.rst index 8773907..cc33d2c 100644 --- a/docs/source/working.rst +++ b/docs/source/working.rst @@ -15,4 +15,7 @@ to interact with the BTrDB database. working/stream-view-data working/streamsets working/multiprocessing + working/multistream + working/arrow-enabled-queries + working/dash working/ray diff --git a/docs/source/working/arrow-enabled-queries.rst b/docs/source/working/arrow-enabled-queries.rst new file mode 100644 index 0000000..8a40a5e --- /dev/null +++ b/docs/source/working/arrow-enabled-queries.rst @@ -0,0 +1,32 @@ +.. -*- mode: rst -*- + +.. _Arrow Page: + +Arrow-enabled Queries +===================== + +In more recent deployments of the BTrDB platform (>=5.30.0), commercial customers also have access to additional accelerated functionality for data fetching and inserting. + +Also, most :code:`StreamSet` based value queries (:code:`AlignedWindows`, :code:`Windows`, :code:`Values`) are multithreaded by default. +This leads to decent performance improvements for fetching and inserting data using the standard :code:`StreamSet` api without any edits by the user. +Refer to :ref:`The StreamSet API ` + +In addition to these improvements to the standard API, commercial customers also have access to additional accelerated data fetching and inserting methods that can dramatically speed up their workflows. + +Apache Arrow Data Format +^^^^^^^^^^^^^^^^^^^^^^^^ + +While keeping our standard API consistent with our :code:`Point` and :code:`StatPoint` :ref:`python object model `, we have also created additional methods that will provide this same type of data, but in a tabular format by default. +Leveraging the `language-agnostic columnar data format Arrow `_, we can transmit our timeseries data in a format that is already optimized for data analytics with well-defined schemas that take the guesswork out of the data types, timezones, etc. +To learn more about these methods, please refer to the :ref:`arrow_ prefixed methods for both Stream and StreamSet objects ` and the :ref:`StreamSet transformer methods `. + +.. _ArrowMultistreamDocs: +True Multistream Support +^^^^^^^^^^^^^^^^^^^^^^^^ + +Until now, there has not been a true multistream query support, our previous api and with the new edits, emulates multistream support with :code:`StreamSet`s and using multithreading. +However, this will still only scale to an amount of streams based on the amount of threads that the python threadpool logic can support. + +Due to this, raw data queries for :code:`StreamSet`s using our arrow api :code:`StreamSet.filter(start=X, end=Y,).arrow_values()` will now perform true multistream queries. +The platform, instead of the python client, will now quickly grab all the stream data for all streams in your streamset, and then package that back to the python client in an :code:`arrow` table! +This leads to data fetch speedups on the order of 10-50x based on the amount and kind of streams. diff --git a/docs/source/working/dash.rst b/docs/source/working/dash.rst new file mode 100644 index 0000000..b6bf10d --- /dev/null +++ b/docs/source/working/dash.rst @@ -0,0 +1,89 @@ +.. -*- mode: rst -*- + +Working with Dash and Plotly +============================ +From `Plotly's getting start guide: `_ "The plotly Python library is an interactive, open-source plotting library that supports over 40 unique chart types covering a wide range of statistical, financial, geographic, scientific, and 3-dimensional use-cases." + +These tools are usable in jupyter notebooks and can also be ran as their own standalone apps using plotly-dash. + + +Below are two examples using the standard API and the Arrow enabled API to retrieve data as a pandas.DataFrame, and then plotting the results. +These examples are based off of the `minimal dash app `_. + +Non-Multistream API +------------------- + +.. code-block:: python + + from dash import Dash, html, dcc, callback, Output, Input + import plotly.express as px + import pandas as pd + import btrdb + + conn = btrdb.connect() + streams = conn.streams_in_collection("YOUR_COLLECTION_HERE") + streamset = conn.streams(*[s.uuid for s in streams]) + latest = streamset.latest() + end = min([pt.time for pt in latest]) + start = end - btrdb.utils.timez.ns_delta(minutes=5) + + df = streamset.filter(start=start, end=end).to_dataframe() + + app = Dash(__name__) + + app.layout = html.Div([ + html.H1(children='Title of Dash App', style={'textAlign':'center'}), + dcc.Dropdown(df.columns, id='dropdown-selection'), + dcc.Graph(id='graph-content') + ]) + + @callback( + Output('graph-content', 'figure'), + Input('dropdown-selection', 'value') + ) + def update_graph(value): + dff = df[value] + return px.line(dff, x=dff.index, y=value) + + if __name__ == '__main__': + app.run(debug=True) + + + +Multistream API +--------------- +.. code-block:: python + + from dash import Dash, html, dcc, callback, Output, Input + import plotly.express as px + import pandas as pd + import btrdb + + conn = btrdb.connect() + streams = conn.streams_in_collection("YOUR_COLLECTION_HERE") + streamset = conn.streams(*[s.uuid for s in streams]) + latest = streamset.latest() + end = min([pt.time for pt in latest]) + start = end - btrdb.utils.timez.ns_delta(minutes=5) + + df = streamset.filter(start=start, end=end).arrow_to_dataframe() + df = df.set_index('time') + + app = Dash(__name__) + + app.layout = html.Div([ + html.H1(children='Title of Dash App', style={'textAlign':'center'}), + dcc.Dropdown(df.columns, id='dropdown-selection'), + dcc.Graph(id='graph-content') + ]) + + @callback( + Output('graph-content', 'figure'), + Input('dropdown-selection', 'value') + ) + def update_graph(value): + dff = df[value] + return px.line(dff, x=dff.index, y=value) + + if __name__ == '__main__': + app.run(debug=True) diff --git a/docs/source/working/multistream.rst b/docs/source/working/multistream.rst new file mode 100644 index 0000000..77548ff --- /dev/null +++ b/docs/source/working/multistream.rst @@ -0,0 +1,6 @@ +.. -*- mode: rst -*- + +Multistream Queries +=================== + +Refer to :ref:`ArrowMultistreamDocs`. diff --git a/docs/source/working/server.rst b/docs/source/working/server.rst index cd479c0..4eaf1b4 100644 --- a/docs/source/working/server.rst +++ b/docs/source/working/server.rst @@ -14,7 +14,7 @@ an API key will raise an exception. Connecting to servers --------------------------- -The btrdb library comes with a high level :code:`connnect` function to interface +The btrdb library comes with a high level :code:`connect` function to interface with a BTrDB server. Upon successfully connecting, you will be returned a :code:`BTrDB` object which is the starting point for all of your server interactions. diff --git a/pyproject.toml b/pyproject.toml index 066ed3b..f42ddf0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "btrdb" -version = "5.28.1" +version = "5.30.2" authors = [ {name="PingThingsIO", email="support@pingthings.io"}, ] @@ -33,6 +33,10 @@ dependencies = [ "pytz", "pyyaml", "certifi", + "pyarrow", + "polars", + "numpy", + "pandas>=2.0", ] [project.optional-dependencies] diff --git a/release.sh b/release.sh index d630e0e..31f849e 100755 --- a/release.sh +++ b/release.sh @@ -37,11 +37,12 @@ echo "Setting version to v$1.$2.$3" VERSION_CODE="__version_info__ = { 'major': $1, 'minor': $2, 'micro': $3, 'releaselevel': 'final'}" sed -i.bak "s/^__version_info__.*$/${VERSION_CODE}/g" btrdb/version.py +sed -i.bak "s/^version.*$/version\ = \"$1.$2.$3\"/g" pyproject.toml git add btrdb/version.py +git add pyproject.toml git commit -m "Release v$1.$2.$3" git tag v$1.$2.$3 git push origin v$1.$2.$3 -sleep 10 git push