Implement use_blob for execute_mdx_csv function

[MariusWirtz]

Solves #884

Add use_blob to execute_mdx_csv, execute_mdx_dataframe, execute_view_csv, execute_view_dataframe

Reduces time to execute MDX by 20-40%
Reduces time to execute NativeView by 40-50%
Reduces memory footprint of Python process significantly in all cases.

[MariusWirtz]

The first results look promising.

On very large datasets using blobs provides 30% or 40% better performance and a much lower memory footprint.

Environment	use_blob	Number of cells	Runtime	Difference in Seconds	Difference in %
Server Linux	True	100'000	00:06.9	00:01.6	+31%
Server Linux	False	100'000	00:05.3
Server Linux	True	500'000	00:10.2	00:01.8	-15%
Server Linux	False	500'000	00:12.0
Server Linux	True	1'000'000	00:14.8	00:05.5	-27%
Server Linux	False	1'000'000	00:20.3
Server Linux	True	2'500'000	00:29.1	00:21.0	-42%
Server Linux	False	2'500'000	00:50.1
Local Windows	True	100'000	00:17.7	00:05.6	+46%
Local Windows	False	100'000	00:12.1
Local Windows	True	500'000	00:29.3	00:02.1	-7%
Local Windows	False	500'000	00:31.4
Local Windows	True	1'000'000	00:44.3	00:11.6	-21%
Local Windows	False	1'000'000	00:55.9
Local Windows	True	2'500'000	01:26.8	00:45.6	-34%
Local Windows	False	2'500'000	02:12.4

from datetime import datetime

from mdxpy import MdxBuilder, MdxHierarchySet

from TM1py.Objects.Cube import Cube
from TM1py.Objects.Dimension import Dimension
from TM1py.Objects.Hierarchy import Hierarchy
from TM1py.Services.TM1Service import TM1Service

PARAMETER_SETS = {

    "Linux Server": {
        "address": "10.77.19.30",
        "port": 8010,
        "ssl": True,
        "user": "Admin",
        "password": "apple"
    },

    "Windows Local": {
        "address": "",
        "port": 8010,
        "ssl": True,
        "user": "Admin",
        "password": "apple"
    }
}

CUBE = "TM1py Cube"
CHUNK_SIZES = [100_000, 500_000, 1_000_000, 2_500_000]
ITERATIONS = 2


def setup(tm1: TM1Service):
    hierarchy = Hierarchy("TM1py Dimension 1", "TM1py Dimension 1")
    cells = dict()
    for e in range(max(CHUNK_SIZES)):
        element = str(e).zfill(7)
        hierarchy.add_element(element, "Numeric")
        cells[element, "Measure"] = e

    dimension = Dimension(name="TM1py Dimension 1", hierarchies=[hierarchy])
    tm1.dimensions.update_or_create(dimension)

    hierarchy = Hierarchy("TM1py Dimension 2", "TM1py Dimension 2")
    hierarchy.add_element("Measure", "Numeric")
    dimension = Dimension(name="TM1py Dimension 2", hierarchies=[hierarchy])
    tm1.dimensions.update_or_create(dimension)

    cube = Cube(name="TM1py Cube", dimensions=["TM1py Dimension 1", "TM1py Dimension 2"])
    tm1.cubes.update_or_create(cube)
    print("Created Cube")

    tm1.cells.write_async(cube.name, cells)
    print("Updated cells")


for name, parameters in PARAMETER_SETS.items():
    with TM1Service(**parameters) as tm1:
        setup(tm1)
        for chunk_size in CHUNK_SIZES:
            query = MdxBuilder.from_cube(CUBE)
            query.columns_non_empty()
            query.add_hierarchy_set_to_column_axis(MdxHierarchySet.all_leaves("TM1py Dimension 1").head(chunk_size))
            query.add_hierarchy_set_to_row_axis(MdxHierarchySet.all_leaves("TM1py Dimension 2"))

            before = datetime.now()
            for _ in range(ITERATIONS):
                csv = tm1.cells.execute_mdx_csv(query, use_blob=True)

            print(f"{name}, use_blob=True, {chunk_size}, {(datetime.now() - before) / ITERATIONS}")

            before = datetime.now()
            for _ in range(ITERATIONS):
                csv = tm1.cells.execute_mdx_csv(query, use_blob=False)
            print(f"{name}, 'use_blob=False', {chunk_size}, {(datetime.now() - before) / ITERATIONS}")

[MariusWirtz]

On native cube views use_blob reliably reduces runtime by 40-50% on datasets > 100k.

Environment	use_blob	Number of cells	Runtime	Difference in Seconds2	Difference in %
Linux Server	use_blob=True	100'000	00:02.9	00:01.1	-28%
Linux Server	use_blob=False	100'000	00:04.0
Linux Server	use_blob=True	500'000	00:06.0	00:03.8	-38%
Linux Server	use_blob=False	500'000	00:09.8
Linux Server	use_blob=True	1'000'000	00:10.0	00:07.0	-41%
Linux Server	use_blob=False	1'000'000	00:17.1
Linux Server	use_blob=True	2'500'000	00:22.7	00:18.4	-45%
Linux Server	use_blob=False	2'500'000	00:41.1
Windows Local	use_blob=True	100'000	00:05.3	00:05.1	-49%
Windows Local	use_blob=False	100'000	00:10.4
Windows Local	use_blob=True	500'000	00:15.1	00:15.7	-51%
Windows Local	use_blob=False	500'000	00:30.8
Windows Local	use_blob=True	1'000'000	00:28.3	00:23.6	-46%
Windows Local	use_blob=False	1'000'000	00:51.9
Windows Local	use_blob=True	2'500'000	01:05.0	01:01.2	-48%
Windows Local	use_blob=False	2'500'000	02:06.1

from datetime import datetime

from TM1py.Objects.Subset import AnonymousSubset

from TM1py.Objects.NativeView import NativeView
from mdxpy import MdxBuilder, MdxHierarchySet

from TM1py.Objects.Cube import Cube
from TM1py.Objects.Dimension import Dimension
from TM1py.Objects.Hierarchy import Hierarchy
from TM1py.Services.TM1Service import TM1Service

PARAMETER_SETS = {

    "Linux Server": {
        "address": "10.77.19.30",
        "port": 8010,
        "ssl": True,
        "user": "Admin",
        "password": "apple"
    },

    "Windows Local": {
        "address": "",
        "port": 8010,
        "ssl": True,
        "user": "Admin",
        "password": "apple"
    }
}

CUBE = "TM1py Cube"
VIEW = "TM1py View"
CHUNK_SIZES = [100_000, 500_000, 1_000_000, 2_500_000]
ITERATIONS = 2


def setup(tm1: TM1Service):
    hierarchy = Hierarchy("TM1py Dimension 1", "TM1py Dimension 1")
    cells = dict()
    for e in range(max(CHUNK_SIZES)):
        element = str(e).zfill(7)
        hierarchy.add_element(element, "Numeric")
        cells[element, "Measure"] = e

    dimension = Dimension(name="TM1py Dimension 1", hierarchies=[hierarchy])
    tm1.dimensions.update_or_create(dimension)

    hierarchy = Hierarchy("TM1py Dimension 2", "TM1py Dimension 2")
    hierarchy.add_element("Measure", "Numeric")
    dimension = Dimension(name="TM1py Dimension 2", hierarchies=[hierarchy])
    tm1.dimensions.update_or_create(dimension)

    cube = Cube(name="TM1py Cube", dimensions=["TM1py Dimension 1", "TM1py Dimension 2"])
    tm1.cubes.update_or_create(cube)
    tm1.cells.write_async(cube.name, cells)


for name, parameters in PARAMETER_SETS.items():
    with TM1Service(**parameters) as tm1:
        setup(tm1)
        for chunk_size in CHUNK_SIZES:

            native_view = NativeView(CUBE, view_name=VIEW)
            native_view.add_row(
                "TM1py Dimension 1",
                AnonymousSubset(
                    dimension_name="TM1py Dimension 1",
                    expression=MdxHierarchySet.all_leaves("TM1py Dimension 1").head(chunk_size).to_mdx()))
            native_view.add_column(
                "TM1py Dimension 1",
                AnonymousSubset(
                    dimension_name="TM1py Dimension 2",
                    expression=MdxHierarchySet.all_leaves("TM1py Dimension 2").to_mdx()))
            tm1.views.update_or_create(native_view, private=False)

            before = datetime.now()
            for _ in range(ITERATIONS):
                csv = tm1.cells.execute_view_csv(CUBE, VIEW, use_blob=True)
            print(f"{name}, use_blob=True, {chunk_size}, {(datetime.now() - before) / ITERATIONS}")

            before = datetime.now()
            for _ in range(ITERATIONS):
                csv = tm1.cells.execute_view_csv(CUBE, VIEW, use_blob=False)
            print(f"{name}, 'use_blob=False', {chunk_size}, {(datetime.now() - before) / ITERATIONS}")