From 37b67ded1bd66a39c40017ea8b56f0e196e554c5 Mon Sep 17 00:00:00 2001 From: Andrew Reusch Date: Thu, 10 Jun 2021 13:42:35 -0700 Subject: [PATCH 1/6] Rename runtime-config to executor-config. --- python/tvm/micro/model_library_format.py | 4 ++-- tests/python/unittest/test_micro_model_library_format.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/tvm/micro/model_library_format.py b/python/tvm/micro/model_library_format.py index 1cc3adf9ae07..ea9495b63188 100644 --- a/python/tvm/micro/model_library_format.py +++ b/python/tvm/micro/model_library_format.py @@ -227,7 +227,7 @@ def export_model_library_format(mod: executor_factory.ExecutorFactoryModule, fil runtime = ["aot"] if is_aot else ["graph"] metadata = { - "version": 2, + "version": 3, "model_name": mod.libmod_name, "export_datetime": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%SZ"), "memory": _build_memory_map(mod), @@ -252,7 +252,7 @@ def export_model_library_format(mod: executor_factory.ExecutorFactoryModule, fil f.write(str(mod.ir_mod)) if not is_aot: - graph_config_dir_path = tempdir.relpath(os.path.join("runtime-config", "graph")) + graph_config_dir_path = tempdir.relpath(os.path.join("executor-config", "graph")) os.makedirs(graph_config_dir_path) with open(os.path.join(graph_config_dir_path, "graph.json"), "w") as f: f.write(mod.get_executor_config()) diff --git a/tests/python/unittest/test_micro_model_library_format.py b/tests/python/unittest/test_micro_model_library_format.py index d2c519da22b5..38c17f6cf4da 100644 --- a/tests/python/unittest/test_micro_model_library_format.py +++ b/tests/python/unittest/test_micro_model_library_format.py @@ -33,7 +33,7 @@ def validate_graph_json(extract_dir, factory): - with open(os.path.join(extract_dir, "runtime-config", "graph", "graph.json")) as graph_f: + with open(os.path.join(extract_dir, "executor-config", "graph", "graph.json")) as graph_f: graph_json = graph_f.read() assert graph_json == factory.graph_json From fe65a82878fc11058c43d27ae576c8e962f70c9d Mon Sep 17 00:00:00 2001 From: Andrew Reusch Date: Thu, 10 Jun 2021 13:49:37 -0700 Subject: [PATCH 2/6] Add documentation. --- docs/dev/index.rst | 1 + docs/dev/model_library_format.rst | 167 ++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 docs/dev/model_library_format.rst diff --git a/docs/dev/index.rst b/docs/dev/index.rst index d03f0fb03496..11ef374ffd37 100644 --- a/docs/dev/index.rst +++ b/docs/dev/index.rst @@ -410,3 +410,4 @@ microTVM :maxdepth: 1 microtvm_design + model_library_format diff --git a/docs/dev/model_library_format.rst b/docs/dev/model_library_format.rst new file mode 100644 index 000000000000..362cdb048977 --- /dev/null +++ b/docs/dev/model_library_format.rst @@ -0,0 +1,167 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Model Library Format +==================== + +About Model Library Format +-------------------------- + +TVM traditionally exports generated libraries as Dynamic Shared Objects +(e.g. DLLs (Windows) or .so (linux)). Inference can be performed on those libraries by loading them +into an executable using ``libtvm_runtime.so``. This process is very dependent on services provided +by traditional OS. + +For deployment to unconventional platforms (e.g. those lacking traditional OS), the microTVM project +can be used to export a generated library in pieces. In this case, microTVM provides another output +format, Model Library Format. Model Library Format is a tarball containing a file for each part of +the TVM compiler output. + +What can be Exported +-------------------- + +At the time of writing, export is limited to full models built with ``tvm.relay.build``. + +Directory Layout +---------------- + +Model Library Format is traditionally contained within a tarball. All paths are relative to the root +of the tarball: + +- ``/`` - Root of the tarball + + - ``codegen`` - Root directory for all generated device code + + - (see `codegen`_ section) + + - ``executor-config/`` - Configuration for the executor which drives model inference + + - ``graph/`` - Root directory containing configuration for the GraphExecutor + + - ``graph.json`` - GraphExecutor JSON configuration + + - ``metadata.json`` - Machine-parseable metadata for this model + + - ``parameters/`` - Root directory where simplified parameters are placed + + - ``.params`` - Parameters for the model tvm.relay._save_params format + + - ``src/`` - Root directory for all source code consumed by TVM + + - ``relay.txt`` - Relay source code for the generated model + +Description of Sub-directories +------------------------------ + +.. _subdir_codegen: + +``codegen`` +^^^^^^^^^^^ + +All TVM-generated code is placed in this directory. At the time of writing, there is 1 file per +Module in the generated Module tree, though this restriction may change in the future. Files in +this directory should have filenames of the form ``/(lib|src)/.``. + +These components are described below: + + * ```` - Identifies the TVM target on which the code should run. Currently, only ``host`` + is supported. + * ```` - A unique slug identifying this file. Currently ``lib``, with ``>` an + autoincrementing integer. + * ```` - Suffix identifying the filename format. Currently ``c`` or ``o``. + +An example directory tree for a CPU-only model is shown below: + +- ``codegen/`` - Codegen directory + + - ``host/`` - Generated code for ``target_host`` + + - ``lib/`` - Generated binary object files + + - ``lib0.o`` - LLVM module (if ``llvm`` target is used) + - ``lib1.o`` - LLVM CRT Metadata Module (if ``llvm`` target is used) + - ``src/`` - Generated C source + + - ``lib0.c`` - C module (if ``c`` target is used) + - ``lib1.c`` - C CRT Metadata module (if ``c`` target is used) + +``executor-config`` +^^^^^^^^^^^^^^^^^^^ + +Contains machine-parseable configuration for executors which can drive model inference. Currently, +only the GraphExecutor produces configuration for this directory, in ``graph/graph.json``. This +file should be read in and the resulting string supplied to the ``GraphExecutor()`` constructor for +parsing. + +``parameters`` +^^^^^^^^^^^^^^ + +Contains machine-parseable parameters. A variety of formats may be provided, but at present, only +the format produced by ``tvm.relay._save_params`` is supplied. When building with +``tvm.relay.build``, the ``name`` parameter is considered to be the model name. A single file is +created in this directory ``.json``. + +``src`` +^^^^^^^ + +Contains source code parsed by TVM. Currently, just the Relay source code is created in +``src/relay.txt``. + +Metadata +-------- + +Machine-parseable metadata is placed in a file ``metadata.json`` at the root of the tarball. +Metadata is a dictionary with these keys: + +- ``export_datetime``: Timestamp when this Model Library Format was generated, in + `strftime `_ + format ``"%Y-%M-%d %H:%M:%SZ",``. +- ``memory``: A summary of the memory usage of each generated function. Documented in + `Memory Usage Summary`_. +- ``model_name``: The name of this model (e.g. the ``name`` parameter supplied to + ``tvm.relay.build``). +- ``runtimes``: A list of runtimes supported by this model. Currently, this list is always + ``["graph"]``. +- ``target``: A dictionary mapping ``device_type`` (the underlying integer, as a string) to the + sub-target which describes that relay backend used for that ``device_type``. +- ``version``: A numeric version number that identifies the format used in this Model Library + Format. This number is incremented when the metadata structure or on-disk structure changes. + This document reflects version ``3``. + +Memory Usage Summary +^^^^^^^^^^^^^^^^^^^^ + +A dictionary with these sub-keys: + + - ``"main"``: ``list[MainFunctionWorkspaceUsage]``. A list summarizing memory usage for each + workspace used by the main function and all sub-functions invoked. + - ``"operator_functions"``: ``map[string, list[FunctionWorkspaceUsage]]``. Maps operator function + name to a list summarizing memory usage for each workpace used by the function. + +A ``MainFunctionWorkspaceUsage`` is a dict with these keys: + +- ``"device"``: ``int``. The ``device_type`` associated with this workspace. +- ``"workspace_size_bytes"``: ``int``. Number of bytes needed in this workspace by this function + and all sub-functions invoked. +- ``"constants_size_bytes"``: ``int``. Size of the constants used by the main function. +- ``"io_size_bytes"``: ``int``. Sum of the sizes of the buffers used from this workspace by this + function and sub-functions. + +A ``FunctionWorkspaceUsage`` is a dict with these keys: + +- ``"device"``: ``int``. The ``device_type`` associated with this workspace. +- ``"workspace_size_bytes"``: ``int``. Number of bytes needed in this workspace by this function. From 861e5101e1fc0013e7bb600e5dabff3b1a3a80ac Mon Sep 17 00:00:00 2001 From: Andrew Reusch Date: Mon, 28 Jun 2021 16:42:15 -0700 Subject: [PATCH 3/6] address comments, make tests pass --- docs/dev/model_library_format.rst | 13 ++++++------- .../unittest/test_micro_model_library_format.py | 6 +++--- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/docs/dev/model_library_format.rst b/docs/dev/model_library_format.rst index 362cdb048977..20fce9d587a2 100644 --- a/docs/dev/model_library_format.rst +++ b/docs/dev/model_library_format.rst @@ -21,10 +21,9 @@ Model Library Format About Model Library Format -------------------------- -TVM traditionally exports generated libraries as Dynamic Shared Objects -(e.g. DLLs (Windows) or .so (linux)). Inference can be performed on those libraries by loading them -into an executable using ``libtvm_runtime.so``. This process is very dependent on services provided -by traditional OS. +TVM traditionally exports generated libraries as Dynamic Shared Objects (e.g. DLLs (Windows) or .so +(linux)). Inferences can be performed using those libraries by loading them into an executable using +``libtvm_runtime.so``. This process is very dependent on services provided by traditional OS. For deployment to unconventional platforms (e.g. those lacking traditional OS), the microTVM project can be used to export a generated library in pieces. In this case, microTVM provides another output @@ -81,7 +80,7 @@ These components are described below: * ```` - Identifies the TVM target on which the code should run. Currently, only ``host`` is supported. * ```` - A unique slug identifying this file. Currently ``lib``, with ``>` an - autoincrementing integer. + auto-incrementing integer. * ```` - Suffix identifying the filename format. Currently ``c`` or ``o``. An example directory tree for a CPU-only model is shown below: @@ -102,7 +101,7 @@ An example directory tree for a CPU-only model is shown below: ``executor-config`` ^^^^^^^^^^^^^^^^^^^ -Contains machine-parseable configuration for executors which can drive model inference. Currently, +Contains machine-parsable configuration for executors which can drive model inference. Currently, only the GraphExecutor produces configuration for this directory, in ``graph/graph.json``. This file should be read in and the resulting string supplied to the ``GraphExecutor()`` constructor for parsing. @@ -134,7 +133,7 @@ Metadata is a dictionary with these keys: `Memory Usage Summary`_. - ``model_name``: The name of this model (e.g. the ``name`` parameter supplied to ``tvm.relay.build``). -- ``runtimes``: A list of runtimes supported by this model. Currently, this list is always +- ``executors``: A list of executors supported by this model. Currently, this list is always ``["graph"]``. - ``target``: A dictionary mapping ``device_type`` (the underlying integer, as a string) to the sub-target which describes that relay backend used for that ``device_type``. diff --git a/tests/python/unittest/test_micro_model_library_format.py b/tests/python/unittest/test_micro_model_library_format.py index 38c17f6cf4da..371b1e3dfe63 100644 --- a/tests/python/unittest/test_micro_model_library_format.py +++ b/tests/python/unittest/test_micro_model_library_format.py @@ -85,7 +85,7 @@ def @main(%a : Tensor[(1, 2), uint8], %b : Tensor[(1, 2), float32], %c : Tensor[ with open(os.path.join(extract_dir, "metadata.json")) as json_f: metadata = json.load(json_f) - assert metadata["version"] == 2 + assert metadata["version"] == 3 assert metadata["model_name"] == "add" export_datetime = datetime.datetime.strptime( metadata["export_datetime"], "%Y-%m-%d %H:%M:%SZ" @@ -165,7 +165,7 @@ def @main(%a : Tensor[(1, 2), uint8], %b : Tensor[(1, 2), float32], %c : Tensor[ with open(os.path.join(extract_dir, "metadata.json")) as json_f: metadata = json.load(json_f) - assert metadata["version"] == 2 + assert metadata["version"] == 3 assert metadata["model_name"] == "add" export_datetime = datetime.datetime.strptime( metadata["export_datetime"], "%Y-%m-%d %H:%M:%SZ" @@ -244,7 +244,7 @@ def @main(%p0: Tensor[(1, 56, 56, 128), int16], %p1: Tensor[(3, 3, 128, 1), int1 with open(os.path.join(extract_dir, "metadata.json")) as json_f: metadata = json.load(json_f) - assert metadata["version"] == 2 + assert metadata["version"] == 3 assert metadata["model_name"] == "qnn_conv2d" export_datetime = datetime.datetime.strptime( metadata["export_datetime"], "%Y-%m-%d %H:%M:%SZ" From d529ab970c69002180fff358acda8b59711910aa Mon Sep 17 00:00:00 2001 From: Andrew Reusch Date: Fri, 9 Jul 2021 09:01:07 -0700 Subject: [PATCH 4/6] fix unit test --- python/tvm/driver/tvmc/model.py | 4 ++-- tests/python/driver/tvmc/test_mlf.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/python/tvm/driver/tvmc/model.py b/python/tvm/driver/tvmc/model.py index 8c8828ddd49b..7dc3fd4cdd36 100644 --- a/python/tvm/driver/tvmc/model.py +++ b/python/tvm/driver/tvmc/model.py @@ -336,8 +336,8 @@ def import_package(self, package_path: str): with open(temp.relpath("metadata.json")) as metadata_json: metadata = json.load(metadata_json) - is_graph_runtime = "graph" in metadata["runtimes"] - graph = temp.relpath("runtime-config/graph/graph.json") if is_graph_runtime else None + has_graph_executor = "graph" in metadata["executors"] + graph = temp.relpath("executor-config/graph/graph.json") if has_graph_executor else None params = temp.relpath("parameters/default.params") self.type = "mlf" diff --git a/tests/python/driver/tvmc/test_mlf.py b/tests/python/driver/tvmc/test_mlf.py index 4669fab916a6..0426f5678153 100644 --- a/tests/python/driver/tvmc/test_mlf.py +++ b/tests/python/driver/tvmc/test_mlf.py @@ -18,6 +18,7 @@ import pytest import os import shlex +import sys import tvm from tvm.driver import tvmc @@ -130,3 +131,7 @@ def test_tvmc_import_package_mlf_aot(tflite_mobilenet_v1_1_quant, tflite_compile assert tvmc_package.graph is None, ".graph must not be set in the MLF archive for AOT executor." assert tvmc_package.params is not None, ".params must be set in the MLF archive." assert tvmc_package.type == "mlf", ".type must be set to 'mlf' in the MLF format." + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__] + sys.argv[1:])) From a0211d9dc423c2a2cb40f7f8e22193e646f7d56b Mon Sep 17 00:00:00 2001 From: Andrew Reusch Date: Fri, 9 Jul 2021 12:13:15 -0700 Subject: [PATCH 5/6] fix sphinx doc errors --- docs/dev/model_library_format.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/dev/model_library_format.rst b/docs/dev/model_library_format.rst index 20fce9d587a2..883c6a666b08 100644 --- a/docs/dev/model_library_format.rst +++ b/docs/dev/model_library_format.rst @@ -79,7 +79,7 @@ These components are described below: * ```` - Identifies the TVM target on which the code should run. Currently, only ``host`` is supported. - * ```` - A unique slug identifying this file. Currently ``lib``, with ``>` an + * ```` - A unique slug identifying this file. Currently ``lib``, with ``>`` an auto-incrementing integer. * ```` - Suffix identifying the filename format. Currently ``c`` or ``o``. @@ -93,6 +93,7 @@ An example directory tree for a CPU-only model is shown below: - ``lib0.o`` - LLVM module (if ``llvm`` target is used) - ``lib1.o`` - LLVM CRT Metadata Module (if ``llvm`` target is used) + - ``src/`` - Generated C source - ``lib0.c`` - C module (if ``c`` target is used) From 5269289fa4ec8e7cd383e0c23fb6e60d14099de0 Mon Sep 17 00:00:00 2001 From: Andrew Reusch Date: Thu, 15 Jul 2021 13:03:27 -0700 Subject: [PATCH 6/6] address manupa comments --- docs/dev/model_library_format.rst | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/dev/model_library_format.rst b/docs/dev/model_library_format.rst index 883c6a666b08..fec90de4bcea 100644 --- a/docs/dev/model_library_format.rst +++ b/docs/dev/model_library_format.rst @@ -25,21 +25,23 @@ TVM traditionally exports generated libraries as Dynamic Shared Objects (e.g. DL (linux)). Inferences can be performed using those libraries by loading them into an executable using ``libtvm_runtime.so``. This process is very dependent on services provided by traditional OS. -For deployment to unconventional platforms (e.g. those lacking traditional OS), the microTVM project -can be used to export a generated library in pieces. In this case, microTVM provides another output -format, Model Library Format. Model Library Format is a tarball containing a file for each part of -the TVM compiler output. +For deployment to unconventional platforms (e.g. those lacking traditional OS), TVM provides another +output format, Model Library Format. Initially, the microTVM project is the primary use case for this +format. Should it become useful in other use cases (and in particular, should it become possible to +export BYOC artifacts in Model Library Format), it could be used as a general-purpose TVM export +format. Model Library Format is a tarball containing a file for each piece of the TVM compiler +output. -What can be Exported --------------------- +What can be Exported? +--------------------- At the time of writing, export is limited to full models built with ``tvm.relay.build``. Directory Layout ---------------- -Model Library Format is traditionally contained within a tarball. All paths are relative to the root -of the tarball: +Model Library Format is contained within a tarball. All paths are relative to the root of the +tarball: - ``/`` - Root of the tarball @@ -140,7 +142,7 @@ Metadata is a dictionary with these keys: sub-target which describes that relay backend used for that ``device_type``. - ``version``: A numeric version number that identifies the format used in this Model Library Format. This number is incremented when the metadata structure or on-disk structure changes. - This document reflects version ``3``. + This document reflects version ``5``. Memory Usage Summary ^^^^^^^^^^^^^^^^^^^^