From 4cd58a38707ca1f2bae482edf676e5b3230a77df Mon Sep 17 00:00:00 2001 From: Deyu Huang Date: Tue, 2 Aug 2022 16:14:56 +0800 Subject: [PATCH 1/3] Remove opset below to 13 ci tests and enhance doc Signed-off-by: Deyu Huang --- README.md | 11 +++++------ .../azure_pipelines/templates/job_generator.yml | 2 +- ci_build/azure_pipelines/templates/unit_test.yml | 2 +- ci_build/azure_pipelines/unit_test-matrix.yml | 13 +------------ ci_build/azure_pipelines/unit_test.yml | 6 +++--- 5 files changed, 11 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index d94f663a7..912f3101f 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ The common issues we run into we try to document here [Troubleshooting Guide](Tr | Build Type | OS | Python | TensorFlow | ONNX opset | Status | | --- | --- | --- | --- | --- | --- | -| Unit Test - Basic | Linux, MacOS\*, Windows\* | 3.7-3.10 | 1.13-1.15, 2.1-2.9 | 9-16 | [![Build Status](https://dev.azure.com/tensorflow-onnx/tensorflow-onnx/_apis/build/status/unit_test?branchName=main)](https://dev.azure.com/tensorflow-onnx/tensorflow-onnx/_build/latest?definitionId=16&branchName=main) | -| Unit Test - Full | Linux, MacOS, Windows | 3.7-3.10 | 1.13-1.15, 2.1-2.9 | 9-16 | [![Build Status](https://dev.azure.com/tensorflow-onnx/tensorflow-onnx/_apis/build/status/unit_test-matrix?branchName=main)](https://dev.azure.com/tensorflow-onnx/tensorflow-onnx/_build/latest?definitionId=18&branchName=main) | | +| Unit Test - Basic | Linux, MacOS\*, Windows\* | 3.7-3.10 | 1.13-1.15, 2.1-2.9 | 13-17 | [![Build Status](https://dev.azure.com/tensorflow-onnx/tensorflow-onnx/_apis/build/status/unit_test?branchName=main)](https://dev.azure.com/tensorflow-onnx/tensorflow-onnx/_build/latest?definitionId=16&branchName=main) | +| Unit Test - Full | Linux, MacOS, Windows | 3.7-3.10 | 1.13-1.15, 2.1-2.9 | 13-17 | [![Build Status](https://dev.azure.com/tensorflow-onnx/tensorflow-onnx/_apis/build/status/unit_test-matrix?branchName=main)](https://dev.azure.com/tensorflow-onnx/tensorflow-onnx/_build/latest?definitionId=18&branchName=main) | |
## Supported Versions @@ -27,7 +27,7 @@ The common issues we run into we try to document here [Troubleshooting Guide](Tr tf2onnx will use the ONNX version installed on your system and installs the latest ONNX version if none is found. -We support and test ONNX opset-9 to opset-17. opset-6 to opset-8 should work but we don't test them. +We support and test ONNX opset-13 to opset-17. opset-6 to opset-12 should work but we don't test them. By default we use ```opset-13``` for the resulting ONNX graph. If you want the graph to be generated with a specific opset, use ```--opset``` in the command line, for example ```--opset 13```. @@ -43,7 +43,6 @@ You can install tf2onnx on top of tf-1.x or tf-2.x. ### Python We support Python ```3.7-3.10```. -Note that on windows for Python > 3.7 the protobuf package doesn't use the cpp implementation and is very slow - we recommend to use Python 3.7 for that reason. ## Prerequisites @@ -83,7 +82,7 @@ or ```python setup.py develop``` -tensorflow-onnx requires onnx-1.5 or better and will install/upgrade onnx if needed. +tensorflow-onnx requires onnx-1.9 or better and will install/upgrade onnx if needed. To create a wheel for distribution: @@ -100,7 +99,7 @@ To get started with `tensorflow-onnx`, run the `t2onnx.convert` command, providi The above command uses a default of `13` for the ONNX opset. If you need a newer opset, or want to limit your model to use an older opset then you can provide the `--opset` argument to the command. If you are unsure about which opset to use, refer to the [ONNX operator documentation](https://github.com/onnx/onnx/releases). -```python -m tf2onnx.convert --saved-model tensorflow-model-path --opset 16 --output model.onnx``` +```python -m tf2onnx.convert --saved-model tensorflow-model-path --opset 17 --output model.onnx``` If your TensorFlow model is in a format other than `saved model`, then you need to provide the inputs and outputs of the model graph. diff --git a/ci_build/azure_pipelines/templates/job_generator.yml b/ci_build/azure_pipelines/templates/job_generator.yml index 1f66aa12d..dcbc08e79 100644 --- a/ci_build/azure_pipelines/templates/job_generator.yml +++ b/ci_build/azure_pipelines/templates/job_generator.yml @@ -5,7 +5,7 @@ parameters: python_versions: ['3.7'] tf_versions: [''] onnx_versions: [''] - onnx_opsets: ['17', '16', '15', '14', '13', '12', '11', '10', '9'] + onnx_opsets: ['17', '16', '15', '14', '13'] onnx_backends: {onnxruntime: ['1.12.0']} job: {} run_setup: 'True' diff --git a/ci_build/azure_pipelines/templates/unit_test.yml b/ci_build/azure_pipelines/templates/unit_test.yml index ff39fc56a..9eb444665 100644 --- a/ci_build/azure_pipelines/templates/unit_test.yml +++ b/ci_build/azure_pipelines/templates/unit_test.yml @@ -1,7 +1,7 @@ # Run unit test parameters: - onnx_opsets: ['17', '16', '15', '14', '13', '12', '11', '10', '9'] + onnx_opsets: ['17', '16', '15', '14', '13'] skip_tflite_tests: 'True' skip_tfjs_tests: 'True' skip_tf_tests: 'False' diff --git a/ci_build/azure_pipelines/unit_test-matrix.yml b/ci_build/azure_pipelines/unit_test-matrix.yml index 8ff0ca47d..664d33169 100644 --- a/ci_build/azure_pipelines/unit_test-matrix.yml +++ b/ci_build/azure_pipelines/unit_test-matrix.yml @@ -7,18 +7,7 @@ stages: parameters: platforms: ['linux', 'windows'] python_versions: ['3.7'] - tf_versions: ['1.14.0'] - onnx_opsets: [''] - job: - steps: - - template: 'unit_test.yml' - report_coverage: 'True' - - - template: 'templates/job_generator.yml' - parameters: - platforms: ['linux', 'windows'] - python_versions: ['3.7'] - tf_versions: ['1.15.2','2.1.0'] + tf_versions: ['1.14.0', '1.15.2'] onnx_opsets: [''] job: steps: diff --git a/ci_build/azure_pipelines/unit_test.yml b/ci_build/azure_pipelines/unit_test.yml index 47ebf15ed..94b3e3e72 100644 --- a/ci_build/azure_pipelines/unit_test.yml +++ b/ci_build/azure_pipelines/unit_test.yml @@ -80,7 +80,7 @@ stages: - template: 'templates/job_generator.yml' parameters: # TFJS tf 2.9 - python_versions: ['3.9'] + python_versions: ['3.10'] tf_versions: ['2.9.1'] onnx_opsets: [''] skip_tfjs_tests: 'False' @@ -93,7 +93,7 @@ stages: - template: 'templates/job_generator.yml' parameters: # TFLite tf 2.9 - python_versions: ['3.8'] + python_versions: ['3.10'] tf_versions: ['2.9.1'] onnx_opsets: [''] skip_tflite_tests: 'False' @@ -162,7 +162,7 @@ stages: python_versions: ['3.9'] platforms: ['windows'] tf_versions: ['2.9.1'] - onnx_opsets: ['15'] + onnx_opsets: ['16'] job: steps: - template: 'unit_test.yml' From ed347ab235852a6798964415e2ee81eca4b8cf68 Mon Sep 17 00:00:00 2001 From: Deyu Huang Date: Wed, 3 Aug 2022 10:42:38 +0800 Subject: [PATCH 2/3] Remove usage of numpy bool aliases for builtins Signed-off-by: Deyu Huang --- tests/test_backend.py | 38 ++++++++++++++--------------- tests/test_onnx_shape_inference.py | 4 +-- tests/test_optimizers.py | 6 ++--- tests/test_tfjs_runner.py | 2 +- tf2onnx/custom_opsets/string_ops.py | 2 +- tf2onnx/onnx_opset/nn.py | 8 +++--- tf2onnx/onnx_opset/tensor.py | 8 +++--- tf2onnx/utils.py | 2 +- 8 files changed, 35 insertions(+), 35 deletions(-) diff --git a/tests/test_backend.py b/tests/test_backend.py index cdbfeebc2..9fb6cedad 100755 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -5,7 +5,7 @@ import os import unittest -from distutils.version import LooseVersion +from packaging.version import Version from itertools import product import numpy as np @@ -72,7 +72,7 @@ matrix_diag_part = tf.compat.v1.matrix_diag_part fake_quant_with_min_max_args = tf.quantization.fake_quant_with_min_max_args fake_quant_with_min_max_vars = tf.quantization.fake_quant_with_min_max_vars -elif LooseVersion(tf.__version__) >= "1.13": +elif Version(tf.__version__) >= Version("1.13"): conv2d_backprop_input = tf.compat.v1.nn.conv2d_backprop_input conv3d_transpose = tf.compat.v1.nn.conv3d_transpose multinomial = tf.compat.v1.random.multinomial @@ -86,7 +86,7 @@ quantize_and_dequantize = tf.compat.v1.quantization.quantize_and_dequantize resize_nearest_neighbor = tf.compat.v1.image.resize_nearest_neighbor resize_bilinear = tf.compat.v1.image.resize_bilinear - if LooseVersion(tf.__version__) >= "1.14": + if Version(tf.__version__) >= Version("1.14"): resize_bilinear_v2 = tf.compat.v2.image.resize is_nan = tf.math.is_nan is_inf = tf.math.is_inf @@ -1320,8 +1320,8 @@ def func(x1): @check_onnxruntime_incompatibility("Add") def test_logicaland(self): - x_val1 = np.array([1, 0, 1, 1], dtype=np.bool).reshape((2, 2)) - x_val2 = np.array([0, 1, 1, 1], dtype=np.bool).reshape((2, 2)) + x_val1 = np.array([1, 0, 1, 1], dtype=bool).reshape((2, 2)) + x_val2 = np.array([0, 1, 1, 1], dtype=bool).reshape((2, 2)) def func(x1, x2): mi = tf.logical_and(x1, x2) return tf.identity(mi, name=_TFOUTPUT) @@ -3505,9 +3505,9 @@ def func(x): def test_where_bool(self): x_val = np.array([1, 2, -3, 4, -5], dtype=np.float32) true_result = np.array([True, False, True, False, True], - dtype=np.bool) + dtype=bool) false_result = np.array([False, True, False, True, True], - dtype=np.bool) + dtype=bool) def func(x): picks = tf.where(x > -1, true_result, false_result) return tf.identity(picks, name=_TFOUTPUT) @@ -3770,21 +3770,21 @@ def func(input_1, input_2): self._run_test_case(func, [_OUTPUT], {_INPUT: input_val_1, _INPUT1: input_val_2}, rtol=1e-4) def test_logical_not(self): - input_val = np.random.randint(0, 2, (10, 20)).astype(np.bool) + input_val = np.random.randint(0, 2, (10, 20)).astype(bool) def func(x): res = tf.logical_not(x) return tf.identity(res, name=_TFOUTPUT) self._run_test_case(func, [_OUTPUT], {_INPUT: input_val}) def test_reduce_all(self): - input_val = np.random.randint(0, 2, (2, 20)).astype(np.bool) + input_val = np.random.randint(0, 2, (2, 20)).astype(bool) def func(x): res = tf.reduce_all(input_tensor=x, keepdims=False) res1 = tf.reduce_all(input_tensor=x, axis=[0], keepdims=False) return tf.identity(res, name=_TFOUTPUT), tf.identity(res1, name=_TFOUTPUT1) self._run_test_case(func, [_OUTPUT, _OUTPUT1], {_INPUT: input_val}) - input_val = np.random.randint(0, 2, (2, 20)).astype(np.bool) + input_val = np.random.randint(0, 2, (2, 20)).astype(bool) def func(input_x): res = tf.reduce_all(input_tensor=input_x, keepdims=True) res1 = tf.reduce_all(input_tensor=input_x, axis=[0], keepdims=True) @@ -3792,14 +3792,14 @@ def func(input_x): self._run_test_case(func, [_OUTPUT, _OUTPUT1], {_INPUT: input_val}) def test_reduce_any(self): - input_val = np.random.randint(0, 2, (2, 20)).astype(np.bool) + input_val = np.random.randint(0, 2, (2, 20)).astype(bool) def func(x): res = tf.reduce_any(input_tensor=x, keepdims=False) res1 = tf.reduce_any(input_tensor=x, axis=[0], keepdims=False) return tf.identity(res, name=_TFOUTPUT), tf.identity(res1, name=_TFOUTPUT1) self._run_test_case(func, [_OUTPUT, _OUTPUT1], {_INPUT: input_val}) - input_val = np.random.randint(0, 2, (2, 20)).astype(np.bool) + input_val = np.random.randint(0, 2, (2, 20)).astype(bool) def func(x): res = tf.reduce_any(input_tensor=x, keepdims=True) res1 = tf.reduce_any(input_tensor=x, axis=[0], keepdims=True) @@ -3808,14 +3808,14 @@ def func(x): @check_opset_min_version(11, "ReduceMin") def test_reduce_all_negative_axis(self): - input_val = np.random.randint(0, 2, (2, 20)).astype(np.bool) + input_val = np.random.randint(0, 2, (2, 20)).astype(bool) def func(x): res = tf.reduce_all(input_tensor=x, keepdims=False) res1 = tf.reduce_all(input_tensor=x, axis=[-1], keepdims=False) return tf.identity(res, name=_TFOUTPUT), tf.identity(res1, name=_TFOUTPUT1) self._run_test_case(func, [_OUTPUT, _OUTPUT1], {_INPUT: input_val}) - input_val = np.random.randint(0, 2, (2, 20)).astype(np.bool) + input_val = np.random.randint(0, 2, (2, 20)).astype(bool) def func(input_x): res = tf.reduce_all(input_tensor=input_x, keepdims=True) res1 = tf.reduce_all(input_tensor=input_x, axis=[-1], keepdims=True) @@ -3824,14 +3824,14 @@ def func(input_x): @check_opset_min_version(11, "ReduceSum") def test_reduce_any_negative_axis(self): - input_val = np.random.randint(0, 2, (2, 20)).astype(np.bool) + input_val = np.random.randint(0, 2, (2, 20)).astype(bool) def func(x): res = tf.reduce_any(input_tensor=x, keepdims=False) res1 = tf.reduce_any(input_tensor=x, axis=[-1], keepdims=False) return tf.identity(res, name=_TFOUTPUT), tf.identity(res1, name=_TFOUTPUT1) self._run_test_case(func, [_OUTPUT, _OUTPUT1], {_INPUT: input_val}) - input_val = np.random.randint(0, 2, (2, 20)).astype(np.bool) + input_val = np.random.randint(0, 2, (2, 20)).astype(bool) def func(x): res = tf.reduce_any(input_tensor=x, keepdims=True) res1 = tf.reduce_any(input_tensor=x, axis=[-1], keepdims=True) @@ -3841,7 +3841,7 @@ def func(x): @check_opset_min_version(11, "ReduceSum") @check_tf_min_version("1.15") def test_reduce_any_empty_axis(self): - input_val = np.random.randint(0, 2, (2, 20)).astype(np.bool) + input_val = np.random.randint(0, 2, (2, 20)).astype(bool) def func(x): res = tf.reduce_any(input_tensor=x, keepdims=False) res1 = tf.reduce_any(input_tensor=x, axis=[], keepdims=False) @@ -3849,7 +3849,7 @@ def func(x): self._run_test_case(func, [_OUTPUT, _OUTPUT1], {_INPUT: input_val}) def test_reduce_all_scalar_axis(self): - input_val = np.random.randint(0, 2, (2, 20)).astype(np.bool) + input_val = np.random.randint(0, 2, (2, 20)).astype(bool) def func(x): res = tf.reduce_all(input_tensor=x, keepdims=False) res1 = tf.reduce_all(input_tensor=x, axis=0, keepdims=False) @@ -3859,7 +3859,7 @@ def func(x): @check_opset_min_version(13, "ReduceSum") @check_tf_min_version("1.15") def test_reduce_any_nonconst_axis(self): - input_val = np.random.randint(0, 2, (2, 20)).astype(np.bool) + input_val = np.random.randint(0, 2, (2, 20)).astype(bool) y_val = np.array([1], np.int32) def func(x, y): res = tf.reduce_any(input_tensor=x, axis=y, keepdims=False) diff --git a/tests/test_onnx_shape_inference.py b/tests/test_onnx_shape_inference.py index 6272fa2ff..244e1b6bb 100644 --- a/tests/test_onnx_shape_inference.py +++ b/tests/test_onnx_shape_inference.py @@ -353,7 +353,7 @@ def test_if(self): sub = else_subgraph.make_node("Sub", [INPUT1, INPUT3]) else_subgraph.add_graph_output(sub.output[0]) - cond = graph.make_const("cond", np.array(True, dtype=np.bool)) + cond = graph.make_const("cond", np.array(True, dtype=bool)) branches = {"then_branch": then_subgraph, "else_branch": else_subgraph} if_node = graph.make_node("If", [cond.output[0]], branches=branches) @@ -381,7 +381,7 @@ def test_loop(self): subgraph.add_graph_output(out.output[0]) max_iter = graph.make_const("max_iter", np.array([10], dtype=np.int64)) - cond_const = graph.make_const("cond_const", np.array([True], dtype=np.bool)) + cond_const = graph.make_const("cond_const", np.array([True], dtype=bool)) branches = {"body": subgraph} loop = graph.make_node("Loop", [max_iter.output[0], cond_const.output[0], INPUT1], output_count=2, branches=branches) diff --git a/tests/test_optimizers.py b/tests/test_optimizers.py index 8261d083f..2495bab58 100644 --- a/tests/test_optimizers.py +++ b/tests/test_optimizers.py @@ -988,7 +988,7 @@ def _define_loop_graph(external_inputs): def _make_loop(external_inputs, outputs): trip_cnt = self._make_onnx_const(np.array(10, dtype=np.int64), "trip_cnt") - cond = self._make_onnx_const(np.array(True, dtype=np.bool), "cond") + cond = self._make_onnx_const(np.array(True, dtype=bool), "cond") sub_graph = _define_loop_graph(external_inputs) loop_node = helper.make_node("Loop", ["trip_cnt", "cond", "cond"], outputs, name="loop", body=sub_graph) @@ -1779,7 +1779,7 @@ def test_identity_in_subgraph_non_graph_output(self): ), ) - cond_value = np.array(True, dtype=np.bool) + cond_value = np.array(True, dtype=bool) node3 = helper.make_node( 'Constant', inputs=[], @@ -1788,7 +1788,7 @@ def test_identity_in_subgraph_non_graph_output(self): name='cond_value', data_type=TensorProto.BOOL, dims=iter_num_value.shape, - vals=cond_value.flatten().astype(np.bool).tolist(), + vals=cond_value.flatten().astype(bool).tolist(), ), ) diff --git a/tests/test_tfjs_runner.py b/tests/test_tfjs_runner.py index b0a2fedba..0f282a4d2 100644 --- a/tests/test_tfjs_runner.py +++ b/tests/test_tfjs_runner.py @@ -17,7 +17,7 @@ class TestTfjsRunner(unittest.TestCase): def test_tfjs_runner(self): float_array = np.array([[1.1, 2.2], [3.3, 4.4]], np.float32) int_array = np.array([[1, 2], [3, 4]], np.int32) - bool_array = np.array([[True, False], [True, True]], np.bool) + bool_array = np.array([[True, False], [True, True]], bool) string_array = np.array([['Hello world', ''], ['π', 'Tensor']], np.str) complex_array = np.array([[1 + 0.1j, 2 + 0.2j], [3 + 0.3j, 4 + 0.4j]], np.complex64) diff --git a/tf2onnx/custom_opsets/string_ops.py b/tf2onnx/custom_opsets/string_ops.py index 31fb0e363..303fcd94b 100644 --- a/tf2onnx/custom_opsets/string_ops.py +++ b/tf2onnx/custom_opsets/string_ops.py @@ -30,7 +30,7 @@ def version_1(cls, ctx, node, **kwargs): del node.attr[a] unsqueeze_node = GraphBuilder(ctx).make_unsqueeze({'data': node.input[1], 'axes': [0]}, return_node=True) - skip_empty_const = ctx.make_const(utils.make_name('skip_empty_const'), np.array([skip_empty], np.bool)) + skip_empty_const = ctx.make_const(utils.make_name('skip_empty_const'), np.array([skip_empty], bool)) ctx.replace_inputs(node, [node.input[0], unsqueeze_node.output[0], skip_empty_const.output[0]]) @tf_op("StringToHashBucketFast", domain=constants.CONTRIB_OPS_DOMAIN) diff --git a/tf2onnx/onnx_opset/nn.py b/tf2onnx/onnx_opset/nn.py index 5ff8567d3..adde19641 100644 --- a/tf2onnx/onnx_opset/nn.py +++ b/tf2onnx/onnx_opset/nn.py @@ -853,7 +853,7 @@ def convert_symmetric_pads(cls, ctx, node): output = node.output[0] shape = ctx.make_node("Shape", [output]).output[0] dims = ctx.make_node("Split", [shape], output_count=rank).output - two_false = ctx.make_const(utils.make_name("two_false"), np.array([False, False], np.bool)).output[0] + two_false = ctx.make_const(utils.make_name("two_false"), np.array([False, False], bool)).output[0] inv_second = ctx.make_const(utils.make_name("inv_second"), np.array([1, -1], np.int64)).output[0] dec_second = ctx.make_const(utils.make_name("dec_second"), np.array([0, 1], np.int64)).output[0] for a in non_zero_axes: @@ -1325,7 +1325,7 @@ def any_version_after11(cls, opset, ctx, node, **kwargs): g.add_graph_output(cond_out_name, TensorProto.BOOL, []) g.add_graph_output(squeeze_x.output[0], ctx.get_dtype(node.input[0]), [-1, -1, -1]) trip_node = ctx.make_node("Size", [box_ind]) - cond_const = ctx.make_const(utils.make_name("cond"), np.ones((), dtype=np.bool)) + cond_const = ctx.make_const(utils.make_name("cond"), np.ones((), dtype=bool)) ctx.remove_node(node.name) branches = {"body": g} inner_loop = ctx.make_node("Loop", [trip_node.output[0], cond_const.output[0]], name=node.name, @@ -1638,7 +1638,7 @@ def version_7(cls, ctx, node, **kwargs): # 2: "loop" to generate mask matrix: generate col or row of matrix one by one g = ctx.create_new_graph_with_same_config() node_name = utils.make_name("const_zero_bool") - const_zero_bool = g.make_const(name=node_name, np_val=np.array([[0]]).astype(np.bool)) + const_zero_bool = g.make_const(name=node_name, np_val=np.array([[0]]).astype(bool)) g.set_dtype(const_zero_bool.output[0], onnx_pb.TensorProto.BOOL) g.add_graph_input("trip", onnx_pb.TensorProto.INT64, []) @@ -1668,7 +1668,7 @@ def version_7(cls, ctx, node, **kwargs): line_num = ctx.make_node(op_type="Gather", inputs=[shape.output[0], col_or_row_num_index.output[0]]) trip_cnt = line_num.output[0] node_name = utils.make_name("true") - cond = ctx.make_const(name=node_name, np_val=np.array(1).astype(np.bool)) + cond = ctx.make_const(name=node_name, np_val=np.array(1).astype(bool)) col_init = one_line.output[0] branches = {"body": g} diff --git a/tf2onnx/onnx_opset/tensor.py b/tf2onnx/onnx_opset/tensor.py index b08188b88..044a253d4 100644 --- a/tf2onnx/onnx_opset/tensor.py +++ b/tf2onnx/onnx_opset/tensor.py @@ -497,7 +497,7 @@ def _make_gathernd_inner_loop(ctx, params, index, dtype): # gather_res = gather(gather_cur, index[i]) scope_name = utils.make_name("gathernd_inner_loop") trip_node = ctx.make_node("Size", [index.output[0]]) - cond_const = ctx.make_const(utils.make_name("cond"), np.ones((), dtype=np.bool)) + cond_const = ctx.make_const(utils.make_name("cond"), np.ones((), dtype=bool)) trip_name = utils.make_name("i") cond_name = utils.make_name("cond") cond_out_name = utils.make_name("cond_out") @@ -548,7 +548,7 @@ def make_gathernd(ctx, params, indices, output, scope_name, t_params, shapes, dt # outter loop for each index # for (int i=0; i Date: Wed, 3 Aug 2022 10:45:42 +0800 Subject: [PATCH 3/3] Use packaging library to avoid DeprecationWarning from distutils Signed-off-by: Deyu Huang --- tests/common.py | 16 ++++++++-------- tests/run_pretrained_models.py | 4 ++-- tests/test_backend.py | 2 +- tf2onnx/convert.py | 8 ++++---- tf2onnx/shape_inference.py | 4 ++-- tf2onnx/tf_loader.py | 14 +++++++------- tf2onnx/tf_utils.py | 4 ++-- 7 files changed, 26 insertions(+), 26 deletions(-) diff --git a/tests/common.py b/tests/common.py index f92fa1f51..82ca09c49 100644 --- a/tests/common.py +++ b/tests/common.py @@ -9,7 +9,7 @@ import unittest from collections import defaultdict -from distutils.version import LooseVersion +from packaging.version import Version from parameterized import parameterized import numpy as np import tensorflow as tf @@ -98,7 +98,7 @@ def _get_backend_version(self): pass if version: - version = LooseVersion(version) + version = Version(version) return version def __str__(self): @@ -178,7 +178,7 @@ def check_opset_after_tf_version(tf_version, required_opset, message=""): """ Skip if tf_version > max_required_version """ config = get_test_config() reason = _append_message("conversion requires opset {} after tf {}".format(required_opset, tf_version), message) - skip = config.tf_version >= LooseVersion(tf_version) and config.opset < required_opset + skip = config.tf_version >= Version(tf_version) and config.opset < required_opset return unittest.skipIf(skip, reason) @@ -284,7 +284,7 @@ def check_tfjs_max_version(max_accepted_version, message=""): except ModuleNotFoundError: can_import = False return unittest.skipIf(can_import and not config.skip_tfjs_tests and \ - tensorflowjs.__version__ > LooseVersion(max_accepted_version), reason) + Version(tensorflowjs.__version__) > Version(max_accepted_version), reason) def check_tfjs_min_version(min_required_version, message=""): """ Skip if tjs_version < min_required_version """ @@ -296,20 +296,20 @@ def check_tfjs_min_version(min_required_version, message=""): except ModuleNotFoundError: can_import = False return unittest.skipIf(can_import and not config.skip_tfjs_tests and \ - tensorflowjs.__version__ < LooseVersion(min_required_version), reason) + Version(tensorflowjs.__version__) < Version(min_required_version), reason) def check_tf_max_version(max_accepted_version, message=""): """ Skip if tf_version > max_required_version """ config = get_test_config() reason = _append_message("conversion requires tf <= {}".format(max_accepted_version), message) - return unittest.skipIf(config.tf_version > LooseVersion(max_accepted_version), reason) + return unittest.skipIf(config.tf_version > Version(max_accepted_version), reason) def check_tf_min_version(min_required_version, message=""): """ Skip if tf_version < min_required_version """ config = get_test_config() reason = _append_message("conversion requires tf >= {}".format(min_required_version), message) - return unittest.skipIf(config.tf_version < LooseVersion(min_required_version), reason) + return unittest.skipIf(config.tf_version < Version(min_required_version), reason) def skip_tf_versions(excluded_versions, message=""): @@ -385,7 +385,7 @@ def check_onnxruntime_min_version(min_required_version, message=""): config = get_test_config() reason = _append_message("conversion requires onnxruntime >= {}".format(min_required_version), message) return unittest.skipIf(config.is_onnxruntime_backend and - config.backend_version < LooseVersion(min_required_version), reason) + config.backend_version < Version(min_required_version), reason) def skip_caffe2_backend(message=""): diff --git a/tests/run_pretrained_models.py b/tests/run_pretrained_models.py index 78f36a79c..626cba0d5 100644 --- a/tests/run_pretrained_models.py +++ b/tests/run_pretrained_models.py @@ -17,7 +17,7 @@ import zipfile import random from collections import namedtuple -from distutils.version import LooseVersion +from packaging.version import Version import yaml @@ -789,7 +789,7 @@ def main(): continue if t.tf_min_version: - if tf_utils.get_tf_version() < LooseVersion(str(t.tf_min_version)): + if tf_utils.get_tf_version() < Version(str(t.tf_min_version)): logger.info("Skip %s: %s %s", test, "Min TF version needed:", t.tf_min_version) continue diff --git a/tests/test_backend.py b/tests/test_backend.py index 9fb6cedad..8876da855 100755 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -5,11 +5,11 @@ import os import unittest -from packaging.version import Version from itertools import product import numpy as np from numpy.testing import assert_almost_equal +from packaging.version import Version import tensorflow as tf from tensorflow.python.ops import lookup_ops diff --git a/tf2onnx/convert.py b/tf2onnx/convert.py index 32c28f0bc..02a78e439 100644 --- a/tf2onnx/convert.py +++ b/tf2onnx/convert.py @@ -10,7 +10,7 @@ import argparse import os import sys -from distutils.version import LooseVersion +from packaging.version import Version os.environ['TF_CPP_MIN_LOG_LEVEL'] = "3" @@ -20,7 +20,7 @@ from tf2onnx import constants, logging, utils, optimizer from tf2onnx import tf_loader from tf2onnx.graph import ExternalTensorStorage -from tf2onnx.tf_utils import compress_graph_def +from tf2onnx.tf_utils import compress_graph_def, get_tf_version @@ -431,7 +431,7 @@ def from_keras(model, input_signature=None, opset=None, custom_ops=None, custom_ Returns: An ONNX model_proto and an external_tensor_storage dict. """ - if LooseVersion(tf.__version__) < "2.0": + if get_tf_version() < Version("2.0"): return _from_keras_tf1(model, opset, custom_ops, custom_op_handlers, custom_rewriter, inputs_as_nchw, outputs_as_nchw, extra_opset, shape_override, target, large_model, output_path) @@ -540,7 +540,7 @@ def from_function(function, input_signature=None, opset=None, custom_ops=None, c Returns: An ONNX model_proto and an external_tensor_storage dict. """ - if LooseVersion(tf.__version__) < "2.0": + if get_tf_version() < Version("2.0"): raise NotImplementedError("from_function requires tf-2.0 or newer") if input_signature is None: diff --git a/tf2onnx/shape_inference.py b/tf2onnx/shape_inference.py index 9a28975b5..853d4835c 100644 --- a/tf2onnx/shape_inference.py +++ b/tf2onnx/shape_inference.py @@ -6,9 +6,9 @@ """ import logging -from distutils.version import LooseVersion from collections import defaultdict import numpy as np +from packaging.version import Version from tf2onnx import utils from tf2onnx.tf_utils import get_tf_tensor_shape, get_tf_const_value, get_tf_shape_attr, get_tf_version from tf2onnx.tf_loader import tf_reload_graph @@ -32,7 +32,7 @@ def infer_shape(tf_graph, shape_override): op_outputs_with_none_shape = check_shape_for_tf_graph(tf_graph) if op_outputs_with_none_shape: - if get_tf_version() > LooseVersion("1.5.0"): + if get_tf_version() > Version("1.5.0"): for op, outs in op_outputs_with_none_shape.items(): logger.warning( "Cannot infer shape for %s: %s", diff --git a/tf2onnx/tf_loader.py b/tf2onnx/tf_loader.py index 22d909b4f..7269a26e0 100644 --- a/tf2onnx/tf_loader.py +++ b/tf2onnx/tf_loader.py @@ -5,7 +5,7 @@ import logging import uuid -from distutils.version import LooseVersion +from packaging.version import Version import tensorflow as tf import numpy as np @@ -75,7 +75,7 @@ def not_implemented_tf_placeholder(*args, **kwargs): tf_placeholder = tf.compat.v1.placeholder tf_placeholder_with_default = tf.compat.v1.placeholder_with_default extract_sub_graph = tf.compat.v1.graph_util.extract_sub_graph -elif LooseVersion(tf.__version__) >= "1.13": +elif Version(tf.__version__) >= Version("1.13"): # 1.13 introduced the compat namespace tf_reset_default_graph = tf.compat.v1.reset_default_graph tf_global_variables = tf.compat.v1.global_variables @@ -162,7 +162,7 @@ def make_tensor_proto_wrapped(values, dtype=None, shape=None, verify_shape=False try: function_converter = _FunctionConverterData - if LooseVersion(tf.__version__) >= "2.6.0": + if Version(tf.__version__) >= Version("2.6.0"): from tensorflow.python.eager import context from tensorflow.python.framework.convert_to_constants import _FunctionConverterDataInEager, \ _FunctionConverterDataInGraph @@ -267,7 +267,7 @@ def from_function(func, input_names, output_names, large_model=False): return convert_variables_to_constants_large_model(func) try: - if get_tf_version() < LooseVersion("2.2"): + if get_tf_version() < Version("2.2"): frozen_func = convert_variables_to_constants_v2(func, lower_control_flow=False) else: frozen_func = convert_variables_to_constants_v2(func, lower_control_flow=False, aggressive_inlining=True) @@ -687,7 +687,7 @@ def tf_optimize_grappler(input_names, output_names, graph_def): 'constfold', 'function' ] - if LooseVersion(tf.__version__) >= "2.5": + if Version(tf.__version__) >= Version("2.5"): # This flag disables folding QDQ nodes around constants in the network (eg: around conv/FC weights) rewrite_options.experimental_disable_folding_quantization_emulation = True @@ -710,7 +710,7 @@ def tf_optimize(input_names, output_names, graph_def): [utils.node_name(i) for i in output_names] graph_def = extract_sub_graph(graph_def, needed_names) - want_grappler = is_tf2() or LooseVersion(tf.__version__) >= "1.15" + want_grappler = is_tf2() or Version(tf.__version__) >= Version("1.15") if want_grappler: graph_def = tf_optimize_grappler(input_names, output_names, graph_def) else: @@ -730,7 +730,7 @@ def tf_optimize(input_names, output_names, graph_def): def tf_reload_graph(tf_graph): """Invoke tensorflow cpp shape inference by reloading graph_def.""" # invoke c api if tf version is below 1.8 - if get_tf_version() < LooseVersion("1.8"): + if get_tf_version() < Version("1.8"): logger.debug( "On TF < 1.8, graph is constructed by python API, " "which doesn't invoke shape inference, please set " diff --git a/tf2onnx/tf_utils.py b/tf2onnx/tf_utils.py index 54abc7071..1f59adae4 100644 --- a/tf2onnx/tf_utils.py +++ b/tf2onnx/tf_utils.py @@ -6,7 +6,7 @@ """ import collections -from distutils.version import LooseVersion +from packaging.version import Version import numpy as np import tensorflow as tf @@ -121,7 +121,7 @@ def get_tf_node_attr(node, name): def get_tf_version(): - return LooseVersion(tf.__version__) + return Version(tf.__version__) def compress_graph_def(graph_def): """