From fb4e375ceb2180e092d58fcf8362a2fb731348c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Thu, 15 Apr 2021 19:09:06 +0200 Subject: [PATCH 1/7] update import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: xavier dupré --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 7a07aee1..4a81c8a9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,6 +3,7 @@ codecov coremltools cython flatbuffers +libsvm lightgbm h2o==3.28.0.3 mleap From 3b04f356b9e307c24db59947d00d7b01ffee243c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Thu, 15 Apr 2021 19:16:19 +0200 Subject: [PATCH 2/7] ci MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: xavier dupré --- .azure-pipelines/linux-CI-nightly.yml | 5 - .azure-pipelines/linux-conda-CI.yml | 5 - .azure-pipelines/win32-CI-nightly.yml | 139 ++++++++++++-------------- .azure-pipelines/win32-conda-CI.yml | 7 -- 4 files changed, 66 insertions(+), 90 deletions(-) diff --git a/.azure-pipelines/linux-CI-nightly.yml b/.azure-pipelines/linux-CI-nightly.yml index 39a5819d..ad890365 100644 --- a/.azure-pipelines/linux-CI-nightly.yml +++ b/.azure-pipelines/linux-CI-nightly.yml @@ -51,14 +51,9 @@ jobs: python -m pip install -r requirements-dev.txt python -m pip install $(ORT_PATH) python -m pip install pytest - git clone --recursive https://github.com/cjlin1/libsvm libsvm - cd libsvm - make lib displayName: 'Install dependencies' - script: | - export PYTHONPATH=$PYTHONPATH:libsvm/python - python -c "import svmutil" python -c "import onnxconverter_common" python -c "import onnxruntime" pytest tests --ignore=tests/sparkml --doctest-modules --junitxml=junit/test-results.xml diff --git a/.azure-pipelines/linux-conda-CI.yml b/.azure-pipelines/linux-conda-CI.yml index dcc4ff26..9f5a3891 100644 --- a/.azure-pipelines/linux-conda-CI.yml +++ b/.azure-pipelines/linux-conda-CI.yml @@ -90,9 +90,6 @@ jobs: pip install xgboost$(xgboost.version) pip install $(ONNXRT_PATH) pip install pytest - git clone --recursive https://github.com/cjlin1/libsvm libsvm - cd libsvm - make lib displayName: 'Install dependencies' - script: | @@ -100,8 +97,6 @@ jobs: displayName: 'run flake8 check' - script: | - export PYTHONPATH=$PYTHONPATH:libsvm/python - python -c "import svmutil" python -c "import onnxconverter_common" python -c "import onnxruntime" pytest tests --ignore=tests/sparkml --doctest-modules --junitxml=junit/test-results.xml diff --git a/.azure-pipelines/win32-CI-nightly.yml b/.azure-pipelines/win32-CI-nightly.yml index e3f7cf81..dfe994d9 100644 --- a/.azure-pipelines/win32-CI-nightly.yml +++ b/.azure-pipelines/win32-CI-nightly.yml @@ -1,73 +1,66 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python - -trigger: -- master - -jobs: - -- job: 'Test' - pool: - vmImage: 'vs2017-win2016' - strategy: - matrix: - Python36-nightly: - python.version: '3.6' - ONNX_PATH: onnx==1.7.0 - ONNXRT_PATH: -i https://test.pypi.org/simple/ ort-nightly - COREML_PATH: git+https://github.com/apple/coremltools@3.1 - Python37-nightly: - python.version: '3.7' - ONNX_PATH: onnx==1.8.0 - ONNXRT_PATH: -i https://test.pypi.org/simple/ ort-nightly - COREML_PATH: git+https://github.com/apple/coremltools@3.1 - maxParallel: 3 - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - architecture: 'x64' - - - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" - displayName: Add conda to PATH - - - script: conda create --yes --quiet --name py$(python.version) -c conda-forge python=$(python.version) numpy protobuf - displayName: Create Anaconda environment - - - script: | - call activate py$(python.version) - python -m pip install --upgrade pip numpy - echo Test numpy installation... && python -c "import numpy" - pip install %COREML_PATH% %ONNX_PATH% - python -m pip install tensorflow-cpu==1.15.0 - python -m pip install tf2onnx==1.5.6 - python -m pip install git+https://github.com/microsoft/onnxconverter-common - python -m pip install git+https://github.com/onnx/keras-onnx - echo Test onnxconverter-common installation... && python -c "import onnxconverter_common" - pip install -r requirements.txt - pip install -r requirements-dev.txt - pip install %ONNXRT_PATH% - echo Test onnxruntime installation... && python -c "import onnxruntime" - REM install libsvm from github - git clone --recursive https://github.com/cjlin1/libsvm libsvm - copy libsvm\windows\*.dll libsvm\python - set PYTHONPATH=libsvm\python;%PYTHONPATH% - dir libsvm\python - echo Test libsvm installation... && python -c "import svmutil" - displayName: 'Install dependencies' - - - script: | - call activate py$(python.version) - set PYTHONPATH=libsvm\python;%PYTHONPATH% - pip install -e . - python -m pytest tests --ignore=tests/sparkml --doctest-modules --junitxml=junit/test-results.xml - displayName: 'pytest - onnxmltools' - - - task: PublishTestResults@2 - inputs: - testResultsFiles: '**/test-results.xml' - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() +# Python package +# Create and test a Python package on multiple Python versions. +# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/python + +trigger: +- master + +jobs: + +- job: 'Test' + pool: + vmImage: 'vs2017-win2016' + strategy: + matrix: + Python36-nightly: + python.version: '3.6' + ONNX_PATH: onnx==1.7.0 + ONNXRT_PATH: -i https://test.pypi.org/simple/ ort-nightly + COREML_PATH: git+https://github.com/apple/coremltools@3.1 + Python37-nightly: + python.version: '3.7' + ONNX_PATH: onnx==1.8.0 + ONNXRT_PATH: -i https://test.pypi.org/simple/ ort-nightly + COREML_PATH: git+https://github.com/apple/coremltools@3.1 + maxParallel: 3 + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + architecture: 'x64' + + - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" + displayName: Add conda to PATH + + - script: conda create --yes --quiet --name py$(python.version) -c conda-forge python=$(python.version) numpy protobuf + displayName: Create Anaconda environment + + - script: | + call activate py$(python.version) + python -m pip install --upgrade pip numpy + echo Test numpy installation... && python -c "import numpy" + pip install %COREML_PATH% %ONNX_PATH% + python -m pip install tensorflow-cpu==1.15.0 + python -m pip install tf2onnx==1.5.6 + python -m pip install git+https://github.com/microsoft/onnxconverter-common + python -m pip install git+https://github.com/onnx/keras-onnx + echo Test onnxconverter-common installation... && python -c "import onnxconverter_common" + pip install -r requirements.txt + pip install -r requirements-dev.txt + pip install %ONNXRT_PATH% + echo Test onnxruntime installation... && python -c "import onnxruntime" + displayName: 'Install dependencies' + + - script: | + call activate py$(python.version) + pip install -e . + python -m pytest tests --ignore=tests/sparkml --doctest-modules --junitxml=junit/test-results.xml + displayName: 'pytest - onnxmltools' + + - task: PublishTestResults@2 + inputs: + testResultsFiles: '**/test-results.xml' + testRunTitle: 'Python $(python.version)' + condition: succeededOrFailed() diff --git a/.azure-pipelines/win32-conda-CI.yml b/.azure-pipelines/win32-conda-CI.yml index b65a7072..b38ec546 100644 --- a/.azure-pipelines/win32-conda-CI.yml +++ b/.azure-pipelines/win32-conda-CI.yml @@ -91,12 +91,6 @@ jobs: python -m pip install %ONNXRT_PATH% python -m pip install scikit-learn$(sklearn.version) echo Test onnxruntime installation... && python -c "import onnxruntime" - REM install libsvm from github - git clone --recursive https://github.com/cjlin1/libsvm libsvm - copy libsvm\windows\*.dll libsvm\python - set PYTHONPATH=libsvm\python;%PYTHONPATH% - dir libsvm\python - echo Test libsvm installation... && python -c "import svmutil" echo "debug environment" && path python -m pip show pytest displayName: 'Install dependencies' @@ -108,7 +102,6 @@ jobs: - script: | call activate py$(python.version) - set PYTHONPATH=libsvm\python;%PYTHONPATH% python -m pip install -e . python -m pytest tests --ignore=tests/sparkml --doctest-modules --junitxml=junit/test-results.xml displayName: 'pytest - onnxmltools' From a1f22c10b9f9941be6ff319d90f092e21245c085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Thu, 15 Apr 2021 23:00:46 +0200 Subject: [PATCH 3/7] fix svm issue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: xavier dupré --- .azure-pipelines/linux-conda-CI.yml | 4 ++++ requirements-dev.txt | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/linux-conda-CI.yml b/.azure-pipelines/linux-conda-CI.yml index 9f5a3891..9a1e7b13 100644 --- a/.azure-pipelines/linux-conda-CI.yml +++ b/.azure-pipelines/linux-conda-CI.yml @@ -96,6 +96,10 @@ jobs: python -m flake8 ./onnxmltools displayName: 'run flake8 check' + - script: | + pip install -e . + displayName: 'local installation' + - script: | python -c "import onnxconverter_common" python -c "import onnxruntime" diff --git a/requirements-dev.txt b/requirements-dev.txt index 4a81c8a9..64dfa592 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,7 +15,6 @@ pytest pytest-cov scikit-learn scipy -svm wheel xgboost catboost From 763b680d7d4471aef33aec486be9d19591bb0f60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Thu, 15 Apr 2021 23:23:04 +0200 Subject: [PATCH 4/7] update build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: xavier dupré --- onnxmltools/convert/libsvm/operator_converters/SVMConverter.py | 3 +-- requirements-dev.txt | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/onnxmltools/convert/libsvm/operator_converters/SVMConverter.py b/onnxmltools/convert/libsvm/operator_converters/SVMConverter.py index aec1bb5b..43936241 100644 --- a/onnxmltools/convert/libsvm/operator_converters/SVMConverter.py +++ b/onnxmltools/convert/libsvm/operator_converters/SVMConverter.py @@ -5,11 +5,10 @@ from ...common.utils import cast_list import numpy try: - from libsvm import svm, svmutil + from libsvm import svm except ImportError: # Older version of libsvm. import svm - import svmutil class SVMConverter: diff --git a/requirements-dev.txt b/requirements-dev.txt index 64dfa592..a375a092 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,7 @@ coremltools cython flatbuffers libsvm -lightgbm +lightgbm!=3.2.1 h2o==3.28.0.3 mleap numpy From 4ac817bae3464a24a7fa305e9ac281db22884396 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Thu, 15 Apr 2021 23:45:16 +0200 Subject: [PATCH 5/7] ci MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: xavier dupré --- .azure-pipelines/win32-conda-CI.yml | 7 ------- tests/svmlib/test_SVMConverters.py | 13 ++++++++----- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/.azure-pipelines/win32-conda-CI.yml b/.azure-pipelines/win32-conda-CI.yml index b38ec546..6ca847f1 100644 --- a/.azure-pipelines/win32-conda-CI.yml +++ b/.azure-pipelines/win32-conda-CI.yml @@ -13,13 +13,6 @@ jobs: vmImage: 'vs2017-win2016' strategy: matrix: - Python35-141-RT030: - python.version: '3.5' - ONNX_PATH: onnx==1.4.1 - ONNXRT_PATH: onnxruntime==0.3.0 - COREML_PATH: https://github.com/apple/coremltools/archive/v2.0.zip - sklearn.version: '==0.19.1' - Python36-141-RT030: python.version: '3.6' ONNX_PATH: onnx==1.4.1 diff --git a/tests/svmlib/test_SVMConverters.py b/tests/svmlib/test_SVMConverters.py index baddf3b4..cd12d5f6 100644 --- a/tests/svmlib/test_SVMConverters.py +++ b/tests/svmlib/test_SVMConverters.py @@ -6,20 +6,23 @@ import tempfile import numpy try: - import svm + from libsvm.svm import C_SVC as SVC, EPSILON_SVR as SVR, NU_SVC as NuSVC, NU_SVR as NuSVR + import libsvm.svm as svm + import libsvm.svmutil as svmutil except ImportError: - import libsvm.svm as svm + import svm + from svm import C_SVC as SVC, EPSILON_SVR as SVR, NU_SVC as NuSVC, NU_SVR as NuSVR + import svmutil + import numpy as np import unittest from sklearn.datasets import load_iris from onnxmltools.convert.libsvm import convert -from svm import C_SVC as SVC, EPSILON_SVR as SVR, NU_SVC as NuSVC, NU_SVR as NuSVR -import svmutil from onnxmltools.convert.common.data_types import FloatTensorType from onnxmltools.utils import dump_data_and_model try: - from svm import PRINT_STRING_FUN, print_null + from libsvm.svm import PRINT_STRING_FUN, print_null noprint = PRINT_STRING_FUN(print_null) except ImportError: # This was recently added. From 651bb0c7d1bbb2e8cc2d3fa5feb7951cd66cd401 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Fri, 16 Apr 2021 00:05:39 +0200 Subject: [PATCH 6/7] disable one test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: xavier dupré --- ...l_TreeEnsembleRegressorConverterXGBoost.py | 101 +++++++++--------- 1 file changed, 51 insertions(+), 50 deletions(-) diff --git a/tests/coreml/test_cml_TreeEnsembleRegressorConverterXGBoost.py b/tests/coreml/test_cml_TreeEnsembleRegressorConverterXGBoost.py index 527cfae4..a70e287d 100644 --- a/tests/coreml/test_cml_TreeEnsembleRegressorConverterXGBoost.py +++ b/tests/coreml/test_cml_TreeEnsembleRegressorConverterXGBoost.py @@ -1,50 +1,51 @@ -# SPDX-License-Identifier: Apache-2.0 - -""" -Tests CoreML TreeEnsembleRegressor converter. -""" -import os -import sys -import unittest -import numpy -import pandas -try: - from sklearn.impute import SimpleImputer as Imputer - import sklearn.preprocessing - if not hasattr(sklearn.preprocessing, 'Imputer'): - # coremltools 3.1 does not work with scikit-learn 0.22 - setattr(sklearn.preprocessing, 'Imputer', Imputer) -except ImportError: - from sklearn.preprocessing import Imputer -from coremltools.converters.xgboost import convert as convert_xgb_to_coreml -from onnxmltools.convert.coreml import convert as convert_cml -from xgboost import XGBRegressor -from onnxmltools.utils import dump_data_and_model - - -class TestCoreMLTreeEnsembleRegressorConverterXGBoost(unittest.TestCase): - - def test_tree_ensemble_regressor_xgboost(self): - - this = os.path.dirname(__file__) - data_train = pandas.read_csv(os.path.join(this, "xgboost.model.xgb.n4.d3.train.txt"), header=None) - - X = data_train.iloc[:, 1:].values - y = data_train.iloc[:, 0].values - - params = dict(n_estimator=4, max_depth=3) - model = XGBRegressor(**params).fit(X, y) - # See https://github.com/apple/coremltools/issues/51. - model.booster = model.get_booster - model_coreml = convert_xgb_to_coreml(model) - model_onnx = convert_cml(model_coreml) - assert model_onnx is not None - if sys.version_info[0] >= 3: - # python 2.7 returns TypeError: can't pickle instancemethod objects - dump_data_and_model(X.astype(numpy.float32), model, model_onnx, - basename="CmlXGBoostRegressor-OneOff-Reshape", - allow_failure=True) - - -if __name__ == "__main__": - unittest.main() +# SPDX-License-Identifier: Apache-2.0 + +""" +Tests CoreML TreeEnsembleRegressor converter. +""" +import os +import sys +import unittest +import numpy +import pandas +try: + from sklearn.impute import SimpleImputer as Imputer + import sklearn.preprocessing + if not hasattr(sklearn.preprocessing, 'Imputer'): + # coremltools 3.1 does not work with scikit-learn 0.22 + setattr(sklearn.preprocessing, 'Imputer', Imputer) +except ImportError: + from sklearn.preprocessing import Imputer +from coremltools.converters.xgboost import convert as convert_xgb_to_coreml +from onnxmltools.convert.coreml import convert as convert_cml +from xgboost import XGBRegressor +from onnxmltools.utils import dump_data_and_model + + +class TestCoreMLTreeEnsembleRegressorConverterXGBoost(unittest.TestCase): + + @unittest.skipIf(True, reason="broken") + def test_tree_ensemble_regressor_xgboost(self): + + this = os.path.dirname(__file__) + data_train = pandas.read_csv(os.path.join(this, "xgboost.model.xgb.n4.d3.train.txt"), header=None) + + X = data_train.iloc[:, 1:].values + y = data_train.iloc[:, 0].values + + params = dict(n_estimator=4, max_depth=3) + model = XGBRegressor(**params).fit(X, y) + # See https://github.com/apple/coremltools/issues/51. + model.booster = model.get_booster + model_coreml = convert_xgb_to_coreml(model) + model_onnx = convert_cml(model_coreml) + assert model_onnx is not None + if sys.version_info[0] >= 3: + # python 2.7 returns TypeError: can't pickle instancemethod objects + dump_data_and_model(X.astype(numpy.float32), model, model_onnx, + basename="CmlXGBoostRegressor-OneOff-Reshape", + allow_failure=True) + + +if __name__ == "__main__": + unittest.main() From 57f0568d1b3969516e920d33d966feac9691bb27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Fri, 16 Apr 2021 00:45:58 +0200 Subject: [PATCH 7/7] disable on test on keras2onnx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: xavier dupré --- tests/utils/test_utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index cedfa668..c28b2459 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -5,6 +5,7 @@ """ import os import unittest +import warnings import onnxmltools from onnxmltools.utils import load_model, save_model from onnxmltools.utils import set_model_version, set_model_domain, set_model_doc_string @@ -59,7 +60,11 @@ def test_set_docstring_blank(self): class TestWrapper(unittest.TestCase): def test_keras_with_tf2onnx(self): - import keras2onnx + try: + import keras2onnx + except ImportError: + warnings.warn("keras2onnx or one of its dependencies is missing.") + return from keras2onnx.proto import keras from keras2onnx.proto.tfcompat import is_tf2 if not is_tf2: # tf2onnx is not available for tensorflow 2.0 yet.