Merge master into temp/docs

README.md

@@ -1,6 +1,6 @@
 # NimbusML
 
-`nimbusml` is a Python module that provides experimental Python bindings for [ML.NET](https://github.com/dotnet/machinelearning). 
+`nimbusml` is a Python module that provides Python bindings for [ML.NET](https://github.com/dotnet/machinelearning). 
 
 ML.NET was originally developed in Microsoft Research and is used across many product groups in Microsoft like Windows, Bing, PowerPoint, Excel and others. `nimbusml` was built to enable data science teams that are more familiar with Python to take advantage of ML.NET's functionality and performance. 
 

build.cmd

@@ -311,6 +311,14 @@ copy  "%BuildOutputDir%%Configuration%\pybridge.pyd" "%__currentScriptDir%src\py
 
 if %PythonVersion% == 2.7 (
     copy "%BuildOutputDir%%Configuration%\Platform\win-x64\publish\*.dll" "%__currentScriptDir%src\python\nimbusml\internal\libs\"
+	:: remove dataprep dlls as its not supported in python 2.7
+	del "%__currentScriptDir%src\python\nimbusml\internal\libs\Microsoft.DPrep.*"
+	del "%__currentScriptDir%src\python\nimbusml\internal\libs\Microsoft.Data.*"
+	del "%__currentScriptDir%src\python\nimbusml\internal\libs\Microsoft.ProgramSynthesis.*"
+	del "%__currentScriptDir%src\python\nimbusml\internal\libs\Microsoft.DataPrep.dll"
+	del "%__currentScriptDir%src\python\nimbusml\internal\libs\ExcelDataReader.dll"
+	del "%__currentScriptDir%src\python\nimbusml\internal\libs\Microsoft.WindowsAzure.Storage.dll"
+	del "%__currentScriptDir%src\python\nimbusml\internal\libs\Microsoft.Workbench.Messaging.SDK.dll"
 ) else (
     for /F "tokens=*" %%A in (build/libs_win.txt) do copy "%BuildOutputDir%%Configuration%\Platform\win-x64\publish\%%A" "%__currentScriptDir%src\python\nimbusml\internal\libs\"
 )
@@ -340,10 +348,15 @@ if "%InstallPythonPackages%" == "True" (
     echo "#################################"
     echo "Installing python packages ... "
     echo "#################################"
+    call "%PythonExe%" -m pip install --upgrade pip
     call "%PythonExe%" -m pip install --upgrade nose pytest graphviz imageio pytest-cov "jupyter_client>=4.4.0" "nbconvert>=4.2.0"
-    if %PythonVersion% == 2.7 ( call "%PythonExe%" -m pip install --upgrade pyzmq )
-    :: Run azureml-dataprep tests only in pyhon 3.7 as its an optional dependency
-    if %PythonVersion% == 3.7 ( call "%PythonExe%" -m pip install --upgrade azureml-dataprep )
+
+    if %PythonVersion% == 2.7 (
+        call "%PythonExe%" -m pip install --upgrade pyzmq
+    ) else (
+        call "%PythonExe%" -m pip install --upgrade "azureml-dataprep>=1.1.12" 
+    )
+
     call "%PythonExe%" -m pip install --upgrade "%__currentScriptDir%target\%WheelFile%"
     call "%PythonExe%" -m pip install "scikit-learn==0.19.2"
 )

build.sh

@@ -52,7 +52,7 @@ while [ "$1" != "" ]; do
             __runTests=true
             __installPythonPackages=true
             ;;
-        --installPythonPackages)
+        --installpythonpackages)
             __installPythonPackages=true
             ;;
         --includeextendedtests)
@@ -219,6 +219,19 @@ then
             ext=*.dylib
 		fi	
 		cp  "${BuildOutputDir}/${__configuration}/Platform/${PublishDir}"/publish/${ext} "${__currentScriptDir}/src/python/nimbusml/internal/libs/"
+		# Obtain "libtensorflow_framework.so.1", which is the upgraded version of "libtensorflow.so". This is required for tests TensorFlowScorer.py to pass in Linux distros with Python 2.7
+		if [ ! "$(uname -s)" = "Darwin" ]
+		then
+			cp  "${BuildOutputDir}/${__configuration}/Platform/${PublishDir}"/publish/libtensorflow_framework.so.1 "${__currentScriptDir}/src/python/nimbusml/internal/libs/"
+		fi
+		# remove dataprep dlls as its not supported in python 2.7
+		rm -f "${__currentScriptDir}/src/python/nimbusml/internal/libs/Microsoft.DPrep.*"
+		rm -f "${__currentScriptDir}/src/python/nimbusml/internal/libs/Microsoft.Data.*"
+		rm -f "${__currentScriptDir}/src/python/nimbusml/internal/libs/Microsoft.ProgramSynthesis.*"
+		rm -f "${__currentScriptDir}/src/python/nimbusml/internal/libs/Microsoft.DataPrep.dll"
+		rm -f "${__currentScriptDir}/src/python/nimbusml/internal/libs/ExcelDataReader.dll"
+		rm -f "${__currentScriptDir}/src/python/nimbusml/internal/libs/Microsoft.WindowsAzure.Storage.dll"
+		rm -f "${__currentScriptDir}/src/python/nimbusml/internal/libs/Microsoft.Workbench.Messaging.SDK.dll"
     else
 		libs_txt=libs_linux.txt
 		if [ "$(uname -s)" = "Darwin" ]
@@ -271,13 +284,14 @@ then
     if [ ${PythonVersion} = 2.7 ]
     then
         "${PythonExe}" -m pip install --upgrade pyzmq
-    elif [ ${PythonVersion} = 3.6 ] && [ "$(uname -s)" = "Darwin" ]
-    then
-        "${PythonExe}" -m pip install --upgrade pytest-remotedata
-    elif [ ${PythonVersion} = 3.7 ]
-    then
-        "${PythonExe}" -m pip install --upgrade azureml-dataprep
-	fi
+    else
+        if [ ${PythonVersion} = 3.6 ] && [ "$(uname -s)" = "Darwin" ]
+        then
+            "${PythonExe}" -m pip install --upgrade pytest-remotedata
+        fi
+
+        "${PythonExe}" -m pip install --upgrade "azureml-dataprep>=1.1.12"
+    fi
     "${PythonExe}" -m pip install --upgrade "${Wheel}"
     "${PythonExe}" -m pip install "scikit-learn==0.19.2"
 fi

build/libs_linux.txt

@@ -12,6 +12,4 @@ libonnxruntime.so
 System.Drawing.Common.dll
 TensorFlow.NET.dll
 NumSharp.Core.dll
-Microsoft.DataPrep.dll
-Microsoft.DPrep.*
 Microsoft.ML.*

build/libs_mac.txt

@@ -12,6 +12,4 @@ libtensorflow_framework.1.dylib
 System.Drawing.Common.dll
 TensorFlow.NET.dll
 NumSharp.Core.dll
-Microsoft.DataPrep.dll
-Microsoft.DPrep.*
 Microsoft.ML.*

build/libs_win.txt

@@ -12,6 +12,4 @@ tensorflow.dll
 TensorFlow.NET.dll
 NumSharp.Core.dll
 System.Drawing.Common.dll
-Microsoft.DataPrep.dll
-Microsoft.DPrep.*
 Microsoft.ML.*

docs/release-notes/release-1.4.0.md

@@ -0,0 +1,57 @@
+# [NimbusML](https://docs.microsoft.com/en-us/nimbusml/overview) 1.4.0
+
+## **New Features**
+
+- **Add initial implementation of DatasetTransformer.**
+
+    [PR#240](https://github.com/microsoft/NimbusML/pull/240)
+    This transform allows a fitted transformer based model to be inserted
+    in to another `Pipeline`.
+
+    ```python
+    Pipeline([
+        DatasetTransformer(transform_model=transform_pipeline.model),
+        OnlineGradientDescentRegressor(label='c2', feature=['c1'])
+    ])
+    ```
+
+## **Bug Fixes**
+
+- **Fixed `classes_` attribute when no `y` input specified **
+
+    [PR#218](https://github.com/microsoft/NimbusML/pull/218)
+    Fix a bug with the classes_ attribute when no y input is specified during fitting.
+    This addresses [issue 216](https://github.com/microsoft/NimbusML/issues/216)
+
+- **Fixed Add NumSharp.Core.dll **
+
+    [PR#220](https://github.com/microsoft/NimbusML/pull/220)
+    Fixed a bug that prevented running TensorFlowScorer.
+    This addresses [issue 219](https://github.com/microsoft/NimbusML/issues/219)
+
+- **Fixed Enable scoring of ML.NET models saved with new TransformerChain format **
+
+    [PR#230](https://github.com/microsoft/NimbusML/pull/230)
+    Fixed error loading a model that was saved with mlnet auto-train.
+    This addresses [issue 201](https://github.com/microsoft/NimbusML/issues/201)
+
+- **Fixed Pass python path to Dprep package **
+
+    [PR#232](https://github.com/microsoft/NimbusML/pull/232)
+    Enable passing python executable to dataprep package, so dataprep can execute python transformations
+
+## **Breaking Changes**
+
+None.
+
+## **Enhancements**
+
+None.
+
+## **Documentation and Samples**
+
+None. 
+
+## **Remarks**
+
+None.

release-next.md

@@ -2,15 +2,91 @@
 
 ## **New Features**
 
-None.
+- **Initial implementation of `csr_matrix` output support.**
+
+    [PR#250](https://github.com/microsoft/NimbusML/pull/250)
+    Add support for data output in `scipy.sparse.csr_matrix` format.
+
+    ```python
+    xf = OneHotVectorizer(columns={'c0':'c0', 'c1':'c1'})
+    xf.fit(train_df)
+    result = xf.transform(train_df, as_csr=True)
+    ```
+
+- **Permutation Feature Importance for model interpretibility.**
+
+    [PR#279](https://github.com/microsoft/NimbusML/pull/279)
+    Adds `permutation_feature_importance()` method to `Pipeline` and
+    predictor estimators, enabling evaluation of model-wide feature
+    importances on any dataset with same schema as the dataset used
+    to fit the `Pipeline`.
+
+    ```python
+    pipe = Pipeline([
+        LogisticRegressionBinaryClassifier(label='label', feature=['feature'])
+    ])
+    pipe.fit(data)
+    pipe.permutation_feature_importance(data)
+    ```
+
+- **Initial implementation of DateTime input and output column support.**
+
+    [PR#290](https://github.com/microsoft/NimbusML/pull/290)
+    Add initial support for input and output of Pandas DateTime columns.
+
+- **Initial implementation of LpScaler.**
+
+    [PR#253](https://github.com/microsoft/NimbusML/pull/253)
+    Normalize vectors (rows) individually by rescaling them to unit norm (L2, L1 or LInf).
+    Performs the following operation on a vector X: Y = (X - M) / D, where M is mean and D
+    is either L2 norm, L1 norm or LInf norm.
+
+- **Add support for variable length vector output.**
+
+    [PR#267](https://github.com/microsoft/NimbusML/pull/267)
+    Support output of columns returned from ML.Net which contain variable length vectors.
+
+- **Save `predictor_model` when pickling a `Pipeline`.**
+
+    [PR#295](https://github.com/microsoft/NimbusML/pull/295)
+
+- **Initial implementation of the WordTokenizer transform.**
+
+    [PR#296](https://github.com/microsoft/NimbusML/pull/296)
+
+- **Add support for summary output from tree based predictors.**
+
+    [PR#298](https://github.com/microsoft/NimbusML/pull/298)
 
 ## **Bug Fixes**
 
-None.
+- **Fixed `Pipeline.transform()` in transform only `Pipeline` fails if y column is provided **
+
+    [PR#294](https://github.com/microsoft/NimbusML/pull/294)
+    Enable calling `.transform()` on a `Pipeline` containing only transforms when the y column is provided 
+
+- **Fix issue when using `predict_proba` or `decision_function` with combined models.**
+
+    [PR#272](https://github.com/microsoft/NimbusML/pull/272)
+
+- **Fix `Pipeline._extract_classes_from_headers` was not checking for valid steps.**
+
+    [PR#292](https://github.com/microsoft/NimbusML/pull/292)
+
+- **Fix BinaryDataStream was not valid as input for transformer.**
+
+    [PR#307](https://github.com/microsoft/NimbusML/pull/307)
+
+- **Fix casing for the installPythonPackages build.sh argument.**
+
+    [PR#256](https://github.com/microsoft/NimbusML/pull/256)
 
 ## **Breaking Changes**
 
-None.
+- **Removed `y` parameter from `Pipeline.transform()`**
+
+    [PR#294](https://github.com/microsoft/NimbusML/pull/294)
+    Removed `y` parameter from `Pipeline.transform()` as it is not needed nor used for transforming data with a fitted `Pipeline`.
 
 ## **Enhancements**