update documentation

Author: Ivan Matantsev

src/Microsoft.ML.Transforms/EntryPoints/TextAnalytics.cs

@@ -138,8 +138,12 @@ public static CommonOutputs.TransformOutput LightLda(IHostEnvironment env, LdaTr
             };
         }
 
-        [TlcModule.EntryPoint(Name = "Transforms.WordEmbeddings", Desc = WordEmbeddingsTransform.Summary,
-            UserName = WordEmbeddingsTransform.UserName)]
+        [TlcModule.EntryPoint(Name = "Transforms.WordEmbeddings",
+            Desc = WordEmbeddingsTransform.Summary,
+            UserName = WordEmbeddingsTransform.UserName,
+            ShortName = WordEmbeddingsTransform.ShortName,
+            XmlInclude = new[] { @"<include file='../Microsoft.ML.Transforms/Text/doc.xml' path='doc/members/member[@name=""WordEmbeddings""]/*' />",
+                                 @"<include file='../Microsoft.ML.Transforms/Text/doc.xml' path='doc/members/example[@name=""WordEmbeddings""]/*' />" })]
         public static CommonOutputs.TransformOutput WordEmbeddings(IHostEnvironment env, WordEmbeddingsTransform.Arguments input)
         {
             Contracts.CheckValue(env, nameof(env));

src/Microsoft.ML.Transforms/Text/WordEmbeddingsTransform.cs

@@ -18,13 +18,14 @@
 using Microsoft.ML.Runtime.Model;
 
 [assembly: LoadableClass(WordEmbeddingsTransform.Summary, typeof(IDataTransform), typeof(WordEmbeddingsTransform), typeof(WordEmbeddingsTransform.Arguments),
-    typeof(SignatureDataTransform), WordEmbeddingsTransform.UserName, "WordEmbeddingsTransform", "WordEmbeddings", DocName = "transform/WordEmbeddingsTransform.md")]
+    typeof(SignatureDataTransform), WordEmbeddingsTransform.UserName, "WordEmbeddingsTransform", WordEmbeddingsTransform.ShortName, DocName = "transform/WordEmbeddingsTransform.md")]
 
 [assembly: LoadableClass(typeof(WordEmbeddingsTransform), null, typeof(SignatureLoadDataTransform),
     WordEmbeddingsTransform.UserName, WordEmbeddingsTransform.LoaderSignature)]
 
 namespace Microsoft.ML.Runtime.Data
 {
+    /// <include file='doc.xml' path='doc/members/member[@name="WordEmbeddings"]/*' />
     public sealed class WordEmbeddingsTransform : OneToOneTransformBase
     {
         public sealed class Column : OneToOneColumn
@@ -62,6 +63,7 @@ public sealed class Arguments : TransformInputBase
         internal const string Summary = "Word Embeddings transform is a text featurizer which converts vectors of text tokens into sentence " +
             "vectors using a pre-trained model";
         internal const string UserName = "Word Embeddings Transform";
+        internal const string ShortName = "WordEmbeddings";
         public const string LoaderSignature = "WordEmbeddingsTransform";
 
         public static VersionInfo GetVersionInfo()

src/Microsoft.ML.Transforms/Text/doc.xml

@@ -184,5 +184,37 @@
       </example>
     </example>
 
+    <member name="WordEmbeddings">
+      <summary>
+        Word Embeddings transform is a text featurizer which converts vectors of text tokens into sentence vectors using a pre-trained model.
+      </summary>
+      <remarks>
+        WordEmbeddings wrap different embedding models, such as GloVe. Users can specify which embedding to use. 
+        The available options are various versions of <a href="https://nlp.stanford.edu/projects/glove/">GloVe Models</a>, <a href="https://en.wikipedia.org/wiki/FastText">FastText</a>, and <a href="http://anthology.aclweb.org/P/P14/P14-1146.pdf">Sswe</a>.
+        <para>
+          Note: As WordEmbedding requires a column with text vector, e.g. %3C'This', 'is', 'good'%3E, users need to create an input column by:
+          <list type="bullet">
+          <item><description>concatenating columns with TX type,</description></item>
+            <item>
+              <description>or using the output_tokens=True for NGramFeaturizer() to convert a column with sentences like "This is good" into %3C'This', 'is', 'good' %3E. 
+              The column for the output token column is renamed with a prefix of '_TranformedText'.</description>
+            </item>
+        </list>
+          In the following example, after the NGramFeaturizer, features named ngram.__ are generated. A new column named ngram_TransformedText is
+          also created with the text vector, similar as running .split(' '). However, due to the variable length of this column it cannot be properly
+          converted to pandas dataframe, thus any pipelines/transforms output this text vector column will throw errors. However, we use 
+          ngram_TransformedText as the input to WordEmbedding, the ngram_TransformedText column will be overwritten by the output from 
+          WordEmbedding. The output from WordEmbedding is named ngram_TransformedText.__
+        </para>
+      </remarks>
+    </member>
+    <example name="WordEmbeddings">
+      <example>
+        <code language="csharp">
+          pipeline.Add(new WordEmbeddings(("InTextCol" , "OutTextCol")));
+        </code>
+      </example>
+    </example>
+
   </members>
 </doc>

src/Microsoft.ML/CSharpApi.cs

@@ -15474,9 +15474,8 @@ public sealed partial class WordEmbeddingsTransformColumn : OneToOneColumn<WordE
 
         }
 
-        /// <summary>
-        /// Word Embeddings transform is a text featurizer which converts vectors of text tokens into sentence vectors using a pre-trained model
-        /// </summary>
+        /// <include file='../Microsoft.ML.Transforms/Text/doc.xml' path='doc/members/member[@name="WordEmbeddings"]/*' />
+        /// <include file='../Microsoft.ML.Transforms/Text/doc.xml' path='doc/members/example[@name="WordEmbeddings"]/*' />
         public sealed partial class WordEmbeddings : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITransformInput, Microsoft.ML.ILearningPipelineItem
         {
 

test/BaselineOutput/Common/EntryPoints/core_manifest.json

@@ -21454,7 +21454,7 @@
       "Name": "Transforms.WordEmbeddings",
       "Desc": "Word Embeddings transform is a text featurizer which converts vectors of text tokens into sentence vectors using a pre-trained model",
       "FriendlyName": "Word Embeddings Transform",
-      "ShortName": null,
+      "ShortName": "WordEmbeddings",
       "Inputs": [
         {
           "Name": "Column",

test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs

@@ -3719,8 +3719,6 @@ public void EntryPointWordEmbeddings()
                 "The quick brown fox jumps over the lazy dog.",
                 "The five boxing wizards jump quickly."
             });
-
-
             var inputFile = new SimpleFileHandle(Env, dataFile, false, false);
             var dataView = ImportTextData.TextLoader(Env, new ImportTextData.LoaderInput()
             {
@@ -3733,19 +3731,17 @@ public void EntryPointWordEmbeddings()
                         {
                             Name = "Text",
                             Source = new [] { new TextLoader.Range() { Min = 0, VariableEnd=true, ForceVector=true} },
-                            
                             Type = DataKind.Text
                         }
                     }
                 },
-
                 InputFile = inputFile,
             }).Data;
             var embedding = Transforms.TextAnalytics.WordEmbeddings(Env, new WordEmbeddingsTransform.Arguments()
             {
-                Data= dataView,
-                Column = new[] {new  WordEmbeddingsTransform.Column { Name = "Features", Source = "Text" } },
-                ModelKind= WordEmbeddingsTransform.PretrainedModelKind.Sswe
+                Data = dataView,
+                Column = new[] { new WordEmbeddingsTransform.Column { Name = "Features", Source = "Text" } },
+                ModelKind = WordEmbeddingsTransform.PretrainedModelKind.Sswe
             });
             var result = embedding.OutputData;
             using (var cursor = result.GetRowCursor((x => true)))