chore: remove storage keys and dotnet (#2205)

* chore: remove storage keys and dotnet * chore: fix style violations * chore: add flags * chore: add publishing cli auth * chore: fix auth error * chore: fix auth error * chore: fix auth error * chore: fix auth error * chore: fix auth error * try v1 * try v2 * hack around timing limitations * try skipping codegen in later stages * final fixes
microsoft · Apr 17, 2024 · d789b02 · d789b02
1 parent 139b13b
commit d789b02
Show file tree

Hide file tree

Showing 74 changed files with 172 additions and 6,060 deletions.
diff --git a/CODEOWNERS b/CODEOWNERS
@@ -29,7 +29,6 @@ explainers/      @memoryz
 
 #Serena's Areas
 website/         @serena-ruan
-core/src/main/dotnet @serena-ruan
 core/src/main/scala/com/microsoft/azure/synapse/ml/codegen @serena-ruan
 
 # Scott's Areas

diff --git a/README.md b/README.md
@@ -81,7 +81,6 @@ First select the correct platform that you are installing SynapseML into:
 - [Apache Livy and HDInsight](#apache-livy-and-hdinsight)
 - [Docker](#docker)
 - [R](#r)
-- [C# (.NET)](#c-net)
 - [Building from source](#building-from-source)
 <!--te-->
 
@@ -237,11 +236,6 @@ To try out SynapseML using the R autogenerated wrappers [see our
 instructions](https://microsoft.github.io/SynapseML/docs/Reference/R%20Setup/).  Note: This feature is still under development
 and some necessary custom wrappers may be missing.
 
-### C# (.NET)
-
-To try out SynapseML with .NET, please follow the [.NET Installation Guide](https://microsoft.github.io/SynapseML/docs/Reference/Dotnet%20Setup/).
-Please note that some classes including the `AzureSearchWriter`, `DiagnosticInfo`, `UDPyFParam`, `ParamSpaceParam`, `BallTreeParam`, `ConditionalBallTreeParam`, `LightGBMBoosterParam` are still under development and not exposed in .NET yet.
-
 ### Building from source
 
 SynapseML has recently transitioned to a new build infrastructure.

diff --git a/build.sbt b/build.sbt
@@ -112,122 +112,6 @@ rootGenDir := {
   join(targetDir, "generated")
 }
 
-// scalastyle:off line.size.limit
-val genSleetConfig = TaskKey[File]("genSleetConfig",
-  "generate sleet.json file for sleet configuration so we can push nuget package to the blob")
-genSleetConfig := {
-  val fileContent =
-    s"""{
-       |  "username": "",
-       |  "useremail": "",
-       |  "sources": [
-       |    {
-       |      "name": "SynapseMLNuget",
-       |      "type": "azure",
-       |      "container": "synapsemlnuget",
-       |      "path": "https://mmlspark.blob.core.windows.net/synapsemlnuget",
-       |      "connectionString": "DefaultEndpointsProtocol=https;AccountName=mmlspark;AccountKey=${Secrets.storageKey};EndpointSuffix=core.windows.net"
-       |    }
-       |  ]
-       |}""".stripMargin
-  val sleetJsonFile = join(rootGenDir.value, "sleet.json")
-  if (sleetJsonFile.exists()) FileUtils.forceDelete(sleetJsonFile)
-  FileUtils.writeStringToFile(sleetJsonFile, fileContent, "utf-8")
-  sleetJsonFile
-}
-// scalastyle:on line.size.limit
-
-val publishDotnetTestBase = TaskKey[Unit]("publishDotnetTestBase",
-  "generate dotnet test helper file with current library version and publish E2E test base")
-publishDotnetTestBase := {
-  val fileContent =
-    s"""// Licensed to the .NET Foundation under one or more agreements.
-       |// The .NET Foundation licenses this file to you under the MIT license.
-       |// See the LICENSE file in the project root for more information.
-       |
-       |namespace SynapseMLtest.Utils
-       |{
-       |    public class Helper
-       |    {
-       |        public static string GetSynapseMLPackage()
-       |        {
-       |            return "com.microsoft.azure:synapseml_2.12:${version.value}";
-       |        }
-       |    }
-       |
-       |}
-       |""".stripMargin
-  val dotnetTestBaseDir = join(baseDirectory.value, "core", "src", "main", "dotnet", "test")
-  val dotnetHelperFile = join(dotnetTestBaseDir, "SynapseMLVersion.cs")
-  if (dotnetHelperFile.exists()) FileUtils.forceDelete(dotnetHelperFile)
-  FileUtils.writeStringToFile(dotnetHelperFile, fileContent, "utf-8")
-
-  val dotnetTestBaseProjContent =
-    s"""<Project Sdk="Microsoft.NET.Sdk">
-       |
-       |  <PropertyGroup>
-       |    <TargetFramework>netstandard2.1</TargetFramework>
-       |    <LangVersion>9.0</LangVersion>
-       |    <AssemblyName>SynapseML.DotnetE2ETest</AssemblyName>
-       |    <IsPackable>true</IsPackable>
-       |    <Description>SynapseML .NET Test Base</Description>
-       |    <Version>${dotnetedVersion(version.value)}</Version>
-       |  </PropertyGroup>
-       |
-       |  <ItemGroup>
-       |    <PackageReference Include="xunit" Version="2.4.1" />
-       |    <PackageReference Include="Microsoft.Spark" Version="2.1.1" />
-       |    <PackageReference Include="IgnoresAccessChecksToGenerator" Version="0.4.0" PrivateAssets="All" />
-       |  </ItemGroup>
-       |
-       |  <ItemGroup>
-       |    <InternalsVisibleTo Include="SynapseML.Cognitive" />
-       |    <InternalsVisibleTo Include="SynapseML.Core" />
-       |    <InternalsVisibleTo Include="SynapseML.DeepLearning" />
-       |    <InternalsVisibleTo Include="SynapseML.Lightgbm" />
-       |    <InternalsVisibleTo Include="SynapseML.Opencv" />
-       |    <InternalsVisibleTo Include="SynapseML.Vw" />
-       |    <InternalsVisibleTo Include="SynapseML.Cognitive.Test" />
-       |    <InternalsVisibleTo Include="SynapseML.Core.Test" />
-       |    <InternalsVisibleTo Include="SynapseML.DeepLearning.Test" />
-       |    <InternalsVisibleTo Include="SynapseML.Lightgbm.Test" />
-       |    <InternalsVisibleTo Include="SynapseML.Opencv.Test" />
-       |    <InternalsVisibleTo Include="SynapseML.Vw.Test" />
-       |  </ItemGroup>
-       |
-       |  <PropertyGroup>
-       |    <InternalsAssemblyNames>Microsoft.Spark</InternalsAssemblyNames>
-       |  </PropertyGroup>
-       |
-       |  <PropertyGroup>
-       |    <InternalsAssemblyUseEmptyMethodBodies>false</InternalsAssemblyUseEmptyMethodBodies>
-       |  </PropertyGroup>
-       |
-       |</Project>""".stripMargin
-  // update the version of current dotnetTestBase assembly
-  val dotnetTestBaseProj = join(dotnetTestBaseDir, "dotnetTestBase.csproj")
-  if (dotnetTestBaseProj.exists()) FileUtils.forceDelete(dotnetTestBaseProj)
-  FileUtils.writeStringToFile(dotnetTestBaseProj, dotnetTestBaseProjContent, "utf-8")
-
-  packDotnetAssemblyCmd(join(dotnetTestBaseDir, "target").getAbsolutePath, dotnetTestBaseDir)
-  val packagePath = join(dotnetTestBaseDir,
-    "target", s"SynapseML.DotnetE2ETest.${dotnetedVersion(version.value)}.nupkg").getAbsolutePath
-  publishDotnetAssemblyCmd(packagePath, genSleetConfig.value)
-}
-
-// This command should be run only when you make an update to DotnetBase proj, and it will override
-// existing nuget package with the same version number
-val publishDotnetBase = TaskKey[Unit]("publishDotnetBase",
-  "publish dotnet base nuget package that contains core elements for SynapseML in C#")
-publishDotnetBase := {
-  val dotnetBaseDir = join(baseDirectory.value, "core", "src", "main", "dotnet", "src")
-  packDotnetAssemblyCmd(join(dotnetBaseDir, "target").getAbsolutePath, dotnetBaseDir)
-  val packagePath = join(dotnetBaseDir,
-    // Update the version whenever there's a new release
-    "target", s"SynapseML.DotnetBase.${dotnetedVersion("1.0.4")}.nupkg").getAbsolutePath
-  publishDotnetAssemblyCmd(packagePath, genSleetConfig.value)
-}
-
 def runTaskForAllInCompile(task: TaskKey[Unit]): Def.Initialize[Task[Seq[Unit]]] = {
   task.all(ScopeFilter(
     inProjects(core, deepLearning, cognitive, vw, lightgbm, opencv),
@@ -246,29 +130,6 @@ generatePythonDoc := {
   runCmd(activateCondaEnv ++ Seq("sphinx-build", "-b", "html", "doc", "../../../doc/pyspark"), dir)
 }
 
-val generateDotnetDoc = TaskKey[Unit]("generateDotnetDoc", "Generate documentation for dotnet classes")
-generateDotnetDoc := {
-  Def.sequential(
-    runTaskForAllInCompile(dotnetCodeGen),
-    runTaskForAllInCompile(mergeDotnetCode)
-  ).value
-  val dotnetSrcDir = join(rootGenDir.value, "src", "dotnet")
-  runCmd(Seq("doxygen", "-g"), dotnetSrcDir)
-  FileUtils.copyFile(join(baseDirectory.value, "README.md"), join(dotnetSrcDir, "README.md"))
-  runCmd(Seq("sed", "-i", s"""s/img width=\"800\"/img width=\"300\"/g""", "README.md"), dotnetSrcDir)
-  val packageName = name.value.split("-").map(_.capitalize).mkString(" ")
-  val fileContent =
-    s"""PROJECT_NAME = "$packageName"
-       |PROJECT_NUMBER = "${dotnetedVersion(version.value)}"
-       |USE_MDFILE_AS_MAINPAGE = "README.md"
-       |RECURSIVE = YES
-       |""".stripMargin
-  val doxygenHelperFile = join(dotnetSrcDir, "DoxygenHelper.txt")
-  if (doxygenHelperFile.exists()) FileUtils.forceDelete(doxygenHelperFile)
-  FileUtils.writeStringToFile(doxygenHelperFile, fileContent, "utf-8")
-  runCmd(Seq("bash", "-c", "cat DoxygenHelper.txt >> Doxyfile", ""), dotnetSrcDir)
-  runCmd(Seq("doxygen"), dotnetSrcDir)
-}
 
 val packageSynapseML = TaskKey[Unit]("packageSynapseML", "package all projects into SynapseML")
 packageSynapseML := {
@@ -337,11 +198,10 @@ publishPypi := {
   )
 }
 
-val publishDocs = TaskKey[Unit]("publishDocs", "publish docs for scala, python and dotnet")
+val publishDocs = TaskKey[Unit]("publishDocs", "publish docs for scala and python")
 publishDocs := {
   Def.sequential(
     generatePythonDoc,
-    generateDotnetDoc,
     (root / Compile / unidoc)
   ).value
   val html =
@@ -358,9 +218,6 @@ publishDocs := {
   if (scalaDir.exists()) FileUtils.forceDelete(scalaDir)
   FileUtils.copyDirectory(join(targetDir, "unidoc"), scalaDir)
   FileUtils.writeStringToFile(join(unifiedDocDir.toString, "index.html"), html, "utf-8")
-  val dotnetDir = join(unifiedDocDir.toString, "dotnet")
-  if (dotnetDir.exists()) FileUtils.forceDelete(dotnetDir)
-  FileUtils.copyDirectory(join(codegenDir, "src", "dotnet", "html"), dotnetDir)
   uploadToBlob(unifiedDocDir.toString, version.value, "docs")
 }
 

diff --git a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/CognitiveServiceBase.scala b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/CognitiveServiceBase.scala
@@ -224,28 +224,6 @@ trait HasCustomCogServiceDomain extends Wrappable with HasURL with HasUrlPath {
       |""".stripMargin
   }
 
-  override def dotnetAdditionalMethods: String = super.dotnetAdditionalMethods + {
-    s"""/// <summary>
-       |/// Sets value for service name
-       |/// </summary>
-       |/// <param name=\"value\">
-       |/// Service name of the cognitive service if it's custom domain
-       |/// </param>
-       |/// <returns> New $dotnetClassName object </returns>
-       |public $dotnetClassName SetCustomServiceName(string value) =>
-       |    $dotnetClassWrapperName(Reference.Invoke(\"setCustomServiceName\", value));
-       |
-       |/// <summary>
-       |/// Sets value for endpoint
-       |/// </summary>
-       |/// <param name=\"value\">
-       |/// Endpoint of the cognitive service
-       |/// </param>
-       |/// <returns> New $dotnetClassName object </returns>
-       |public $dotnetClassName SetEndpoint(string value) =>
-       |    $dotnetClassWrapperName(Reference.Invoke(\"setEndpoint\", value));
-       |""".stripMargin
-  }
 }
 
 trait HasAPIVersion extends HasServiceParams {
@@ -403,18 +381,6 @@ trait HasSetLinkedService extends Wrappable with HasURL with HasSubscriptionKey
       |""".stripMargin
   }
 
-  override def dotnetAdditionalMethods: String = super.dotnetAdditionalMethods + {
-    s"""/// <summary>
-       |/// Sets value for linkedService
-       |/// </summary>
-       |/// <param name=\"value\">
-       |/// linkedService name
-       |/// </param>
-       |/// <returns> New $dotnetClassName object </returns>
-       |public $dotnetClassName SetLinkedService(string value) =>
-       |    $dotnetClassWrapperName(Reference.Invoke(\"setLinkedService\", value));
-       |""".stripMargin
-  }
 
   def setLinkedService(v: String): this.type = {
     val classPath = "mssparkutils.cognitiveService"
@@ -458,20 +424,6 @@ trait HasSetLocation extends Wrappable with HasURL with HasUrlPath with DomainHe
       |""".stripMargin
   }
 
-  override def dotnetAdditionalMethods: String = super.dotnetAdditionalMethods + {
-    s"""/// <summary>
-       |/// Sets value for location
-       |/// </summary>
-       |/// <param name=\"value\">
-       |/// Location of the cognitive service
-       |/// </param>
-       |/// <returns> New $dotnetClassName object </returns>
-       |public $dotnetClassName SetLocation(string value) =>
-       |    $dotnetClassWrapperName(Reference.Invoke(\"setLocation\", value));
-       |""".stripMargin
-  }
-
-
   def setLocation(v: String): this.type = {
     val domain = getLocationDomain(v)
     setUrl(s"https://$v.api.cognitive.microsoft.$domain/" + urlPath.stripPrefix("/"))

diff --git a/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/text/TextAnalytics.scala b/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/text/TextAnalytics.scala
@@ -465,19 +465,6 @@ class AnalyzeHealthText(override val uid: String)
     super.postprocessResponse(processedResponseOpt.orNull)
   }
 
-  /*
-    override private[ml] def dotnetTestValue(v: Seq[TextAnalyzeTask]): String =
-      v.map(x => s"new TextAnalyzeTask(new Dictionary<string, string>" +
-        s"${DotnetWrappableParam.dotnetDefaultRender(x.parameters)})").mkString(",")
-  */
-
-  /*
-  override def rConstructorLine(v: Seq[TextAnalyzeTask]): String = {
-    val className =  "com.microsoft.azure.synapse.ml.cognitive.text.TextAnalyzeTask"
-    val elements = v.map(x => s"""invoke_new(sc, "${className}", ${RWrappableParam.rDefaultRender(x.parameters)})""")
-    s"${rName(v)}=${elements}".replace("=List(", "=list(")
-  }*/
-
   override def postprocessResponseUdf: UserDefinedFunction = {
     UDFUtils.oldUdf(postprocessResponse _, ArrayType(UnpackedAHTResponse.schema))
   }

diff --git a/...ive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/SearchWriterSuite.scala b/...ive/src/test/scala/com/microsoft/azure/synapse/ml/services/search/SearchWriterSuite.scala
@@ -131,7 +131,9 @@ class SearchWriterSuite extends TestBase with AzureSearchKey with IndexJsonGette
   lazy val indexName: String = generateIndexName()
 
   override def beforeAll(): Unit = {
-    print("WARNING CREATING SEARCH ENGINE!")
+    println("WARNING CREATING SEARCH ENGINE!")
+    println("materializing keys for tests")
+    println((openAIAPIKey + openAIAPIKeyGpt4 + openAIServiceName + cognitiveKey).length)
     SearchIndex.createIfNoneExists(azureSearchKey,
       testServiceName,
       createSimpleIndexJson(indexName))