chore: fix errors in build pipeline

cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/form/FormRecognizerV3.scala

@@ -40,7 +40,7 @@ class AnalyzeDocument(override val uid: String) extends CognitiveServicesBaseNoH
   with HasImageInput with HasSetLocation with SynapseMLLogging with HasSetLinkedService {
   logClass(FeatureNames.AiServices.Anomaly)
 
-  setDefault(apiVersion -> Left("2022-08-31"))
+  setDefault(apiVersion -> Left("2023-07-31"))
 
   def this() = this(Identifiable.randomUID("AnalyzeDocument"))
 
@@ -60,6 +60,30 @@ class AnalyzeDocument(override val uid: String) extends CognitiveServicesBaseNoH
 
   def getStringIndexTypeCol: String = getVectorParam(stringIndexType)
 
+
+  val features = new ServiceParam[Seq[String]](this, "features",
+    "List of optional analysis features. (barcodes,formulas,keyValuePairs,languages,ocrHighResolution,styleFont)",
+    {
+    case Left(s) => s.forall(entry => Set(
+      "barcodes",
+      "formulas",
+      "keyValuePairs",
+      "languages",
+      "ocrHighResolution",
+      "styleFont"
+    )(entry))
+    case Right(_) => true
+  }, isURLParam = true)
+
+  def setFeatures(v: Seq[String]): this.type = setScalarParam(features, v)
+
+  def setFeaturesCol(v: String): this.type = setVectorParam(features, v)
+
+  def getFeatures: Seq[String] = getScalarParam(features)
+
+  def getFeaturesCol: String = getVectorParam(features)
+
+
   override protected def responseDataType: DataType = AnalyzeDocumentResponse.schema
 
   override protected def prepareEntity: Row => Option[AbstractHttpEntity] = {

cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/form/FormRecognizerV3Schemas.scala

@@ -35,7 +35,8 @@ case class PageResultV3(pageNumber: Int,
                         spans: Seq[FormSpan],
                         words: Option[Seq[FormWord]],
                         selectionMarks: Option[Seq[FormSelectionMark]],
-                        lines: Option[Seq[FormLine]])
+                        lines: Option[Seq[FormLine]],
+                        barcodes: Option[Seq[FormBarcode]])
 
 case class DocumentParagraph(role: Option[String],
                              content: String,
@@ -50,6 +51,12 @@ case class FormSelectionMark(state: String, polygon: Option[Seq[Double]], confid
 
 case class FormLine(content: String, polygon: Option[Seq[Double]], spans: Option[Seq[FormSpan]])
 
+case class FormBarcode(confidence: Option[Double],
+                       kind: Option[String],
+                       polygon: Option[Seq[Double]],
+                       span: Option[FormSpan],
+                       value: Option[String])
+
 case class TableResultV3(rowCount: Int,
                          columnCount: Int,
                          boundingRegions: Option[Seq[BoundingRegion]],

cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/CognitiveServicesCommon.scala

@@ -7,4 +7,5 @@ import com.microsoft.azure.synapse.ml.Secrets
 
 trait CognitiveKey {
   lazy val cognitiveKey = sys.env.getOrElse("COGNITIVE_API_KEY", Secrets.CognitiveApiKey)
+  lazy val cognitiveLoc = sys.env.getOrElse("COGNITIVE_API_LOC", "eastus")
 }

cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/FormRecognizerSuite.scala

@@ -3,14 +3,15 @@
 
 package com.microsoft.azure.synapse.ml.services.form
 
-import com.microsoft.azure.synapse.ml.services._
-import com.microsoft.azure.synapse.ml.services.bing.BingImageSearch
-import com.microsoft.azure.synapse.ml.services.form.FormsFlatteners._
 import com.microsoft.azure.synapse.ml.core.env.StreamUtilities.using
 import com.microsoft.azure.synapse.ml.core.spark.FluentAPI._
 import com.microsoft.azure.synapse.ml.core.test.base.{Flaky, TestBase}
 import com.microsoft.azure.synapse.ml.core.test.fuzzing.{TestObject, TransformerFuzzing}
+import com.microsoft.azure.synapse.ml.io.http.RESTHelpers
 import com.microsoft.azure.synapse.ml.io.http.RESTHelpers.retry
+import com.microsoft.azure.synapse.ml.services._
+import com.microsoft.azure.synapse.ml.services.bing.BingImageSearch
+import com.microsoft.azure.synapse.ml.services.form.FormsFlatteners._
 import com.microsoft.azure.synapse.ml.stages.UDFTransformer
 import org.apache.commons.io.IOUtils
 import org.apache.http.client.methods._
@@ -23,6 +24,8 @@ import org.scalactic.Equality
 import spray.json._
 
 import java.net.URI
+import java.time.{ZoneOffset, ZonedDateTime}
+import scala.annotation.tailrec
 
 object TrainCustomModelProtocol extends DefaultJsonProtocol {
   implicit val SourceFilterEnc: RootJsonFormat[SourceFilter] = jsonFormat2(SourceFilter)
@@ -173,8 +176,8 @@ class AnalyzeLayoutSuite extends TransformerFuzzing[AnalyzeLayout] with FormReco
 
   test("Basic Usage with URL") {
     val results = imageDf1.mlTransform(analyzeLayout,
-      flattenReadResults("layout", "readlayout"),
-      flattenPageResults("layout", "pageLayout"))
+        flattenReadResults("layout", "readlayout"),
+        flattenPageResults("layout", "pageLayout"))
       .select("readlayout", "pageLayout")
       .collect()
     val headStr = results.head.getString(0)
@@ -186,8 +189,8 @@ class AnalyzeLayoutSuite extends TransformerFuzzing[AnalyzeLayout] with FormReco
 
   test("Basic Usage with pdf") {
     val results = pdfDf1.mlTransform(analyzeLayout,
-      flattenReadResults("layout", "readlayout"),
-      flattenPageResults("layout", "pageLayout"))
+        flattenReadResults("layout", "readlayout"),
+        flattenPageResults("layout", "pageLayout"))
       .select("readlayout", "pageLayout")
       .collect()
     val headStr = results.head.getString(0)
@@ -199,8 +202,8 @@ class AnalyzeLayoutSuite extends TransformerFuzzing[AnalyzeLayout] with FormReco
 
   test("Basic Usage with Bytes") {
     val results = bytesDF1.mlTransform(bytesAnalyzeLayout,
-      flattenReadResults("layout", "readlayout"),
-      flattenPageResults("layout", "pageLayout"))
+        flattenReadResults("layout", "readlayout"),
+        flattenPageResults("layout", "pageLayout"))
       .select("readlayout", "pageLayout")
       .collect()
     val headStr = results.head.getString(0)
@@ -237,8 +240,8 @@ class AnalyzeReceiptsSuite extends TransformerFuzzing[AnalyzeReceipts] with Form
 
   test("Basic Usage with URL") {
     val results = imageDf2.mlTransform(analyzeReceipts,
-      flattenReadResults("receipts", "readReceipts"),
-      flattenDocumentResults("receipts", "docReceipts"))
+        flattenReadResults("receipts", "readReceipts"),
+        flattenDocumentResults("receipts", "docReceipts"))
       .select("readReceipts", "docReceipts")
       .collect()
     val headStr = results.head.getString(0)
@@ -249,8 +252,8 @@ class AnalyzeReceiptsSuite extends TransformerFuzzing[AnalyzeReceipts] with Form
 
   test("Basic Usage with Bytes") {
     val results = bytesDF2.mlTransform(bytesAnalyzeReceipts,
-      flattenReadResults("receipts", "readReceipts"),
-      flattenDocumentResults("receipts", "docReceipts"))
+        flattenReadResults("receipts", "readReceipts"),
+        flattenDocumentResults("receipts", "docReceipts"))
       .select("readReceipts", "docReceipts")
       .collect()
     val headStr = results.head.getString(0)
@@ -285,8 +288,8 @@ class AnalyzeBusinessCardsSuite extends TransformerFuzzing[AnalyzeBusinessCards]
 
   test("Basic Usage with URL") {
     val results = imageDf3.mlTransform(analyzeBusinessCards,
-      flattenReadResults("businessCards", "readBusinessCards"),
-      flattenDocumentResults("businessCards", "docBusinessCards"))
+        flattenReadResults("businessCards", "readBusinessCards"),
+        flattenDocumentResults("businessCards", "docBusinessCards"))
       .select("readBusinessCards", "docBusinessCards")
       .collect()
     val headStr = results.head.getString(0)
@@ -298,8 +301,8 @@ class AnalyzeBusinessCardsSuite extends TransformerFuzzing[AnalyzeBusinessCards]
 
   test("Basic Usage with Bytes") {
     val results = bytesDF3.mlTransform(bytesAnalyzeBusinessCards,
-      flattenReadResults("businessCards", "readBusinessCards"),
-      flattenDocumentResults("businessCards", "docBusinessCards"))
+        flattenReadResults("businessCards", "readBusinessCards"),
+        flattenDocumentResults("businessCards", "docBusinessCards"))
       .select("readBusinessCards", "docBusinessCards")
       .collect()
     val headStr = results.head.getString(0)
@@ -335,8 +338,8 @@ class AnalyzeInvoicesSuite extends TransformerFuzzing[AnalyzeInvoices] with Form
 
   test("Basic Usage with URL") {
     val results = imageDf4.mlTransform(analyzeInvoices,
-      flattenReadResults("invoices", "readInvoices"),
-      flattenDocumentResults("invoices", "docInvoices"))
+        flattenReadResults("invoices", "readInvoices"),
+        flattenDocumentResults("invoices", "docInvoices"))
       .select("readInvoices", "docInvoices")
       .collect()
     val headStr = results.head.getString(0)
@@ -347,8 +350,8 @@ class AnalyzeInvoicesSuite extends TransformerFuzzing[AnalyzeInvoices] with Form
 
   test("Basic Usage with pdf") {
     val results = pdfDf2.mlTransform(analyzeInvoices,
-      flattenReadResults("invoices", "readInvoices"),
-      flattenDocumentResults("invoices", "docInvoices"))
+        flattenReadResults("invoices", "readInvoices"),
+        flattenDocumentResults("invoices", "docInvoices"))
       .select("readInvoices", "docInvoices")
       .collect()
     val headStr = results.head.getString(0)
@@ -359,8 +362,8 @@ class AnalyzeInvoicesSuite extends TransformerFuzzing[AnalyzeInvoices] with Form
 
   test("Basic Usage with Bytes") {
     val results = bytesDF4.mlTransform(bytesAnalyzeInvoices,
-      flattenReadResults("invoices", "readInvoices"),
-      flattenDocumentResults("invoices", "docInvoices"))
+        flattenReadResults("invoices", "readInvoices"),
+        flattenDocumentResults("invoices", "docInvoices"))
       .select("readInvoices", "docInvoices")
       .collect()
     val headStr = results.head.getString(0)
@@ -395,8 +398,8 @@ class AnalyzeIDDocumentsSuite extends TransformerFuzzing[AnalyzeIDDocuments] wit
 
   test("Basic Usage with URL") {
     val results = imageDf5.mlTransform(analyzeIDDocuments,
-      flattenReadResults("ids", "readIds"),
-      flattenDocumentResults("ids", "docIds"))
+        flattenReadResults("ids", "readIds"),
+        flattenDocumentResults("ids", "docIds"))
       .select("readIds", "docIds")
       .collect()
     val headStr = results.head.getString(0)
@@ -407,8 +410,8 @@ class AnalyzeIDDocumentsSuite extends TransformerFuzzing[AnalyzeIDDocuments] wit
 
   test("Basic Usage with Bytes") {
     val results = bytesDF5.mlTransform(bytesAnalyzeIDDocuments,
-      flattenReadResults("ids", "readIds"),
-      flattenDocumentResults("ids", "docIds"))
+        flattenReadResults("ids", "readIds"),
+        flattenDocumentResults("ids", "docIds"))
       .select("readIds", "docIds")
       .collect()
     val headStr = results.head.getString(0)
@@ -424,7 +427,7 @@ class AnalyzeIDDocumentsSuite extends TransformerFuzzing[AnalyzeIDDocuments] wit
   override def reader: MLReadable[_] = AnalyzeIDDocuments
 }
 
-trait CustomModelUtils extends TestBase {
+trait CustomModelUtils extends TestBase with CognitiveKey {
 
   lazy val trainingDataSAS: String = "https://mmlspark.blob.core.windows.net/datasets"
 
@@ -433,7 +436,7 @@ trait CustomModelUtils extends TestBase {
 
   var modelToDelete = false
 
-  lazy val modelId: Option[String] = retry(List(10000, 20000, 30000), () => {
+  lazy val modelId: Option[String] = retry(List.fill(60)(10000), () => {
     val resp = FormRecognizerUtils.formGet(getRequestUrl)
     val modelInfo = resp.parseJson.asJsObject.fields.getOrElse("modelInfo", "")
     val status = modelInfo match {
@@ -452,7 +455,49 @@ trait CustomModelUtils extends TestBase {
     }
   })
 
+  private def fetchModels(url: String, accumulatedModels: Seq[JsObject] = Seq.empty): Seq[JsObject] = {
+    val request = new HttpGet(url)
+    request.addHeader("Ocp-Apim-Subscription-Key", cognitiveKey)
+    val response = RESTHelpers.safeSend(request, close = false)
+    val content: String = IOUtils.toString(response.getEntity.getContent, "utf-8")
+    val parsedResponse = JsonParser(content).asJsObject
+    response.close()
+
+    val models = parsedResponse.fields("modelList").convertTo[JsArray].elements.map(_.asJsObject)
+    println(s"Found ${models.length} more models")
+    val allModels = accumulatedModels ++ models
+
+    parsedResponse.fields.get("nextLink") match {
+      case Some(JsString(nextLink)) =>
+        try {
+          fetchModels(nextLink, allModels)
+        } catch {
+          case _: org.apache.http.client.ClientProtocolException =>
+            allModels.toSet.toList
+        }
+      case _ => allModels.toSet.toList
+    }
+  }
+
+  def deleteOldModels(): Unit = {
+    val initialUrl = "https://eastus.api.cognitive.microsoft.com/formrecognizer/v2.1/custom/models"
+    val allModels = fetchModels(initialUrl)
+    println(s"found ${allModels.length} models")
+
+    val modelsToDelete = allModels.filter { model =>
+      val createdDateTime = ZonedDateTime.parse(model.fields("createdDateTime").convertTo[String])
+      createdDateTime.isBefore(ZonedDateTime.now(ZoneOffset.UTC).minusHours(24))
+    }.map(_.fields("modelId").convertTo[String])
+
+    modelsToDelete.foreach { modelId =>
+      FormRecognizerUtils.formDelete(modelId)
+      println(s"Deleted $modelId")
+    }
+
+  }
+
   override def afterAll(): Unit = {
+    deleteOldModels()
     if (modelToDelete) {
       modelId.foreach(FormRecognizerUtils.formDelete(_))
     }
@@ -483,7 +528,7 @@ class ListCustomModelsSuite extends TransformerFuzzing[ListCustomModels]
   test("List model list details") {
     print(modelId) // Trigger model creation
     val results = pathDf.mlTransform(listCustomModels,
-      flattenModelList("models", "modelIds"))
+        flattenModelList("models", "modelIds"))
       .select("modelIds")
       .collect()
     assert(results.head.getString(0) != "")
@@ -570,9 +615,9 @@ class AnalyzeCustomModelSuite extends TransformerFuzzing[AnalyzeCustomModel]
 
   test("Basic Usage with URL") {
     val results = imageDf4.mlTransform(analyzeCustomModel,
-      flattenReadResults("form", "readForm"),
-      flattenPageResults("form", "pageForm"),
-      flattenDocumentResults("form", "docForm"))
+        flattenReadResults("form", "readForm"),
+        flattenPageResults("form", "pageForm"),
+        flattenDocumentResults("form", "docForm"))
       .select("readForm", "pageForm", "docForm")
       .collect()
     assert(results.head.getString(0) === "")
@@ -583,9 +628,9 @@ class AnalyzeCustomModelSuite extends TransformerFuzzing[AnalyzeCustomModel]
 
   test("Basic Usage with Bytes") {
     val results = bytesDF4.mlTransform(bytesAnalyzeCustomModel,
-      flattenReadResults("form", "readForm"),
-      flattenPageResults("form", "pageForm"),
-      flattenDocumentResults("form", "docForm"))
+        flattenReadResults("form", "readForm"),
+        flattenPageResults("form", "pageForm"),
+        flattenDocumentResults("form", "docForm"))
       .select("readForm", "pageForm", "docForm")
       .collect()
     assert(results.head.getString(0) === "")

cognitive/src/test/scala/com/microsoft/azure/synapse/ml/services/translate/TranslatorSuite.scala

@@ -24,7 +24,7 @@ trait TranslatorUtils extends TestBase {
 
   lazy val textDf1: DataFrame = Seq(List("Bye")).toDF("text")
 
-  lazy val  textDf2: DataFrame = Seq(List("Good morning", "Bye")).toDF("text")
+  lazy val textDf2: DataFrame = Seq(List("Good morning", "Bye")).toDF("text")
 
   lazy val textDf3: DataFrame = Seq(List("This is fucked.")).toDF("text")
 
@@ -35,7 +35,7 @@ trait TranslatorUtils extends TestBase {
     "or phrase</mstrans:dictionary> is a dictionary entry.")).toDF("text")
 
   lazy val textDf6: DataFrame = Seq(("Hi, this is Synapse!", "zh-Hans"),
-    (null, "zh-Hans"), ("test", null))  //scalastyle:ignore null
+    (null, "zh-Hans"), ("test", null)) //scalastyle:ignore null
     .toDF("text", "language")
 
   lazy val emptyDf: DataFrame = Seq("").toDF()
@@ -53,7 +53,7 @@ class TranslateSuite extends TransformerFuzzing[Translate]
     .setConcurrency(5)
 
   def getTranslationTextResult(translator: Translate,
-                          df: DataFrame): DataFrame = {
+                               df: DataFrame): DataFrame = {
     translator
       .transform(df)
       .withColumn("translation", flatten(col("translation.translations")))
@@ -190,8 +190,8 @@ class TransliterateSuite extends TransformerFuzzing[Transliterate]
       .withColumn("script", col("result.script"))
       .select("text", "script").collect()
 
-    assert(TransliterateSuite.stripInvalid(results.head.getSeq(0).mkString("\n")) === "Kon'nichiwa\nsayonara")
-    assert(TransliterateSuite.stripInvalid(results.head.getSeq(1).mkString("\n")) === "Latn\nLatn")
+    assert(TransliterateSuite.stripInvalid(results.head.getSeq(0).mkString("\n")).contains("Kon'nichiwa"))
+    assert(TransliterateSuite.stripInvalid(results.head.getSeq(1).mkString("\n")).contains("Latn"))
   }
 
   test("Throw errors if required fields not set") {
@@ -213,6 +213,7 @@ class TransliterateSuite extends TransformerFuzzing[Transliterate]
       o.map(t => (TransliterateSuite.stripInvalid(t._1), t._2))
     }
   }
+
   override def assertDFEq(df1: DataFrame, df2: DataFrame)(implicit eq: Equality[DataFrame]): Unit = {
     val column = "result"
     super.assertDFEq(