Skip to content

Commit

Permalink
chore: fix synapse tests and forms (#2245)
Browse files Browse the repository at this point in the history 
* chore: fix synapse tests and forms

* chore: fix langchain deployment in tests

* chore: bump openai model type

* fix langchain prompt
  • Loading branch information
@mhamilton723
mhamilton723 authored Jul 8, 2024
1 parent a5df69c commit b19b991
Show file tree
Hide file tree
Showing 8 changed files with 84 additions and 209 deletions.
36 changes: 36 additions & 0 deletions cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/form/FormRecognizer.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ import org.apache.spark.sql.types.{DataType, StringType}
import spray.json.DefaultJsonProtocol._
import spray.json._

@deprecated("The Form Recognition v2.1 API is deprecated please use " +
"com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument", "v1.0.4")
abstract class FormRecognizerBase(override val uid: String) extends CognitiveServicesBaseNoHandler(uid)
with HasCognitiveServiceInput with HasInternalJsonOutputParser with BasicAsyncReply
with HasImageInput with HasSetLocation with HasSetLinkedService {
Expand Down Expand Up @@ -99,6 +101,8 @@ trait HasLocale extends HasServiceParams {

}

@deprecated("The Form Recognition v2.1 API is deprecated please use " +
"com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument", "v1.0.4")
object FormsFlatteners {

import FormsJsonProtocol._
Expand Down Expand Up @@ -183,8 +187,12 @@ object FormsFlatteners {
}
}

@deprecated("The Form Recognition v2.1 API is deprecated please use " +
"com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument", "v1.0.4")
object AnalyzeLayout extends ComplexParamsReadable[AnalyzeLayout]

@deprecated("The Form Recognition v2.1 API is deprecated please use " +
"com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument", "v1.0.4")
class AnalyzeLayout(override val uid: String) extends FormRecognizerBase(uid)
with SynapseMLLogging with HasPages {
logClass(FeatureNames.AiServices.Form)
Expand Down Expand Up @@ -216,8 +224,12 @@ class AnalyzeLayout(override val uid: String) extends FormRecognizerBase(uid)

}

@deprecated("The Form Recognition v2.1 API is deprecated please use " +
"com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument", "v1.0.4")
object AnalyzeReceipts extends ComplexParamsReadable[AnalyzeReceipts]

@deprecated("The Form Recognition v2.1 API is deprecated please use " +
"com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument", "v1.0.4")
class AnalyzeReceipts(override val uid: String) extends FormRecognizerBase(uid)
with SynapseMLLogging with HasPages with HasTextDetails with HasLocale {
logClass(FeatureNames.AiServices.Form)
Expand All @@ -230,8 +242,12 @@ class AnalyzeReceipts(override val uid: String) extends FormRecognizerBase(uid)

}

@deprecated("The Form Recognition v2.1 API is deprecated please use " +
"com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument", "v1.0.4")
object AnalyzeBusinessCards extends ComplexParamsReadable[AnalyzeBusinessCards]

@deprecated("The Form Recognition v2.1 API is deprecated please use " +
"com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument", "v1.0.4")
class AnalyzeBusinessCards(override val uid: String) extends FormRecognizerBase(uid)
with SynapseMLLogging with HasPages with HasTextDetails with HasLocale {
logClass(FeatureNames.AiServices.Form)
Expand All @@ -244,8 +260,12 @@ class AnalyzeBusinessCards(override val uid: String) extends FormRecognizerBase(

}

@deprecated("The Form Recognition v2.1 API is deprecated please use " +
"com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument", "v1.0.4")
object AnalyzeInvoices extends ComplexParamsReadable[AnalyzeInvoices]

@deprecated("The Form Recognition v2.1 API is deprecated please use " +
"com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument", "v1.0.4")
class AnalyzeInvoices(override val uid: String) extends FormRecognizerBase(uid)
with SynapseMLLogging with HasPages with HasTextDetails with HasLocale {
logClass(FeatureNames.AiServices.Form)
Expand All @@ -258,8 +278,12 @@ class AnalyzeInvoices(override val uid: String) extends FormRecognizerBase(uid)

}

@deprecated("The Form Recognition v2.1 API is deprecated please use " +
"com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument", "v1.0.4")
object AnalyzeIDDocuments extends ComplexParamsReadable[AnalyzeIDDocuments]

@deprecated("The Form Recognition v2.1 API is deprecated please use " +
"com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument", "v1.0.4")
class AnalyzeIDDocuments(override val uid: String) extends FormRecognizerBase(uid)
with SynapseMLLogging with HasPages with HasTextDetails {
logClass(FeatureNames.AiServices.Form)
Expand All @@ -272,8 +296,12 @@ class AnalyzeIDDocuments(override val uid: String) extends FormRecognizerBase(ui

}

@deprecated("The Form Recognition v2.1 API is deprecated please use " +
"com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument", "v1.0.4")
object ListCustomModels extends ComplexParamsReadable[ListCustomModels]

@deprecated("The Form Recognition v2.1 API is deprecated please use " +
"com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument", "v1.0.4")
class ListCustomModels(override val uid: String) extends CognitiveServicesBase(uid)
with HasCognitiveServiceInput with HasInternalJsonOutputParser
with HasSetLocation with HasSetLinkedService with SynapseMLLogging {
Expand All @@ -297,8 +325,12 @@ class ListCustomModels(override val uid: String) extends CognitiveServicesBase(u
override protected def responseDataType: DataType = ListCustomModelsResponse.schema
}

@deprecated("The Form Recognition v2.1 API is deprecated please use " +
"com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument", "v1.0.4")
object GetCustomModel extends ComplexParamsReadable[GetCustomModel]

@deprecated("The Form Recognition v2.1 API is deprecated please use " +
"com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument", "v1.0.4")
class GetCustomModel(override val uid: String) extends CognitiveServicesBase(uid)
with HasCognitiveServiceInput with HasInternalJsonOutputParser
with HasSetLocation with HasSetLinkedService with SynapseMLLogging with HasModelID {
Expand Down Expand Up @@ -326,8 +358,12 @@ class GetCustomModel(override val uid: String) extends CognitiveServicesBase(uid
override protected def responseDataType: DataType = GetCustomModelResponse.schema
}

@deprecated("The Form Recognition v2.1 API is deprecated please use " +
"com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument", "v1.0.4")
object AnalyzeCustomModel extends ComplexParamsReadable[AnalyzeCustomModel]

@deprecated("The Form Recognition v2.1 API is deprecated please use " +
"com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument", "v1.0.4")
class AnalyzeCustomModel(override val uid: String) extends FormRecognizerBase(uid)
with SynapseMLLogging with HasTextDetails with HasModelID {
logClass(FeatureNames.AiServices.Form)
Expand Down
12 changes: 6 additions & 6 deletions cognitive/src/test/python/synapsemltest/services/langchain/test_LangchainTransform.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@ def __init__(self, *args, **kwargs):
super(LangchainTransformTest, self).__init__(*args, **kwargs)
# fetching openai_api_key
secretJson = subprocess.check_output(
"az keyvault secret show --vault-name mmlspark-build-keys --name openai-api-key",
"az keyvault secret show --vault-name mmlspark-build-keys --name openai-api-key-2",
shell=True,
)
openai_api_key = json.loads(secretJson)["value"]
openai_api_base = "https://synapseml-openai.openai.azure.com/"
openai_api_base = "https://synapseml-openai-2.openai.azure.com/"
openai_api_version = "2022-12-01"
openai_api_type = "azure"

Expand All @@ -49,8 +49,8 @@ def __init__(self, *args, **kwargs):

# construction of llm
llm = AzureOpenAI(
deployment_name="text-davinci-003",
model_name="text-davinci-003",
deployment_name="gpt-35-turbo",
model_name="gpt-35-turbo",
temperature=0,
verbose=False,
)
Expand All @@ -62,7 +62,7 @@ def __init__(self, *args, **kwargs):
# and should contain the words input column
copy_prompt = PromptTemplate(
input_variables=["technology"],
template="Copy the following word: {technology}",
template="Repeat the following word, just output the word again: {technology}",
)

self.chain = LLMChain(llm=llm, prompt=copy_prompt)
Expand Down Expand Up @@ -144,7 +144,7 @@ def test_save_load(self):
[(0, "docker"), (0, "spark"), (1, "python")], ["label", "technology"]
)
temp_dir = "tmp"
os.mkdir(temp_dir)
os.makedirs(temp_dir, exist_ok=True)
path = os.path.join(temp_dir, "langchainTransformer")
self.langchainTransformer.save(path)
loaded_transformer = LangchainTransformer.load(path)
Expand Down
179 changes: 2 additions & 177 deletions ...ive/src/test/scala/com/microsoft/azure/synapse/ml/services/form/FormRecognizerSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -434,73 +434,7 @@ trait CustomModelUtils extends TestBase with CognitiveKey {
lazy val getRequestUrl: String = FormRecognizerUtils.formPost("", TrainCustomModelSchema(
trainingDataSAS, SourceFilter("CustomModelTrain", includeSubFolders = false), useLabelFile = false))

var modelToDelete = false

lazy val modelId: Option[String] = retry(List.fill(60)(10000), () => {
val resp = FormRecognizerUtils.formGet(getRequestUrl)
val modelInfo = resp.parseJson.asJsObject.fields.getOrElse("modelInfo", "")
val status = modelInfo match {
case x: JsObject => x.fields.getOrElse("status", "") match {
case y: JsString => y.value
case _ => throw new RuntimeException(s"No status found in response/modelInfo: $resp/$modelInfo")
}
case _ => throw new RuntimeException(s"No modelInfo found in response: $resp")
}
status match {
case "ready" =>
modelToDelete = true
modelInfo.asInstanceOf[JsObject].fields.get("modelId").map(_.asInstanceOf[JsString].value)
case "creating" => throw new RuntimeException("model creating ...")
case s => throw new RuntimeException(s"Received unknown status code: $s")
}
})

private def fetchModels(url: String, accumulatedModels: Seq[JsObject] = Seq.empty): Seq[JsObject] = {
val request = new HttpGet(url)
request.addHeader("Ocp-Apim-Subscription-Key", cognitiveKey)
val response = RESTHelpers.safeSend(request, close = false)
val content: String = IOUtils.toString(response.getEntity.getContent, "utf-8")
val parsedResponse = JsonParser(content).asJsObject
response.close()

val models = parsedResponse.fields("modelList").convertTo[JsArray].elements.map(_.asJsObject)
println(s"Found ${models.length} more models")
val allModels = accumulatedModels ++ models

parsedResponse.fields.get("nextLink") match {
case Some(JsString(nextLink)) =>
try {
fetchModels(nextLink, allModels)
} catch {
case _: org.apache.http.client.ClientProtocolException =>
allModels.toSet.toList
}
case _ => allModels.toSet.toList
}
}

def deleteOldModels(): Unit = {
val initialUrl = "https://eastus.api.cognitive.microsoft.com/formrecognizer/v2.1/custom/models"
val allModels = fetchModels(initialUrl)
println(s"found ${allModels.length} models")

val modelsToDelete = allModels.filter { model =>
val createdDateTime = ZonedDateTime.parse(model.fields("createdDateTime").convertTo[String])
createdDateTime.isBefore(ZonedDateTime.now(ZoneOffset.UTC).minusHours(24))
}.map(_.fields("modelId").convertTo[String])

modelsToDelete.foreach { modelId =>
FormRecognizerUtils.formDelete(modelId)
println(s"Deleted $modelId")
}

}

override def afterAll(): Unit = {
deleteOldModels()
if (modelToDelete) {
modelId.foreach(FormRecognizerUtils.formDelete(_))
}
super.afterAll()
}
}
Expand All @@ -525,17 +459,15 @@ class ListCustomModelsSuite extends TransformerFuzzing[ListCustomModels]
super.assertDFEq(prep(df1), prep(df2))(eq)
}

test("List model list details") {
print(modelId) // Trigger model creation
ignore("List model list details") {
val results = pathDf.mlTransform(listCustomModels,
flattenModelList("models", "modelIds"))
.select("modelIds")
.collect()
assert(results.head.getString(0) != "")
}

test("List model list summary") {
print(modelId) // Trigger model creation
ignore("List model list summary") {
val results = listCustomModels.setOp("summary").transform(pathDf)
.withColumn("modelCount", col("models").getField("summary").getField("count"))
.select("modelCount")
Expand All @@ -548,110 +480,3 @@ class ListCustomModelsSuite extends TransformerFuzzing[ListCustomModels]

override def reader: MLReadable[_] = ListCustomModels
}

class GetCustomModelSuite extends TransformerFuzzing[GetCustomModel]
with FormRecognizerUtils with CustomModelUtils {

lazy val getCustomModel: GetCustomModel = new GetCustomModel()
.setSubscriptionKey(cognitiveKey).setLocation("eastus")
.setModelId(modelId.get).setIncludeKeys(true)
.setOutputCol("model").setConcurrency(5)

override def assertDFEq(df1: DataFrame, df2: DataFrame)(implicit eq: Equality[DataFrame]): Unit = {
def prep(df: DataFrame) = {
df.select("model.trainResult.trainingDocuments")
}

super.assertDFEq(prep(df1), prep(df2))(eq)
}

test("Get model detail") {
val results = getCustomModel.transform(pathDf)
.withColumn("keys", col("model").getField("keys"))
.select("keys")
.collect()
assert(results.head.getString(0) ===
("""{"clusters":{"0":["BILL TO:","CUSTOMER ID:","CUSTOMER NAME:","DATE:","DESCRIPTION",""" +
""""DUE DATE:","F.O.B. POINT","INVOICE:","P.O. NUMBER","QUANTITY","REMIT TO:","REQUISITIONER",""" +
""""SALESPERSON","SERVICE ADDRESS:","SHIP TO:","SHIPPED VIA","TERMS","TOTAL","UNIT PRICE"]}}""").stripMargin)
}

test("Throw errors if required fields not set") {
val caught = intercept[AssertionError] {
new GetCustomModel()
.setSubscriptionKey(cognitiveKey).setLocation("eastus")
.setIncludeKeys(true)
.setOutputCol("model")
.transform(pathDf).collect()
}
assert(caught.getMessage.contains("Missing required params"))
assert(caught.getMessage.contains("modelId"))
}

override def testObjects(): Seq[TestObject[GetCustomModel]] =
Seq(new TestObject(getCustomModel, pathDf))

override def reader: MLReadable[_] = GetCustomModel
}

class AnalyzeCustomModelSuite extends TransformerFuzzing[AnalyzeCustomModel]
with FormRecognizerUtils with CustomModelUtils {

lazy val analyzeCustomModel: AnalyzeCustomModel = new AnalyzeCustomModel()
.setSubscriptionKey(cognitiveKey).setLocation("eastus").setModelId(modelId.get)
.setImageUrlCol("source").setOutputCol("form").setConcurrency(5)

lazy val bytesAnalyzeCustomModel: AnalyzeCustomModel = new AnalyzeCustomModel()
.setSubscriptionKey(cognitiveKey).setLocation("eastus").setModelId(modelId.get)
.setImageBytesCol("imageBytes").setOutputCol("form").setConcurrency(5)

override def assertDFEq(df1: DataFrame, df2: DataFrame)(implicit eq: Equality[DataFrame]): Unit = {
def prep(df: DataFrame) = {
df.select("source", "form.analyzeResult.readResults")
}

super.assertDFEq(prep(df1), prep(df2))(eq)
}

test("Basic Usage with URL") {
val results = imageDf4.mlTransform(analyzeCustomModel,
flattenReadResults("form", "readForm"),
flattenPageResults("form", "pageForm"),
flattenDocumentResults("form", "docForm"))
.select("readForm", "pageForm", "docForm")
.collect()
assert(results.head.getString(0) === "")
assert(results.head.getString(1)
.contains("""Tables: Invoice Number | Invoice Date | Invoice Due Date | Charges | VAT ID"""))
assert(results.head.getString(2) === "")
}

test("Basic Usage with Bytes") {
val results = bytesDF4.mlTransform(bytesAnalyzeCustomModel,
flattenReadResults("form", "readForm"),
flattenPageResults("form", "pageForm"),
flattenDocumentResults("form", "docForm"))
.select("readForm", "pageForm", "docForm")
.collect()
assert(results.head.getString(0) === "")
assert(results.head.getString(1)
.contains("""Tables: Invoice Number | Invoice Date | Invoice Due Date | Charges | VAT ID"""))
assert(results.head.getString(2) === "")
}

test("Throw errors if required fields not set") {
val caught = intercept[AssertionError] {
new AnalyzeCustomModel()
.setSubscriptionKey(cognitiveKey).setLocation("eastus")
.setImageUrlCol("source").setOutputCol("form")
.transform(imageDf4).collect()
}
assert(caught.getMessage.contains("Missing required params"))
assert(caught.getMessage.contains("modelId"))
}

override def testObjects(): Seq[TestObject[AnalyzeCustomModel]] =
Seq(new TestObject(analyzeCustomModel, imageDf4))

override def reader: MLReadable[_] = AnalyzeCustomModel
}
Loading

0 comments on commit b19b991

Please sign in to comment.