From 62a23dc91f5168be50919da26cc688e876cbc8ba Mon Sep 17 00:00:00 2001 From: Roslyn Lu Date: Thu, 24 Mar 2022 20:10:06 -0700 Subject: [PATCH] [Stream Analytics] Fix s360 for 2021 preview version (#18305) * Add blockchain to latest profile * Add additional types * add azure function * add authenticationMode to blob input, AzFunction and DW ex files * update api version in 2021 preview ex files * fix api version * fix password * try to run model validation again * update pe to readonly * 2021 preview features * fix validation errors * more fixes * add aggregate func, which should be GA * rename new refreshType object Co-authored-by: Mark Cowlishaw Co-authored-by: Roslyn Lu --- .../2020-03-01-preview/privateEndpoints.json | 3 +- .../Function_Create_AzureMLService.json | 4 +- .../examples/Function_Create_CSharp.json | 3 +- .../examples/Input_Create_EventGrid.json | 122 ++++++++++++ .../Input_Create_GatewayMessageBus.json | 60 ++++++ .../Input_Create_Reference_Blob_CSV.json | 7 +- .../Input_Create_Stream_EventHub_JSON.json | 3 + .../examples/Output_Create_Blob_CSV.json | 4 +- .../examples/Output_Create_DataWarehouse.json | 3 +- .../examples/Output_Create_DocumentDB.json | 3 +- .../examples/Output_Create_EventHub_JSON.json | 4 + .../Output_Create_GatewayMessageBus.json | 57 ++++++ .../examples/Output_Create_PostgreSQL.json | 73 +++++++ .../examples/Output_Get_EventHub_JSON.json | 12 +- .../StreamingJob_Create_CompleteJob.json | 17 +- .../preview/2021-10-01-preview/functions.json | 27 ++- .../preview/2021-10-01-preview/inputs.json | 175 ++++++++++++++++- .../preview/2021-10-01-preview/outputs.json | 180 +++++++++++++++++- .../2021-10-01-preview/streamingjobs.json | 48 ++++- .../stable/2020-03-01/functions.json | 56 +++--- 20 files changed, 812 insertions(+), 49 deletions(-) create mode 100644 specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Input_Create_EventGrid.json create mode 100644 specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Input_Create_GatewayMessageBus.json create mode 100644 specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_GatewayMessageBus.json create mode 100644 specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_PostgreSQL.json diff --git a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2020-03-01-preview/privateEndpoints.json b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2020-03-01-preview/privateEndpoints.json index f090e917309e..2372f93eb1a8 100644 --- a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2020-03-01-preview/privateEndpoints.json +++ b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2020-03-01-preview/privateEndpoints.json @@ -298,7 +298,8 @@ }, "requestMessage": { "description": "A message passed to the owner of the remote resource with this connection request. Restricted to 140 chars.", - "type": "string" + "type": "string", + "readOnly": true }, "privateLinkServiceConnectionState": { "description": "A collection of read-only information about the state of the connection to the private remote resource.", diff --git a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Function_Create_AzureMLService.json b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Function_Create_AzureMLService.json index 0104db4f7310..0f0fd43c9ac2 100644 --- a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Function_Create_AzureMLService.json +++ b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Function_Create_AzureMLService.json @@ -36,7 +36,9 @@ } ], "batchSize": 1000, - "numberOfParallelRequests": 1 + "numberOfParallelRequests": 1, + "inputRequestName": "Inputs", + "outputResponseName": "Results" } } } diff --git a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Function_Create_CSharp.json b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Function_Create_CSharp.json index 15a815cb4b6c..32f5737a20ea 100644 --- a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Function_Create_CSharp.json +++ b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Function_Create_CSharp.json @@ -22,7 +22,8 @@ "properties": { "dllPath": "ASAEdgeApplication2_CodeBehind", "class": "ASAEdgeUDFDemo.Class1", - "method": "SquareFunction" + "method": "SquareFunction", + "updateMode": "Static" } } } diff --git a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Input_Create_EventGrid.json b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Input_Create_EventGrid.json new file mode 100644 index 000000000000..9e21f4c93412 --- /dev/null +++ b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Input_Create_EventGrid.json @@ -0,0 +1,122 @@ +{ + "parameters": { + "api-version": "2021-10-01-preview", + "subscriptionId": "56b5e0a9-b645-407d-99b0-c64f86013e3d", + "resourceGroupName": "sjrg3467", + "jobName": "sj9742", + "inputName": "input7970", + "input": { + "properties": { + "type": "Stream", + "datasource": { + "type": "Microsoft.EventGrid/EventSubscriptions", + "properties": { + "subscriber": { + "type": "Microsoft.EventHub/EventHub", + "properties": { + "serviceBusNamespace": "sdktest", + "sharedAccessPolicyName": "RootManageSharedAccessKey", + "sharedAccessPolicyKey": "someSharedAccessPolicyKey==", + "consumerGroupName": "sdkconsumergroup", + "eventHubName": "sdkeventhub", + "partitionCount": 16, + "authenticationMode": "Msi" + } + }, + "schema": "CloudEventSchema", + "storageAccounts": [ + { + "authenticationMode": "Msi", + "accountName": "myaccount", + "accountKey": "myaccountkey" + } + ], + "eventTypes": [ + "Microsoft.Storage.BlobCreated" + ] + } + } + } + } + }, + "responses": { + "201": { + "headers": { + "ETag": "bd8009b3-8165-4bd3-aad2-29a2b378dd14" + }, + "body": { + "id": "/subscriptions/56b5e0a9-b645-407d-99b0-c64f86013e3d/resourceGroups/sjrg3467/providers/Microsoft.StreamAnalytics/streamingjobs/sj9742/inputs/input7970", + "name": "input7970", + "type": "Microsoft.StreamAnalytics/streamingjobs/inputs", + "properties": { + "type": "Stream", + "datasource": { + "type": "Microsoft.EventGrid/EventSubscriptions", + "properties": { + "subscriber": { + "type": "Microsoft.EventHub/EventHub", + "properties": { + "serviceBusNamespace": "sdktest", + "sharedAccessPolicyName": "RootManageSharedAccessKey", + "consumerGroupName": "sdkconsumergroup", + "eventHubName": "sdkeventhub", + "partitionCount": 16, + "authenticationMode": "Msi" + } + }, + "schema": "CloudEventSchema", + "storageAccounts": [ + { + "authenticationMode": "Msi", + "accountName": "myaccount" + } + ], + "eventTypes": [ + "Microsoft.Storage.BlobCreated" + ] + } + } + } + } + }, + "200": { + "headers": { + "ETag": "bd8009b3-8165-4bd3-aad2-29a2b378dd14" + }, + "body": { + "id": "/subscriptions/56b5e0a9-b645-407d-99b0-c64f86013e3d/resourceGroups/sjrg3467/providers/Microsoft.StreamAnalytics/streamingjobs/sj9742/inputs/input7970", + "name": "input7970", + "type": "Microsoft.StreamAnalytics/streamingjobs/inputs", + "properties": { + "type": "Stream", + "datasource": { + "type": "Microsoft.EventGrid/EventSubscriptions", + "properties": { + "subscriber": { + "type": "Microsoft.EventHub/EventHub", + "properties": { + "serviceBusNamespace": "sdktest", + "sharedAccessPolicyName": "RootManageSharedAccessKey", + "consumerGroupName": "sdkconsumergroup", + "eventHubName": "sdkeventhub", + "partitionCount": 16, + "authenticationMode": "Msi" + } + }, + "schema": "CloudEventSchema", + "storageAccounts": [ + { + "authenticationMode": "Msi", + "accountName": "myaccount" + } + ], + "eventTypes": [ + "Microsoft.Storage.BlobCreated" + ] + } + } + } + } + } + } +} diff --git a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Input_Create_GatewayMessageBus.json b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Input_Create_GatewayMessageBus.json new file mode 100644 index 000000000000..60ee2b82c07e --- /dev/null +++ b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Input_Create_GatewayMessageBus.json @@ -0,0 +1,60 @@ +{ + "parameters": { + "api-version": "2021-10-01-preview", + "subscriptionId": "56b5e0a9-b645-407d-99b0-c64f86013e3d", + "resourceGroupName": "sjrg3467", + "jobName": "sj9742", + "inputName": "input7970", + "input": { + "properties": { + "type": "Stream", + "datasource": { + "type": "GatewayMessageBus", + "properties": { + "topic": "EdgeTopic1" + } + } + } + } + }, + "responses": { + "201": { + "headers": { + "ETag": "bd8009b3-8165-4bd3-aad2-29a2b378dd14" + }, + "body": { + "id": "/subscriptions/56b5e0a9-b645-407d-99b0-c64f86013e3d/resourceGroups/sjrg3467/providers/Microsoft.StreamAnalytics/streamingjobs/sj9742/inputs/input7970", + "name": "input7970", + "type": "Microsoft.StreamAnalytics/streamingjobs/inputs", + "properties": { + "type": "Stream", + "datasource": { + "type": "GatewayMessageBus", + "properties": { + "topic": "EdgeTopic1" + } + } + } + } + }, + "200": { + "headers": { + "ETag": "bd8009b3-8165-4bd3-aad2-29a2b378dd14" + }, + "body": { + "id": "/subscriptions/56b5e0a9-b645-407d-99b0-c64f86013e3d/resourceGroups/sjrg3467/providers/Microsoft.StreamAnalytics/streamingjobs/sj9742/inputs/input7970", + "name": "input7970", + "type": "Microsoft.StreamAnalytics/streamingjobs/inputs", + "properties": { + "type": "Stream", + "datasource": { + "type": "GatewayMessageBus", + "properties": { + "topic": "EdgeTopic1" + } + } + } + } + } + } +} diff --git a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Input_Create_Reference_Blob_CSV.json b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Input_Create_Reference_Blob_CSV.json index 98a9e9c53869..e512edbe685f 100644 --- a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Input_Create_Reference_Blob_CSV.json +++ b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Input_Create_Reference_Blob_CSV.json @@ -20,7 +20,12 @@ "container": "state", "pathPattern": "{date}/{time}", "dateFormat": "yyyy/MM/dd", - "timeFormat": "HH" + "timeFormat": "HH", + "blobName": "myblobinput", + "deltaPathPattern": "/testBlob/{date}/delta/{time}/", + "sourcePartitionCount": 16, + "fullSnapshotRefreshRate": "16:14:30", + "deltaSnapshotRefreshRate": "16:14:30" } }, "serialization": { diff --git a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Input_Create_Stream_EventHub_JSON.json b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Input_Create_Stream_EventHub_JSON.json index fa18e2822ee2..9a8f0522f6e5 100644 --- a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Input_Create_Stream_EventHub_JSON.json +++ b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Input_Create_Stream_EventHub_JSON.json @@ -23,6 +23,9 @@ "properties": { "encoding": "UTF8" } + }, + "watermarkSettings": { + "watermarkMode": "ReadWatermark" } } } diff --git a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_Blob_CSV.json b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_Blob_CSV.json index 1ee343cc3561..091cac1eaad6 100644 --- a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_Blob_CSV.json +++ b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_Blob_CSV.json @@ -19,7 +19,9 @@ "container": "state", "pathPattern": "{date}/{time}", "dateFormat": "yyyy/MM/dd", - "timeFormat": "HH" + "timeFormat": "HH", + "blobPathPrefix": "my/path", + "blobWriteMode": "Once" } }, "serialization": { diff --git a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_DataWarehouse.json b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_DataWarehouse.json index 198e9c118eae..f2f1c8ca0726 100644 --- a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_DataWarehouse.json +++ b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_DataWarehouse.json @@ -14,7 +14,8 @@ "server": "asatestserver", "database": "zhayaSQLpool", "user": "tolladmin", - "password": "password123" + "password": "password123", + "authenticationMode": "Msi" } } } diff --git a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_DocumentDB.json b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_DocumentDB.json index e3c602f3adc5..2e69dac0593e 100644 --- a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_DocumentDB.json +++ b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_DocumentDB.json @@ -15,7 +15,8 @@ "database": "db01", "collectionNamePattern": "collection", "partitionKey": "key", - "documentId": "documentId" + "documentId": "documentId", + "authenticationMode": "Msi" } } } diff --git a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_EventHub_JSON.json b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_EventHub_JSON.json index b3329a3fe5b9..67e34434fc88 100644 --- a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_EventHub_JSON.json +++ b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_EventHub_JSON.json @@ -23,6 +23,10 @@ "encoding": "UTF8", "format": "Array" } + }, + "watermarkSettings": { + "watermarkMode": "SendCurrentPartitionWatermark", + "maxWatermarkDifferenceAcrossPartitions": "16:14:30" } } } diff --git a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_GatewayMessageBus.json b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_GatewayMessageBus.json new file mode 100644 index 000000000000..7fc9e0200cb6 --- /dev/null +++ b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_GatewayMessageBus.json @@ -0,0 +1,57 @@ +{ + "parameters": { + "api-version": "2021-10-01-preview", + "subscriptionId": "56b5e0a9-b645-407d-99b0-c64f86013e3d", + "resourceGroupName": "sjrg7983", + "jobName": "sj2331", + "outputName": "output3022", + "output": { + "properties": { + "datasource": { + "type": "GatewayMessageBus", + "properties": { + "topic": "EdgeTopic1" + } + } + } + } + }, + "responses": { + "201": { + "headers": { + "ETag": "ccf8b864-259e-43c5-a628-ba10858b2c07" + }, + "body": { + "id": "/subscriptions/56b5e0a9-b645-407d-99b0-c64f86013e3d/resourceGroups/sjrg7983/providers/Microsoft.StreamAnalytics/streamingjobs/sj2331/outputs/output3022", + "name": "output3022", + "type": "Microsoft.StreamAnalytics/streamingjobs/outputs", + "properties": { + "datasource": { + "type": "GatewayMessageBus", + "properties": { + "topic": "EdgeTopic1" + } + } + } + } + }, + "200": { + "headers": { + "ETag": "ccf8b864-259e-43c5-a628-ba10858b2c07" + }, + "body": { + "id": "/subscriptions/56b5e0a9-b645-407d-99b0-c64f86013e3d/resourceGroups/sjrg7983/providers/Microsoft.StreamAnalytics/streamingjobs/sj2331/outputs/output3022", + "name": "output3022", + "type": "Microsoft.StreamAnalytics/streamingjobs/outputs", + "properties": { + "datasource": { + "type": "GatewayMessageBus", + "properties": { + "topic": "EdgeTopic1" + } + } + } + } + } + } +} diff --git a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_PostgreSQL.json b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_PostgreSQL.json new file mode 100644 index 000000000000..6535e7416948 --- /dev/null +++ b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Create_PostgreSQL.json @@ -0,0 +1,73 @@ +{ + "parameters": { + "api-version": "2021-10-01-preview", + "subscriptionId": "56b5e0a9-b645-407d-99b0-c64f86013e3d", + "resourceGroupName": "sjrg7983", + "jobName": "sj2331", + "outputName": "output3022", + "output": { + "properties": { + "datasource": { + "type": "Microsoft.DBForPostgreSQL/servers/databases", + "properties": { + "server": "someServer", + "database": "someDatabase", + "user": "user", + "password": "somePassword", + "table": "someTable", + "maxWriterCount": 1, + "authenticationMode": "Msi" + } + } + } + } + }, + "responses": { + "201": { + "headers": { + "ETag": "ccf8b864-259e-43c5-a628-ba10858b2c07" + }, + "body": { + "id": "/subscriptions/56b5e0a9-b645-407d-99b0-c64f86013e3d/resourceGroups/sjrg7983/providers/Microsoft.StreamAnalytics/streamingjobs/sj2331/outputs/output3022", + "name": "output3022", + "type": "Microsoft.StreamAnalytics/streamingjobs/outputs", + "properties": { + "datasource": { + "type": "Microsoft.DBForPostgreSQL/servers/databases", + "properties": { + "server": "someServer", + "database": "someDatabase", + "user": "user", + "table": "someTable", + "maxWriterCount": 1, + "authenticationMode": "Msi" + } + } + } + } + }, + "200": { + "headers": { + "ETag": "ccf8b864-259e-43c5-a628-ba10858b2c07" + }, + "body": { + "id": "/subscriptions/56b5e0a9-b645-407d-99b0-c64f86013e3d/resourceGroups/sjrg7983/providers/Microsoft.StreamAnalytics/streamingjobs/sj2331/outputs/output3022", + "name": "output3022", + "type": "Microsoft.StreamAnalytics/streamingjobs/outputs", + "properties": { + "datasource": { + "type": "Microsoft.DBForPostgreSQL/servers/databases", + "properties": { + "server": "someServer", + "database": "someDatabase", + "user": "user", + "table": "someTable", + "maxWriterCount": 1, + "authenticationMode": "Msi" + } + } + } + } + } + } +} diff --git a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Get_EventHub_JSON.json b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Get_EventHub_JSON.json index 83e3467e978f..d383d27d9fc7 100644 --- a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Get_EventHub_JSON.json +++ b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/Output_Get_EventHub_JSON.json @@ -31,7 +31,17 @@ "encoding": "UTF8", "format": "Array" } - } + }, + "lastOutputEventTimestamps": [ + { + "lastOutputEventTime": "2009-06-15T13:45:30", + "lastUpdateTime": "2009-06-15T13:45:30" + }, + { + "lastOutputEventTime": "2009-07-15T13:45:30", + "lastUpdateTime": "2009-06-15T13:45:30" + } + ] } } } diff --git a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/StreamingJob_Create_CompleteJob.json b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/StreamingJob_Create_CompleteJob.json index c9e6f9f57fed..5c3c3e7b8844 100644 --- a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/StreamingJob_Create_CompleteJob.json +++ b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/examples/StreamingJob_Create_CompleteJob.json @@ -66,7 +66,22 @@ "name": "outputtest" } ], - "functions": [] + "functions": [], + "externals": { + "storageAccount": { + "accountName": "mystorageaccount", + "accountKey": "mykey" + }, + "container": "mycontainer", + "path": "UserCustomCode.zip", + "refreshConfiguration": { + "pathPattern": "{date}\\\\{time}", + "dateFormat": "yyyy-dd-MM", + "timeFormat": "HH", + "refreshInterval": "00:01:00", + "refreshType": "Nonblocking" + } + } }, "location": "West US", "tags": { diff --git a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/functions.json b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/functions.json index 7dd88e823c7c..f47ab4da7c98 100644 --- a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/functions.json +++ b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/functions.json @@ -711,10 +711,6 @@ "description": "The binding properties associated with a CSharp function.", "type": "object", "properties": { - "script": { - "type": "string", - "description": "The Csharp code containing a single function definition." - }, "dllPath": { "type": "string", "description": "The Csharp code containing a single function definition." @@ -726,9 +722,24 @@ "method": { "type": "string", "description": "The Csharp code containing a single function definition." + }, + "updateMode": { + "$ref": "#/definitions/UpdateMode" } } }, + "UpdateMode": { + "type": "string", + "description": "Refresh modes for Stream Analytics functions.", + "enum": [ + "Static", + "Refreshable" + ], + "x-ms-enum": { + "name": "UpdateMode", + "modelAsString": true + } + }, "FunctionRetrieveDefaultDefinitionParameters": { "description": "Parameters used to specify the type of function to retrieve the default definition for.", "type": "object", @@ -919,6 +930,14 @@ "type": "integer", "format": "int32", "description": "The number of parallel requests that will be sent per partition of your job to the machine learning service. Default is 1." + }, + "inputRequestName": { + "type": "string", + "description": "Label for the input request object." + }, + "outputResponseName": { + "type": "string", + "description": "Label for the output request object." } } }, diff --git a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/inputs.json b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/inputs.json index 4f6cf61c66b4..4a246448c8da 100644 --- a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/inputs.json +++ b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/inputs.json @@ -52,6 +52,12 @@ }, "Create a stream IoT Hub input with Avro serialization": { "$ref": "./examples/Input_Create_Stream_IoTHub_Avro.json" + }, + "Create a Gateway Message Bus input": { + "$ref": "./examples/Input_Create_GatewayMessageBus.json" + }, + "Create an Event Grid input": { + "$ref": "./examples/Input_Create_EventGrid.json" } }, "parameters": [ @@ -466,6 +472,10 @@ "partitionKey": { "type": "string", "description": "partitionKey Describes a key in the input data which is used for partitioning the input data" + }, + "watermarkSettings": { + "$ref": "#/definitions/InputWatermarkProperties", + "description": "Settings which determine whether to read watermark events." } }, "required": [ @@ -597,6 +607,11 @@ "consumerGroupName": { "type": "string", "description": "The name of an Event Hub Consumer Group that should be used to read events from the Event Hub. Specifying distinct consumer group names for multiple inputs allows each of those inputs to receive the same events from the Event Hub. If not specified, the input uses the Event Hub’s default consumer group." + }, + "prefetchCount": { + "type": "integer", + "format": "int32", + "description": "The number of messages that the message receiver can simultaneously request." } } }, @@ -700,7 +715,29 @@ "$ref": "#/definitions/BlobDataSourceProperties" } ], - "properties": {} + "properties": { + "blobName": { + "type": "string", + "description": "The name of the blob input." + }, + "deltaPathPattern": { + "type": "string", + "description": "The path pattern of the delta snapshot." + }, + "sourcePartitionCount": { + "type": "integer", + "format": "int32", + "description": "The partition count of the blob input data source. Range 1 - 256." + }, + "fullSnapshotRefreshRate": { + "type": "string", + "description": "The refresh interval of the blob input data source." + }, + "deltaSnapshotRefreshRate": { + "type": "string", + "description": "The interval that the user generates a delta snapshot of this reference blob input data source." + } + } }, "RawReferenceInputDataSource": { "description": "Describes a raw input data source that contains reference data. This data source type is only applicable/usable when using the query testing API. You cannot create a job with this data source type or add an input of this data source type to an existing job.", @@ -778,6 +815,10 @@ "type": "string", "description": "The account key for the Azure Storage account. Required on PUT (CreateOrReplace) requests.", "x-ms-secret": true + }, + "authenticationMode": { + "$ref": "../../common/v1/definitions.json#/definitions/AuthenticationMode", + "description": "Authentication Mode." } } }, @@ -816,6 +857,11 @@ "eventHubName": { "type": "string", "description": "The name of the Event Hub. Required on PUT (CreateOrReplace) requests." + }, + "partitionCount": { + "type": "integer", + "format": "int32", + "description": "The partition count of the event hub data source. Range 1 - 256." } } }, @@ -1179,10 +1225,6 @@ "type": "string", "x-ms-secret": true }, - "table": { - "description": "This element is associated with the datasource element. The name of the table in the Azure SQL database..", - "type": "string" - }, "refreshType": { "type": "object", "description": "Indicates the type of data refresh option.", @@ -1199,9 +1241,110 @@ "deltaSnapshotQuery": { "description": "This element is associated with the datasource element. This query is used to fetch incremental changes from the SQL database. To use this option, we recommend using temporal tables in Azure SQL Database.", "type": "string" + }, + "authenticationMode": { + "$ref": "../../common/v1/definitions.json#/definitions/AuthenticationMode", + "description": "Authentication Mode." + } + } + }, + "GatewayMessageBusStreamInputDataSource": { + "description": "Describes a blob input data source that contains stream data.", + "type": "object", + "x-ms-discriminator-value": "GatewayMessageBus", + "allOf": [ + { + "$ref": "#/definitions/StreamInputDataSource" + } + ], + "properties": { + "properties": { + "$ref": "#/definitions/GatewayMessageBusStreamInputDataSourceProperties", + "description": "The properties that are associated with a gateway message bus input containing stream data.", + "x-ms-client-flatten": true + } + } + }, + "GatewayMessageBusStreamInputDataSourceProperties": { + "description": "The properties that are associated with a gateway message bus input containing stream data.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/GatewayMessageBusSourceProperties" + } + ] + }, + "GatewayMessageBusSourceProperties": { + "description": "The properties that are associated with a gateway message bus datasource.", + "type": "object", + "properties": { + "topic": { + "type": "string", + "description": "The name of the Service Bus topic." + } + } + }, + "EventGridStreamInputDataSource": { + "description": "Describes an event grid input data source that contains stream data.", + "type": "object", + "x-ms-discriminator-value": "Microsoft.EventGrid/EventSubscriptions", + "allOf": [ + { + "$ref": "#/definitions/StreamInputDataSource" + } + ], + "properties": { + "properties": { + "$ref": "#/definitions/EventGridStreamInputDataSourceProperties", + "description": "The properties that are associated with an event grid input containing stream data.", + "x-ms-client-flatten": true } } }, + "EventGridStreamInputDataSourceProperties": { + "description": "The properties that are associated with an event grid input containing stream data.", + "type": "object", + "properties": { + "subscriber": { + "type": "object", + "description": "Subscribers for the Event Grid. Currently only EventHub Subscriber is supported.", + "$ref": "#/definitions/EventHubV2StreamInputDataSource" + }, + "schema": { + "type": "object", + "description": "Indicates the Event Grid schema type.", + "$ref": "#/definitions/EventGridEventSchemaType" + }, + "storageAccounts": { + "type": "array", + "items": { + "$ref": "#/definitions/StorageAccount" + }, + "description": "A list of one or more Azure Storage accounts. Required on PUT (CreateOrReplace) requests.", + "x-ms-identifiers": [] + }, + "eventTypes": { + "type": "array", + "items": { + "type": "string", + "description": "Event Type." + }, + "description": "List of Event Types that are supported by the Event Grid adapter." + } + } + }, + "EventGridEventSchemaType": { + "type": "string", + "description": "Supported Event Grid schema types.", + "enum": [ + "EventGridEventSchema", + "CloudEventSchema" + ], + "x-ms-enum": { + "name": "EventGridEventSchemaType", + "modelAsString": true + } + }, "RefreshType": { "type": "string", "description": "Indicates the type of data refresh option.", @@ -1214,6 +1357,28 @@ "name": "RefreshType", "modelAsString": true } + }, + "InputWatermarkProperties": { + "type": "object", + "description": "Settings which determine whether to read watermark events.", + "properties": { + "watermarkMode": { + "$ref": "#/definitions/InputWatermarkMode", + "description": "The input watermark mode." + } + } + }, + "InputWatermarkMode": { + "type": "string", + "description": "The input watermark mode.", + "enum": [ + "None", + "ReadWatermark" + ], + "x-ms-enum": { + "name": "InputWatermarkMode", + "modelAsString": true + } } }, "parameters": { diff --git a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/outputs.json b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/outputs.json index 6b9066de20d2..0918fd20dad3 100644 --- a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/outputs.json +++ b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/outputs.json @@ -73,6 +73,12 @@ }, "Create an Azure Data Lake Store output with JSON serialization": { "$ref": "./examples/Output_Create_AzureDataLakeStore_JSON.json" + }, + "Create a Gateway Message Bus output": { + "$ref": "./examples/Output_Create_GatewayMessageBus.json" + }, + "Create a PostgreSQL output": { + "$ref": "./examples/Output_Create_PostgreSQL.json" } }, "parameters": [ @@ -526,6 +532,19 @@ "readOnly": true, "type": "string", "description": "The current entity tag for the output. This is an opaque string. You can use it to detect whether the resource has changed between requests. You can also use it in the If-Match or If-None-Match headers for write operations for optimistic concurrency." + }, + "lastOutputEventTimestamps": { + "type": "array", + "items": { + "$ref": "#/definitions/LastOutputEventTimestamp" + }, + "readOnly": true, + "description": "A list of the last output event times for each output partition. The index of the array corresponds to the partition number.", + "x-ms-identifiers": [] + }, + "watermarkSettings": { + "$ref": "#/definitions/OutputWatermarkProperties", + "description": "Settings which determine whether to send watermarks to downstream." } } }, @@ -594,7 +613,29 @@ { "$ref": "./inputs.json#/definitions/BlobDataSourceProperties" } - ] + ], + "properties": { + "blobPathPrefix": { + "type": "string", + "description": "Blob path prefix." + }, + "blobWriteMode": { + "$ref": "#/definitions/BlobWriteMode", + "description": "Blob write mode." + } + } + }, + "BlobWriteMode": { + "type": "string", + "description": "Determines whether blob blocks are either committed automatically or appended.", + "enum": [ + "Append", + "Once" + ], + "x-ms-enum": { + "name": "BlobWriteMode", + "modelAsString": true + } }, "AzureTableOutputDataSource": { "type": "object", @@ -786,6 +827,72 @@ "type": "string", "description": "The password that will be used to connect to the Azure SQL database. Required on PUT (CreateOrReplace) requests.", "x-ms-secret": true + }, + "authenticationMode": { + "$ref": "../../common/v1/definitions.json#/definitions/AuthenticationMode", + "description": "Authentication Mode." + } + } + }, + "PostgreSQLOutputDataSource": { + "type": "object", + "description": "Describes a PostgreSQL output data source.", + "x-ms-discriminator-value": "Microsoft.DBForPostgreSQL/servers/databases", + "allOf": [ + { + "$ref": "#/definitions/OutputDataSource" + } + ], + "properties": { + "properties": { + "$ref": "#/definitions/PostgreSQLOutputDataSourceProperties", + "description": "The properties that are associated with a PostgreSQL output data source. Required on PUT (CreateOrReplace) requests.", + "x-ms-client-flatten": true + } + } + }, + "PostgreSQLOutputDataSourceProperties": { + "type": "object", + "description": "The properties that are associated with a PostgreSQL output.", + "allOf": [ + { + "$ref": "#/definitions/PostgreSQLDataSourceProperties" + } + ], + "properties": {} + }, + "PostgreSQLDataSourceProperties": { + "type": "object", + "description": "The properties that are associated with an Azure SQL database data source.", + "properties": { + "server": { + "type": "string", + "description": "The name of the SQL server containing the Azure SQL database. Required on PUT (CreateOrReplace) requests." + }, + "database": { + "type": "string", + "description": "The name of the Azure SQL database. Required on PUT (CreateOrReplace) requests." + }, + "table": { + "type": "string", + "description": "The name of the table in the Azure SQL database. Required on PUT (CreateOrReplace) requests." + }, + "user": { + "type": "string", + "description": "The user name that will be used to connect to the Azure SQL database. Required on PUT (CreateOrReplace) requests." + }, + "password": { + "type": "string", + "description": "The password that will be used to connect to the Azure SQL database. Required on PUT (CreateOrReplace) requests.", + "x-ms-secret": true + }, + "maxWriterCount": { + "type": "number", + "description": "Max Writer count, currently only 1(single writer) and 0(based on query partition) are available. Optional on PUT requests." + }, + "authenticationMode": { + "$ref": "../../common/v1/definitions.json#/definitions/AuthenticationMode", + "description": "Authentication Mode." } } }, @@ -834,6 +941,10 @@ "documentId": { "type": "string", "description": "The name of the field in output events used to specify the primary key which insert or update operations are based on." + }, + "authenticationMode": { + "$ref": "../../common/v1/definitions.json#/definitions/AuthenticationMode", + "description": "Authentication Mode." } } }, @@ -1069,6 +1180,32 @@ } } }, + "GatewayMessageBusOutputDataSource": { + "type": "object", + "description": "Describes a Gateway Message Bus output data source.", + "x-ms-discriminator-value": "GatewayMessageBus", + "allOf": [ + { + "$ref": "#/definitions/OutputDataSource" + } + ], + "properties": { + "properties": { + "$ref": "#/definitions/GatewayMessageBusOutputDataSourceProperties", + "description": "The properties that are associated with a Gateway Message Bus output. Required on PUT (CreateOrReplace) requests.", + "x-ms-client-flatten": true + } + } + }, + "GatewayMessageBusOutputDataSourceProperties": { + "type": "object", + "description": "The properties that are associated with a Gateway Message Bus.", + "allOf": [ + { + "$ref": "./inputs.json#/definitions/GatewayMessageBusSourceProperties" + } + ] + }, "OutputListResult": { "type": "object", "properties": { @@ -1144,6 +1281,47 @@ } } }, + "LastOutputEventTimestamp": { + "type": "object", + "description": "An output event timestamp.", + "properties": { + "lastOutputEventTime": { + "type": "string", + "description": "The last output event time." + }, + "lastUpdateTime": { + "type": "string", + "description": "The time that the last update happened." + } + } + }, + "OutputWatermarkProperties": { + "type": "object", + "description": "Settings which determine whether to send watermarks to downstream.", + "properties": { + "watermarkMode": { + "$ref": "#/definitions/OutputWatermarkMode", + "description": "The output watermark mode." + }, + "maxWatermarkDifferenceAcrossPartitions": { + "type": "string", + "description": "Describes the maximal delta between the fastest and slowest partitions, so the out of order window that catches all necessary events in downstream jobs is well defined." + } + } + }, + "OutputWatermarkMode": { + "type": "string", + "description": "The output watermark mode.", + "enum": [ + "None", + "SendCurrentPartitionWatermark", + "SendLowestWatermarkAcrossPartitions" + ], + "x-ms-enum": { + "name": "OutputWatermarkMode", + "modelAsString": true + } + }, "SubResource": { "type": "object", "description": "The base sub-resource model definition.", diff --git a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/streamingjobs.json b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/streamingjobs.json index 1a221e44deea..bf56d8b827a6 100644 --- a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/streamingjobs.json +++ b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/preview/2021-10-01-preview/streamingjobs.json @@ -960,12 +960,56 @@ "$ref": "./inputs.json#/definitions/StorageAccount" }, "container": { - "type": "string" + "type": "string", + "description": "The UserCustomCode container." }, "path": { - "type": "string" + "type": "string", + "description": "The UserCustomCode path." + }, + "refreshConfiguration": { + "$ref": "#/definitions/RefreshConfiguration", + "description": "The refresh parameters for any/all updatable user defined functions present in the job config." } } + }, + "RefreshConfiguration": { + "description": "The refresh parameters for any/all updatable user defined functions present in the job config.", + "type": "object", + "properties": { + "pathPattern": { + "type": "string", + "description": "The blob path pattern. Not a regular expression. It represents a pattern against which blob names will be matched to determine whether or not they should be included as input or output to the job. See https://docs.microsoft.com/en-us/rest/api/streamanalytics/stream-analytics-input or https://docs.microsoft.com/en-us/rest/api/streamanalytics/stream-analytics-output for a more detailed explanation and example." + }, + "dateFormat": { + "type": "string", + "description": "The date format. Wherever {date} appears in pathPattern, the value of this property is used as the date format instead." + }, + "timeFormat": { + "type": "string", + "description": "The time format. Wherever {time} appears in pathPattern, the value of this property is used as the time format instead." + }, + "refreshInterval": { + "type": "string", + "description": "The refresh interval." + }, + "refreshType": { + "$ref": "#/definitions/UpdatableUdfRefreshType", + "description": "This property indicates which data refresh option to use, Blocking or Nonblocking." + } + } + }, + "UpdatableUdfRefreshType": { + "type": "string", + "description": "This property indicates which data refresh option to use, Blocking or Nonblocking.", + "enum": [ + "Blocking", + "Nonblocking" + ], + "x-ms-enum": { + "name": "RefreshType", + "modelAsString": true + } } } } diff --git a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/stable/2020-03-01/functions.json b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/stable/2020-03-01/functions.json index 5a166669be61..b32f7f839bde 100644 --- a/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/stable/2020-03-01/functions.json +++ b/specification/streamanalytics/resource-manager/Microsoft.StreamAnalytics/stable/2020-03-01/functions.json @@ -485,53 +485,43 @@ "readOnly": true, "type": "string", "description": "The current entity tag for the function. This is an opaque string. You can use it to detect whether the resource has changed between requests. You can also use it in the If-Match or If-None-Match headers for write operations for optimistic concurrency." + }, + "properties": { + "$ref": "#/definitions/FunctionConfiguration", + "x-ms-client-flatten": true } }, "required": [ "type" ] }, - "ScalarFunctionProperties": { - "type": "object", - "description": "The properties that are associated with a scalar function.", - "x-ms-discriminator-value": "Scalar", - "allOf": [ - { - "$ref": "#/definitions/FunctionProperties" - } - ], - "properties": { - "properties": { - "$ref": "#/definitions/ScalarFunctionConfiguration", - "description": "Describes the configuration of the scalar function.", - "x-ms-client-flatten": true - } - } - }, - "ScalarFunctionConfiguration": { + "FunctionConfiguration": { "type": "object", - "description": "Describes the configuration of the scalar function.", "properties": { "inputs": { "type": "array", "items": { "$ref": "#/definitions/FunctionInput" - }, - "x-ms-identifiers": [ - "dataType" - ], - "description": "A list of inputs describing the parameters of the function." + } }, "output": { - "$ref": "#/definitions/FunctionOutput", - "description": "The output of the function." + "$ref": "#/definitions/FunctionOutput" }, "binding": { - "$ref": "#/definitions/FunctionBinding", - "description": "The physical binding of the function. For example, in the Azure Machine Learning web service’s case, this describes the endpoint." + "$ref": "#/definitions/FunctionBinding" } } }, + "ScalarFunctionProperties": { + "description": "The properties that are associated with a scalar function.", + "type": "object", + "x-ms-discriminator-value": "Scalar", + "allOf": [ + { + "$ref": "#/definitions/FunctionProperties" + } + ] + }, "FunctionInput": { "type": "object", "description": "Describes one input parameter of a function.", @@ -826,6 +816,16 @@ } }, "x-ms-azure-resource": true + }, + "AggregateFunctionProperties": { + "description": "The properties that are associated with an aggregate function.", + "type": "object", + "x-ms-discriminator-value": "Aggregate", + "allOf": [ + { + "$ref": "#/definitions/FunctionProperties" + } + ] } }, "parameters": {