From 990ae3f3b4d4c388dd6d0736cd16a9d0bfe1e5cb Mon Sep 17 00:00:00 2001 From: Zhiyong Yang Date: Wed, 25 Mar 2020 13:58:43 +0800 Subject: [PATCH 1/2] add sharepointonlinelist connector --- .../2018-06-01/entityTypes/Dataset.json | 26 ++++++++++ .../2018-06-01/entityTypes/LinkedService.json | 51 +++++++++++++++++++ .../2018-06-01/entityTypes/Pipeline.json | 39 ++++++++++++++ 3 files changed, 116 insertions(+) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json index 8a4bb5225c35..8e1e0fcb1665 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Dataset.json @@ -3103,6 +3103,32 @@ "description": "The table name of the Snowflake database. Type: string (or Expression with resultType string)." } } + }, + "SharePointOnlineListResourceDataset": { + "x-ms-discriminator-value": "SharePointOnlineListResource", + "description": "The sharepoint online list resource dataset.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Dataset" + } + ], + "properties": { + "typeProperties": { + "description": "Sharepoint online list dataset properties.", + "x-ms-client-flatten": true, + "$ref": "#/definitions/SharePointOnlineListDatasetTypeProperties" + } + } + }, + "SharePointOnlineListDatasetTypeProperties": { + "description": "Sharepoint online list dataset properties.", + "properties": { + "listName": { + "type": "object", + "description": "The name of the SharePoint Online list. Type: string (or Expression with resultType string)." + } + } } } } diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json index 993816e34b28..fb66ce6759a2 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/LinkedService.json @@ -5729,6 +5729,57 @@ "required": [ "connectionString" ] + }, + "SharePointOnlineListLinkedService": { + "x-ms-discriminator-value": "SharePointOnlineList", + "description": "SharePoint Online List linked service.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/LinkedService" + } + ], + "properties": { + "typeProperties": { + "description": "SharePoint Online List linked service properties.", + "x-ms-client-flatten": true, + "$ref": "#/definitions/SharePointOnlineListLinkedServiceTypeProperties" + } + }, + "required": [ + "typeProperties" + ] + }, + "SharePointOnlineListLinkedServiceTypeProperties": { + "description": "SharePoint Online List linked service properties.", + "properties": { + "siteUrl": { + "type": "object", + "description": "The URL of the SharePoint Online site. For example, https://contoso.sharepoint.com/sites/siteName. Type: string (or Expression with resultType string)." + }, + "tenantId": { + "type": "object", + "description": "The tenant ID under which your application resides. You can find it from Azure portal Active Directory overview page. Type: string (or Expression with resultType string)." + }, + "servicePrincipalId": { + "type": "object", + "description": "The application (client) ID of your application registered in Azure Active Directory. Make sure to grant SharePoint site permission to this application. Type: string (or Expression with resultType string)." + }, + "servicePrincipalKey": { + "description": "The client secret of your application registered in Azure Active Directory. Type: string (or Expression with resultType string).", + "$ref": "../datafactory.json#/definitions/SecretBase" + }, + "encryptedCredential": { + "type": "object", + "description": "The encrypted credential used for authentication. Credentials are encrypted using the integration runtime credential manager. Type: string (or Expression with resultType string)." + } + }, + "required": [ + "siteUrl", + "tenantId", + "servicePrincipalId", + "servicePrincipalKey" + ] } } } diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json index 28298bcea8a4..c9ce9c294c05 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json @@ -1535,6 +1535,10 @@ "type": "object", "description": "OData query. For example, \"$top=1\". Type: string (or Expression with resultType string)." }, + "httpRequestTimeout": { + "type": "object", + "description": "The timeout (TimeSpan) to get an HTTP response. It is the timeout to get a response, not the timeout to read response data. Default value: 00:05:00. Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))." + }, "additionalColumns": { "type": "array", "description": "Specifies the additional columns to be added to source data. Type: array of objects (or Expression with resultType array of objects).", @@ -1617,6 +1621,10 @@ "query": { "type": "object", "description": "SAP Cloud for Customer OData query. For example, \"$top=1\". Type: string (or Expression with resultType string)." + }, + "httpRequestTimeout": { + "type": "object", + "description": "The timeout (TimeSpan) to get an HTTP response. It is the timeout to get a response, not the timeout to read response data. Default value: 00:05:00. Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))." } } }, @@ -1632,6 +1640,10 @@ "query": { "type": "object", "description": "SAP ECC OData query. For example, \"$top=1\". Type: string (or Expression with resultType string)." + }, + "httpRequestTimeout": { + "type": "object", + "description": "The timeout (TimeSpan) to get an HTTP response. It is the timeout to get a response, not the timeout to read response data. Default value: 00:05:00. Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))." } } }, @@ -2990,6 +3002,10 @@ "query": { "type": "object", "description": "A query to retrieve data from source. Type: string (or Expression with resultType string)." + }, + "httpRequestTimeout": { + "type": "object", + "description": "The timeout (TimeSpan) to get an HTTP response. It is the timeout to get a response, not the timeout to read response data. Default value: 00:05:00. Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))." } } }, @@ -3209,6 +3225,10 @@ "name": "SapCloudForCustomerSinkWriteBehavior", "modelAsString": true } + }, + "httpRequestTimeout": { + "type": "object", + "description": "The timeout (TimeSpan) to get an HTTP response. It is the timeout to get a response, not the timeout to read response data. Default value: 00:05:00. Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))." } } }, @@ -6209,6 +6229,25 @@ "required": [ "dataFlow" ] + }, + "SharePointOnlineListSource": { + "description": "A copy activity source for sharePoint online list source.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/CopySource" + } + ], + "properties": { + "query": { + "type": "object", + "description": "The OData query to filter the data in SharePoint Online list. For example, \"$top=1\". Type: string (or Expression with resultType string)." + }, + "httpRequestTimeout": { + "type": "object", + "description": "The wait time to get a response from SharePoint Online. Default value is 5 minutes (00:05:00). Type: string (or Expression with resultType string), pattern: ((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))." + } + } } } } From d56ceaaea5b673babd7f6ec712598ba874e46219 Mon Sep 17 00:00:00 2001 From: Zhiyong Yang Date: Thu, 16 Apr 2020 11:39:06 +0800 Subject: [PATCH 2/2] support parallel read of sql sources --- .../2018-06-01/entityTypes/Pipeline.json | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json index ec82ec1adffc..c80e37c45c41 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json @@ -1996,6 +1996,14 @@ "isolationLevel": { "description": "Specifies the transaction locking behavior for the SQL source. Allowed values: ReadCommitted/ReadUncommitted/RepeatableRead/Serializable/Snapshot. The default value is ReadCommitted. Type: string (or Expression with resultType string).", "type": "object" + }, + "partitionOption": { + "description": "The partition mechanism that will be used for Sql read in parallel.", + "$ref": "#/definitions/SqlPartitionOption" + }, + "partitionSettings": { + "description": "The settings that will be leveraged for Sql source partitioning.", + "$ref": "#/definitions/SqlPartitionSettings" } } }, @@ -2026,6 +2034,14 @@ "produceAdditionalTypes": { "description": "Which additional types to produce.", "type": "object" + }, + "partitionOption": { + "description": "The partition mechanism that will be used for Sql read in parallel.", + "$ref": "#/definitions/SqlPartitionOption" + }, + "partitionSettings": { + "description": "The settings that will be leveraged for Sql source partitioning.", + "$ref": "#/definitions/SqlPartitionSettings" } } }, @@ -2056,6 +2072,14 @@ "produceAdditionalTypes": { "description": "Which additional types to produce.", "type": "object" + }, + "partitionOption": { + "description": "The partition mechanism that will be used for Sql read in parallel.", + "$ref": "#/definitions/SqlPartitionOption" + }, + "partitionSettings": { + "description": "The settings that will be leveraged for Sql source partitioning.", + "$ref": "#/definitions/SqlPartitionSettings" } } }, @@ -2086,6 +2110,14 @@ "produceAdditionalTypes": { "description": "Which additional types to produce.", "type": "object" + }, + "partitionOption": { + "description": "The partition mechanism that will be used for Sql read in parallel.", + "$ref": "#/definitions/SqlPartitionOption" + }, + "partitionSettings": { + "description": "The settings that will be leveraged for Sql source partitioning.", + "$ref": "#/definitions/SqlPartitionSettings" } } }, @@ -2109,9 +2141,48 @@ "storedProcedureParameters": { "type": "object", "description": "Value and type setting for stored procedure parameters. Example: \"{Parameter1: {value: \"1\", type: \"int\"}}\". Type: object (or Expression with resultType object), itemType: StoredProcedureParameter." + }, + "partitionOption": { + "description": "The partition mechanism that will be used for Sql read in parallel.", + "$ref": "#/definitions/SqlPartitionOption" + }, + "partitionSettings": { + "description": "The settings that will be leveraged for Sql source partitioning.", + "$ref": "#/definitions/SqlPartitionSettings" } } }, + "SqlPartitionSettings": { + "description": "The settings that will be leveraged for Sql source partitioning.", + "type": "object", + "properties": { + "partitionColumnName": { + "type": "object", + "description": "The name of the column in integer or datetime type that will be used for proceeding partitioning. If not specified, the primary key of the table is auto-detected and used as the partition column. Type: string (or Expression with resultType string)." + }, + "partitionUpperBound": { + "type": "object", + "description": "The maximum value of column specified in partitionColumnName that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)." + }, + "partitionLowerBound": { + "type": "object", + "description": "The minimum value of column specified in partitionColumnName that will be used for proceeding range partitioning. Type: string (or Expression with resultType string)." + } + } + }, + "SqlPartitionOption": { + "description": "The partition mechanism that will be used for Sql read in parallel.", + "type": "string", + "enum": [ + "None", + "PhysicalPartitionsOfTable", + "DynamicRange" + ], + "x-ms-enum": { + "name": "SqlPartitionOption", + "modelAsString": true + } + }, "FileSystemSource": { "description": "A copy activity file system source.", "type": "object",