Merge pull request #2742 from Azure/autogenerate

Merge Autogenerate
Azure · Jan 13, 2023 · 5c1df54 · 5c1df54
2 parents 44d7f44 + 8bd08c4
commit 5c1df54
Show file tree

Hide file tree

Showing 13 changed files with 18,852 additions and 156 deletions.
diff --git a/generator/resources.json b/generator/resources.json
diff --git a/schemas/2018-06-01/Microsoft.DataFactory.json b/schemas/2018-06-01/Microsoft.DataFactory.json
@@ -27270,6 +27270,35 @@
       ],
       "description": "A copy activity snowflake source."
     },
+    "SparkConfigurationParametrizationReference": {
+      "type": "object",
+      "properties": {
+        "referenceName": {
+          "type": "object",
+          "properties": {},
+          "description": "Reference spark configuration name. Type: string (or Expression with resultType string)."
+        },
+        "type": {
+          "oneOf": [
+            {
+              "type": "string",
+              "enum": [
+                "SparkConfigurationReference"
+              ]
+            },
+            {
+              "$ref": "https://schema.management.azure.com/schemas/common/definitions.json#/definitions/expression"
+            }
+          ],
+          "description": "Spark configuration reference type."
+        }
+      },
+      "required": [
+        "referenceName",
+        "type"
+      ],
+      "description": "Spark configuration reference."
+    },
     "SparkDatasetTypeProperties": {
       "type": "object",
       "properties": {
@@ -29246,6 +29275,22 @@
           "properties": {},
           "description": "Spark configuration properties, which will override the 'conf' of the spark job definition you provide."
         },
+        "configurationType": {
+          "oneOf": [
+            {
+              "type": "string",
+              "enum": [
+                "Default",
+                "Customized",
+                "Artifact"
+              ]
+            },
+            {
+              "$ref": "https://schema.management.azure.com/schemas/common/definitions.json#/definitions/expression"
+            }
+          ],
+          "description": "The type of the spark config."
+        },
         "driverSize": {
           "type": "object",
           "properties": {},
@@ -29292,30 +29337,45 @@
           "description": "Additional files used for reference in the main definition file, which will override the 'jars' and 'files' of the spark job definition you provide."
         },
         "numExecutors": {
+          "type": "object",
+          "properties": {},
+          "description": "Number of executors to launch for this job, which will override the 'numExecutors' of the spark job definition you provide. Type: integer (or Expression with resultType integer)."
+        },
+        "pythonCodeReference": {
           "oneOf": [
             {
-              "type": "integer"
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {}
+              }
             },
             {
               "$ref": "https://schema.management.azure.com/schemas/common/definitions.json#/definitions/expression"
             }
           ],
-          "description": "Number of executors to launch for this job, which will override the 'numExecutors' of the spark job definition you provide."
+          "description": "Additional python code files used for reference in the main definition file, which will override the 'pyFiles' of the spark job definition you provide."
         },
-        "pythonCodeReference": {
+        "scanFolder": {
+          "type": "object",
+          "properties": {},
+          "description": "Scanning subfolders from the root folder of the main definition file, these files will be added as reference files. The folders named 'jars', 'pyFiles', 'files' or 'archives' will be scanned, and the folders name are case sensitive. Type: boolean (or Expression with resultType boolean)."
+        },
+        "sparkConfig": {
           "oneOf": [
             {
-              "type": "array",
-              "items": {
+              "type": "object",
+              "additionalProperties": {
                 "type": "object",
                 "properties": {}
-              }
+              },
+              "properties": {}
             },
             {
               "$ref": "https://schema.management.azure.com/schemas/common/definitions.json#/definitions/expression"
             }
           ],
-          "description": "Additional python code files used for reference in the main definition file, which will override the 'pyFiles' of the spark job definition you provide."
+          "description": "Spark configuration property."
         },
         "sparkJob": {
           "oneOf": [
@@ -29338,6 +29398,17 @@
             }
           ],
           "description": "Big data pool reference type."
+        },
+        "targetSparkConfiguration": {
+          "oneOf": [
+            {
+              "$ref": "#/definitions/SparkConfigurationParametrizationReference"
+            },
+            {
+              "$ref": "https://schema.management.azure.com/schemas/common/definitions.json#/definitions/expression"
+            }
+          ],
+          "description": "Spark configuration reference."
         }
       },
       "required": [