Merge pull request #82 from mrpaulandrew/develop

v1.9.2
mrpaulandrew · Nov 19, 2020 · 77ad1f8 · 77ad1f8
2 parents f42ec8b + 209d381
commit 77ad1f8
Show file tree

Hide file tree

Showing 50 changed files with 5,134 additions and 65 deletions.
diff --git a/ARM Templates/Data Factory/v1.9.2 Export.json b/ARM Templates/Data Factory/v1.9.2 Export.json
diff --git a/DataFactory/pipeline/01-Grandparent.json b/DataFactory/pipeline/01-Grandparent.json
@@ -34,7 +34,7 @@
 			"name": "_ProcFwk"
 		},
 		"annotations": [
-			"ADF.procfwk",
+			"procfwk",
 			"Grandparent"
 		]
 	}

diff --git a/DataFactory/pipeline/02-Parent.json b/DataFactory/pipeline/02-Parent.json
@@ -276,7 +276,7 @@
 			},
 			{
 				"name": "Check Previous Execution",
-				"description": "",
+				"description": "Query the current execution table for worker pipelines that require a clean up from the previous execution run.",
 				"type": "Lookup",
 				"dependsOn": [
 					{
@@ -352,7 +352,7 @@
 								"timeout": "0.00:10:00",
 								"retry": 0,
 								"retryIntervalInSeconds": 30,
-								"secureOutput": false,
+								"secureOutput": true,
 								"secureInput": false
 							},
 							"userProperties": [],
@@ -909,6 +909,7 @@
 			},
 			{
 				"name": "Is Parent Already Running",
+				"description": "Establish before anything else if the parent pipeline is already running. Batch execution aware.",
 				"type": "ExecutePipeline",
 				"dependsOn": [],
 				"userProperties": [],
@@ -947,7 +948,7 @@
 					}
 				],
 				"policy": {
-					"timeout": "7.00:00:00",
+					"timeout": "0.00:10:00",
 					"retry": 0,
 					"retryIntervalInSeconds": 30,
 					"secureOutput": false,
@@ -994,7 +995,7 @@
 			"name": "_ProcFwk"
 		},
 		"annotations": [
-			"ADF.procfwk",
+			"procfwk",
 			"Parent"
 		]
 	}

diff --git a/DataFactory/pipeline/03-Child.json b/DataFactory/pipeline/03-Child.json
@@ -11,7 +11,7 @@
 				"policy": {
 					"timeout": "0.00:10:00",
 					"retry": 0,
-					"retryIntervalInSeconds": 0,
+					"retryIntervalInSeconds": 30,
 					"secureOutput": false,
 					"secureInput": false
 				},
@@ -111,7 +111,7 @@
 			"name": "_ProcFwk"
 		},
 		"annotations": [
-			"ADF.procfwk",
+			"procfwk",
 			"Child"
 		]
 	}

diff --git a/DataFactory/pipeline/04-Infant.json b/DataFactory/pipeline/04-Infant.json
@@ -360,6 +360,7 @@
 					"ifTrueActivities": [
 						{
 							"name": "Get Email Parts",
+							"description": "Return all required content from the metadata database to send an email alerting using the procfwk. The lookup returns the exact content for the function body request.",
 							"type": "Lookup",
 							"dependsOn": [],
 							"policy": {
@@ -395,6 +396,7 @@
 						},
 						{
 							"name": "Send Email",
+							"description": "Use an Azure Function to perform an SMTP client email send operation.",
 							"type": "AzureFunctionActivity",
 							"dependsOn": [
 								{
@@ -1194,6 +1196,7 @@
 			},
 			{
 				"name": "Validate Pipeline",
+				"description": "Query the target data factory and establish if the provided worker pipeline name is valid.",
 				"type": "AzureFunctionActivity",
 				"dependsOn": [
 					{
@@ -1246,11 +1249,11 @@
 					}
 				],
 				"policy": {
-					"timeout": "7.00:00:00",
+					"timeout": "0.00:10:00",
 					"retry": 0,
 					"retryIntervalInSeconds": 30,
 					"secureOutput": false,
-					"secureInput": false
+					"secureInput": true
 				},
 				"userProperties": [],
 				"typeProperties": {
@@ -1268,6 +1271,7 @@
 			},
 			{
 				"name": "Is Target Worker Validate",
+				"description": "True = the worker pipeline name is valid.\nFalse = the worker pipeline name is invalid. Raise an exception.",
 				"type": "IfCondition",
 				"dependsOn": [
 					{
@@ -1286,6 +1290,7 @@
 					"ifFalseActivities": [
 						{
 							"name": "Throw Exception - Invalid Infant",
+							"description": "Throw an exception with details about the invalid worker pipeline name.",
 							"type": "ExecutePipeline",
 							"dependsOn": [],
 							"userProperties": [],
@@ -1305,6 +1310,7 @@
 						},
 						{
 							"name": "Update Execution With Invalid Worker",
+							"description": "Update the current execution table with an informed status for the worker pipeline that couldn't be executed.",
 							"type": "SqlServerStoredProcedure",
 							"dependsOn": [],
 							"policy": {
@@ -1497,7 +1503,7 @@
 			"name": "_ProcFwk"
 		},
 		"annotations": [
-			"ADF.procfwk",
+			"procfwk",
 			"Infant"
 		]
 	}

diff --git a/DataFactory/pipeline/Check For Running Pipeline.json b/DataFactory/pipeline/Check For Running Pipeline.json
@@ -1,13 +1,15 @@
 {
 	"name": "Check For Running Pipeline",
 	"properties": {
+		"description": "For a given pipeline and optional batch name establish if a pipeline run is already in progress. Throw an exception if it it.",
 		"activities": [
 			{
 				"name": "Get Subscription",
+				"description": "Use the Azure Management API to return the current subscription.",
 				"type": "WebActivity",
 				"dependsOn": [],
 				"policy": {
-					"timeout": "7.00:00:00",
+					"timeout": "0.00:10:00",
 					"retry": 0,
 					"retryIntervalInSeconds": 30,
 					"secureOutput": false,
@@ -25,6 +27,7 @@
 			},
 			{
 				"name": "Get Pipeline Runs",
+				"description": "Use the Azure Management API to return a list of pipeline runs within the given time window.",
 				"type": "WebActivity",
 				"dependsOn": [
 					{
@@ -41,7 +44,7 @@
 					}
 				],
 				"policy": {
-					"timeout": "7.00:00:00",
+					"timeout": "0.00:10:00",
 					"retry": 0,
 					"retryIntervalInSeconds": 30,
 					"secureOutput": false,
@@ -66,6 +69,7 @@
 			},
 			{
 				"name": "Set Parsed Subscription",
+				"description": "Parse the subscription string value to return just the subscription ID.",
 				"type": "SetVariable",
 				"dependsOn": [
 					{
@@ -86,6 +90,7 @@
 			},
 			{
 				"name": "Filter Running Pipelines",
+				"description": "Filter the pipeline runs results for pipelines that exclude the current triggered run and that are currently running (in progress or queued).",
 				"type": "Filter",
 				"dependsOn": [
 					{
@@ -109,10 +114,11 @@
 			},
 			{
 				"name": "Get Resource Group",
+				"description": "Using the metadata properties table return the framework factory resource group name.",
 				"type": "Lookup",
 				"dependsOn": [],
 				"policy": {
-					"timeout": "7.00:00:00",
+					"timeout": "0.00:10:00",
 					"retry": 0,
 					"retryIntervalInSeconds": 30,
 					"secureOutput": false,
@@ -140,10 +146,11 @@
 			},
 			{
 				"name": "Get Query Run Days Value",
+				"description": "Using the metadata properties table return the run days value to provide the API request with a date range for pipeline executions.",
 				"type": "Lookup",
 				"dependsOn": [],
 				"policy": {
-					"timeout": "7.00:00:00",
+					"timeout": "0.00:10:00",
 					"retry": 0,
 					"retryIntervalInSeconds": 30,
 					"secureOutput": false,
@@ -171,6 +178,7 @@
 			},
 			{
 				"name": "If Pipeline Is Running",
+				"description": "If the running pipeline count is greater than or equal to one.\nTrue = raise an exception.",
 				"type": "IfCondition",
 				"dependsOn": [
 					{
@@ -188,7 +196,8 @@
 					},
 					"ifTrueActivities": [
 						{
-							"name": "Throw Exceptiion - Pipeline Running",
+							"name": "Throw Exception - Pipeline Running",
+							"description": "Using the utils pipeline raise an exception to stop the new trigger while a run is already in progress.",
 							"type": "ExecutePipeline",
 							"dependsOn": [],
 							"userProperties": [],
@@ -211,10 +220,11 @@
 			},
 			{
 				"name": "Get Execution Batch Status",
+				"description": "Using the metadata properties table return the flag to indicate if batch execution setting are enabled or disabled.",
 				"type": "Lookup",
 				"dependsOn": [],
 				"policy": {
-					"timeout": "7.00:00:00",
+					"timeout": "0.00:10:00",
 					"retry": 0,
 					"retryIntervalInSeconds": 30,
 					"secureOutput": false,
@@ -242,6 +252,7 @@
 			},
 			{
 				"name": "If Using Batch Executions",
+				"description": "True = batch executions are enabled.\nFalse = batch execution are disabled.",
 				"type": "IfCondition",
 				"dependsOn": [
 					{
@@ -266,6 +277,7 @@
 					"ifFalseActivities": [
 						{
 							"name": "Set Run Count Without Batch",
+							"description": "Set the pipelines running count variable to be tested later.",
 							"type": "SetVariable",
 							"dependsOn": [],
 							"userProperties": [],
@@ -281,6 +293,7 @@
 					"ifTrueActivities": [
 						{
 							"name": "Filter for Batch Name",
+							"description": "Further filter the return pipeline runs for any running pipelines with the same batch name value.",
 							"type": "Filter",
 							"dependsOn": [],
 							"userProperties": [],
@@ -297,6 +310,7 @@
 						},
 						{
 							"name": "Set Run Count for Batch",
+							"description": "Set the resulting pipeline running count variable to be tested later.",
 							"type": "SetVariable",
 							"dependsOn": [
 								{
@@ -320,6 +334,7 @@
 			},
 			{
 				"name": "Check for Valid Pipeline Name",
+				"description": "Use the Azure Management API to return and establish if the framework pipeline exists in the target Data Factory, including being deployed.",
 				"type": "WebActivity",
 				"dependsOn": [
 					{
@@ -336,7 +351,7 @@
 					}
 				],
 				"policy": {
-					"timeout": "7.00:00:00",
+					"timeout": "0.00:10:00",
 					"retry": 0,
 					"retryIntervalInSeconds": 30,
 					"secureOutput": false,
@@ -379,6 +394,9 @@
 		"folder": {
 			"name": "_ProcFwk/_ProcFwkUtils"
 		},
-		"annotations": []
+		"annotations": [
+			"procfwk",
+			"Utils"
+		]
 	}
 }
diff --git a/DataFactory/pipeline/Email Sender.json b/DataFactory/pipeline/Email Sender.json
@@ -1,13 +1,15 @@
 {
 	"name": "Email Sender",
 	"properties": {
+		"description": "Provide a simple abstract over the send email function with request body item exposed as pipeline parameters.",
 		"activities": [
 			{
 				"name": "Send Email",
+				"description": "Use an Azure Function to perform an SMTP client email send operation.",
 				"type": "AzureFunctionActivity",
 				"dependsOn": [],
 				"policy": {
-					"timeout": "7.00:00:00",
+					"timeout": "0.00:10:00",
 					"retry": 0,
 					"retryIntervalInSeconds": 30,
 					"secureOutput": false,
@@ -51,6 +53,9 @@
 		"folder": {
 			"name": "_ProcFwk/_ProcFwkUtils"
 		},
-		"annotations": []
+		"annotations": [
+			"procfwk",
+			"Utils"
+		]
 	}
 }
diff --git a/DataFactory/pipeline/Throw Exception.json b/DataFactory/pipeline/Throw Exception.json
@@ -1,13 +1,15 @@
 {
 	"name": "Throw Exception",
 	"properties": {
+		"description": "Provide a simple way of throwing an exception within Data Factory using TSQL error handling.",
 		"activities": [
 			{
 				"name": "Raise Error",
+				"description": "Using a SQL database to raise an error/exception but wrapped up as a data factory pipeline. Error message information exposed as a pipeline parameter.",
 				"type": "Lookup",
 				"dependsOn": [],
 				"policy": {
-					"timeout": "7.00:00:00",
+					"timeout": "0.00:10:00",
 					"retry": 0,
 					"retryIntervalInSeconds": 30,
 					"secureOutput": false,
@@ -40,6 +42,9 @@
 		"folder": {
 			"name": "_ProcFwk/_ProcFwkUtils"
 		},
-		"annotations": []
+		"annotations": [
+			"procfwk",
+			"Utils"
+		]
 	}
 }
diff --git a/FactoryTesting/Pipelines/01-Grandparent/Given300WorkerPipelines.cs b/FactoryTesting/Pipelines/01-Grandparent/Given300WorkerPipelines.cs
@@ -23,18 +23,18 @@ public async Task WhenPipelineRun()
 
         #region Integration tests
 
-        [Test, Order(1)]
+        [Test]
         public void ThenPipelineOutcomeIsSucceeded()
         {
             _helper.RunOutcome.Should().Be("Succeeded");
         }
-        [Test, Order(2)]
+        [Test]
         public void ThenCurrentExecutionTableIsEmpty()
         {
             _helper.RowCount("procfwk.CurrentExecution").Should().Be(0);
         }
 
-        [Test, Order(3)]
+        [Test]
         public void Then300ExecutionLogRecords()
         {
             _helper.RowCount("procfwk.ExecutionLog").Should().Be(300);