From d3d4bdbeba1f40f13d1d901f5ca08733ffc3475c Mon Sep 17 00:00:00 2001 From: Harivallabha Rangarajan Date: Tue, 25 Jan 2022 16:47:54 +0530 Subject: [PATCH 01/10] Add scripts for scale test automation --- .../queryLoadTestResults.ps1 | 82 ++++++++++++++ .../test-service-load/queryResultsAuto.ps1 | 24 ++++ .../registerScheduledTasks.ps1 | 77 +++++++++++++ .../test-service-load/startScaleTestAuto.ps1 | 104 ++++++++++++++++++ .../test-service-load/stopScaleTestAuto.ps1 | 58 ++++++++++ 5 files changed, 345 insertions(+) create mode 100644 packages/test/test-service-load/queryLoadTestResults.ps1 create mode 100644 packages/test/test-service-load/queryResultsAuto.ps1 create mode 100644 packages/test/test-service-load/registerScheduledTasks.ps1 create mode 100644 packages/test/test-service-load/startScaleTestAuto.ps1 create mode 100644 packages/test/test-service-load/stopScaleTestAuto.ps1 diff --git a/packages/test/test-service-load/queryLoadTestResults.ps1 b/packages/test/test-service-load/queryLoadTestResults.ps1 new file mode 100644 index 000000000000..1f26c31a7d03 --- /dev/null +++ b/packages/test/test-service-load/queryLoadTestResults.ps1 @@ -0,0 +1,82 @@ +<# + Make sure that the appropriate environment variables are set. + Uses the python json.tool to pretty-print the results JSON object. +#> + +function QueryLoadTestResults{ + <# + .SYNOPSIS + Fetches the results of the load test run. + + .DESCRIPTION + Queries the results of the scale test by calling the REST API exposed by AppInsights. + Generates the corresponding CURL command given the KustoQuery, and appropriate parameters. + Saves the results as a pretty-printed JSON object, to an output file. + + .PARAMETER AppID + Application ID for the AppInsights Client + + .PARAMETER APIKey + APIKey to call the AppInsights REST API + + .PARAMETER Timespan + Timespan to look at, in the AppInsight logs + + .PARAMETER KustoQueryName + Name of the KustoQuery to be executed + #> + [CmdletBinding()] + Param( + [Parameter(Mandatory = $false, HelpMessage = 'App ID corresponding to the AppInsight')] + [string]$AppID = "", + + [Parameter(Mandatory = $false, HelpMessage = 'API Key with the relevant permission, for the AppInsight')] + [string]$APIKey = "", + + [Parameter(Mandatory = $false, HelpMessage = 'Timespan for the query')] + [string]$Timespan = "PT12H", + + [Parameter(Mandatory = $false, HelpMessage = 'Kusto Query to get the results of the Load Test')] + [string]$KustoQueryName = "TestRunSummary" + ) + + # AppInsights URL + $SiteURL = "https://api.applicationinsights.io" + + # Get the required environment variables + $LoadTestGuid = [System.Environment]::GetEnvironmentVariable('LoadTestGuid') + $LoadTestStartTime = [System.Environment]::GetEnvironmentVariable('LoadTestStartTime') + $LoadTestStopTime = [System.Environment]::GetEnvironmentVariable('LoadTestStopTime') + + if ($AppID -eq "") + { + $AppID = [System.Environment]::GetEnvironmentVariable('AppInsightAppID') + } + + if ($APIKey -eq "") + { + $APIKey = [System.Environment]::GetEnvironmentVariable('AppInsightAPIKey') + } + + if($KustoQueryName -eq "TestRunSummary") + { + # KustoQuery to fetch the TestRunSummary results + $KustoQuery = 'let searchTestUid = "{0}";let searchTimeStart = datetime({1});let searchTimeEnd = datetime({2});customEvents| where timestamp between(searchTimeStart .. searchTimeEnd)| extend testUid = tostring(customDimensions.testUid)| where testUid == searchTestUid| extend category = tostring(customDimensions.category), error = tostring(customDimensions.error), runId = toint(customDimensions.runId), eventName = tostring(customDimensions.eventName)| where category == "error" and error !has "deprecated" and error !contains "fluid:telemetry:SummaryStatus Behind" and eventName != "fluid:telemetry:SummaryStatus:Behind" and error !has "MaxListenersExceededWarning" and eventName != "Runner Error"| summarize errCnt = count() by error, eventName| summarize errCount = sum(errCnt), errors = make_bag(pack(iif(isnotempty(error), error, "Unknown"), errCnt)) by eventName| order by errCount' -f $LoadTestGuid,$LoadTestStartTime,$LoadTestStopTime + } + + # Encode Kusto Query into the URI space + $URLQuery = [uri]::EscapeDataString($KustoQuery) + + $SaveFileSuffix = "_{0}.txt" -f $KustoQueryName + $SaveResults = (Get-Location).ToString() +"\out\" + ` + (Get-Date -Format "dddd MM_dd_yyyy HH_mm").ToString() + $SaveFileSuffix + + # CURL command to fetch the results as a JSON object, and pretty print & pipe the JSON into an output file + curl.exe "$SiteURL/v1/apps/$AppID/query?timespan=$Timespan&query=$URLQuery" -H "x-api-key: $APIKey" ` + | python -m json.tool | Out-File -FilePath $SaveResults + + Write-Host "Results of the latest load test, TestGuid: $LoadTestGuid fetched and written to $SaveResults" ` + -ForegroundColor Green +} + + diff --git a/packages/test/test-service-load/queryResultsAuto.ps1 b/packages/test/test-service-load/queryResultsAuto.ps1 new file mode 100644 index 000000000000..be8e79bfa549 --- /dev/null +++ b/packages/test/test-service-load/queryResultsAuto.ps1 @@ -0,0 +1,24 @@ +<# + Script to fetch the results at the end of a load test run. + Triggered as a scheduled task at the specified time. + + Make sure that the appropriate environment variables are set. +#> + +Param( + [Parameter(Mandatory = $false, HelpMessage = 'Load Test Scripts Directory')] + [string]$LoadTestDir = "C:\loadtest" +) + +# Make sure this points to the right directory in your local setup (wherever the load test scripts are present) +cd $LoadTestDir + +# Register the QueryLoadTestResults cmdlet function +. .\queryLoadTestResults.ps1 + +# Fetch the required environment variables +$AppID = [System.Environment]::GetEnvironmentVariable('AppInsightAppID') +$APIKey = [System.Environment]::GetEnvironmentVariable('AppInsightAPIKey') + +Write-Host "Fetching the Latest Results" -ForegroundColor Green +QueryLoadTestResults -KustoQueryName "TestRunSummary" -AppID $AppID -APIKey $APIKey \ No newline at end of file diff --git a/packages/test/test-service-load/registerScheduledTasks.ps1 b/packages/test/test-service-load/registerScheduledTasks.ps1 new file mode 100644 index 000000000000..d23a8401f2a0 --- /dev/null +++ b/packages/test/test-service-load/registerScheduledTasks.ps1 @@ -0,0 +1,77 @@ +<# + Script to register and run startScaleTestAuto.ps1, stopScaleTestAuto.ps1 and queryResultsAuto.ps1 + as Scheduled Tasks. + + Starts the Fluid Scale test at the StartTime specified, and stops the test at the specified StopTime. + It then fetches the results of the scale test by running the queryResultsAuto.ps1 script at QueryTime. + + The times are in Coordinated Universal Time (UTC) unless specified otherwise. +#> + +Param( + [Parameter(Mandatory = $false, HelpMessage = 'Start time for the load test')] + [string]$StartTime = "03:30 am", + + [Parameter(Mandatory = $false, HelpMessage = 'Location of the start script')] + [string]$StartScriptPath = "C:\loadtest\startScaleTestAuto.ps1", + + [Parameter(Mandatory = $false, HelpMessage = 'Stop time for the load test')] + [string]$StopTime = "01:30 pm", + + [Parameter(Mandatory = $false, HelpMessage = 'Location of the stop script')] + [string]$StopScriptPath = "C:\loadtest\stopScaleTestAuto.ps1", + + [Parameter(Mandatory = $false, HelpMessage = 'Time at which load test results are to be fetched')] + [string]$QueryTime = "02:00 pm", + + [Parameter(Mandatory = $false, HelpMessage = 'Location of the fetch results script')] + [string]$QueryScriptPath = "C:\loadtest\queryResultsAuto.ps1" +) + +function registerTask { + <# + .SYNOPSIS + Registers the task with the given name, to be triggered at the specified time. + + .PARAMETER TaskName + Name of the Scheduled Task + + .PARAMETER ScheduledTime + Time at which the Scheduled Task is to be triggered + + .PARAMETER ScriptPath + Path to the script that is to be executed as a scheduled task + #> + Param( + [string]$TaskName, + [Parameter()] + [string]$ScheduledTime, + [Parameter()] + [string]$ScriptPath + ) + + # Initialize common variables + $ExecProg = "C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe" # Make sure this points to the appropriate location in your setup + $Principal = New-ScheduledTaskPrincipal -UserID "$($env:USERDOMAIN)\$($env:USERNAME)" ` + -LogonType ServiceAccount -RunLevel Highest + $Settings = New-ScheduledTaskSettingsSet -Compatibility Win8 + + # Execute Action at the given Trigger + $Trigger = New-ScheduledTaskTrigger -Daily -At $ScheduledTime + $Action = New-ScheduledTaskAction -Execute $ExecProg -Argument "-File $ScriptPath" + Register-ScheduledTask -TaskName $TaskName -Trigger $Trigger -Action $Action ` + -Principal $Principal -Settings $Settings + $OutputMessage ="Successfully Registered the {0} Scheduled Task" -f $TaskName + Write-Host $OutputMessage -ForegroundColor Green +} + + +# Register the Start-Load-Test task at the specified start time +registerTask -TaskName "Start-Load-Test" -ScheduledTime $StartTime -ScriptPath $StartScriptPath + +# Register the Stop-Load-Test task at the specified stop time +registerTask -TaskName "Stop-Load-Test" -ScheduledTime $StopTime -ScriptPath $StopScriptPath + +# Register the fetch results task at the specified time +registerTask -TaskName "Fetch-Load-Test-Results" -ScheduledTime $QueryTime -ScriptPath $QueryScriptPath + diff --git a/packages/test/test-service-load/startScaleTestAuto.ps1 b/packages/test/test-service-load/startScaleTestAuto.ps1 new file mode 100644 index 000000000000..36f425a0845e --- /dev/null +++ b/packages/test/test-service-load/startScaleTestAuto.ps1 @@ -0,0 +1,104 @@ +<# + Script to start the scale test runs: + 1. Logs in to the AKS portal using a service principal + 2. Powers on the Managed Cluster + 3. Scales up the nodes + 4. Triggers the scale test + 5. Deletes the namespace after a specified time, to handle the cold start issue + 6. Re-Triggers the scale test + + Make sure that all the appropriate environment variables are set. +#> + +Param( + [Parameter(Mandatory = $false, HelpMessage = 'Number of documents to run test on')] + [ValidateRange(1, 2400)] + [int]$NumOfDocs = 10, + + [Parameter(Mandatory = $false, HelpMessage = 'Profile to run test with')] + [string]$Profile = 'scale', + + [Parameter(Mandatory = $false, HelpMessage = 'AKS Namespace.')] + [ValidateScript({ ( $NumOfDocs -le 10 ) -or ($_ -eq 'fluid-scale-test' ) })] + [string]$Namespace = 'fluid-scale-test', + + [Parameter(Mandatory = $false, HelpMessage = 'Folder to create in Storage for test files')] + [string]$TestDocFolder = [Math]::Floor([decimal](Get-Date(Get-Date).ToUniversalTime() -uformat '%s')), + + [Parameter(Mandatory = $false, HelpMessage = 'File with tenants and users information')] + [string]$TestTenantConfig = '.\testTenantConfig_vmss.json', + + [Parameter(Mandatory = $false, HelpMessage = 'Number of nodes to which the node pool should be scaled')] + [string]$NodeCount = '5', + + [Parameter(Mandatory = $false, HelpMessage = 'Name of the resource group')] + [string]$ResourceGroup= 'Fluid.Load.Test', + + [Parameter(Mandatory = $false, HelpMessage = 'Name of the AKS Cluster')] + [string]$AKSClusterName = 'LoadTestAks1', + + [Parameter(Mandatory = $false, HelpMessage = 'Name of the node pool to be scaled')] + [string]$NodePoolName = 'testpods', + + # Restart the load test after a specified time to avoid the Cold Start issue + [Parameter(Mandatory = $false, HelpMessage = 'Time (in min) after which the scale test is re-triggered')] + [int]$RestartTime = 60, + + [Parameter(Mandatory = $false, HelpMessage = 'Load Test Scripts Directory')] + [string]$LoadTestDir = "C:\loadtest" +) + +# Make sure this points to the right directory in your local setup (wherever the load test scripts are present) +cd $LoadTestDir + +Write-Host "Logging in to the AKS portal" -ForegroundColor Green +# Service principal based authentication +az login --service-principal -u $env:AKSServicePrincipalID -p $env:AKSServicePrincipalKey --tenant $env:AKSTenant + +Write-Host "Loading variables and preparing for run" -ForegroundColor Green +. .\runloadtest.ps1 + +Write-Host "Powering On Managed Cluster" -ForegroundColor Green +az aks start -n $AKSClusterName -g $ResourceGroup + +Write-Host "Scaling the Node Pool" -ForegroundColor Green +# Manually scale the Node Pool as appropriate +$_ScaleNodesJob = { param($nodecount, $resourcegroup, $aksclustername, $nodepoolname) ` + az aks scale --resource-group $resourcegroup --name $aksclustername ` + --node-count $nodecount --nodepool-name $nodepoolname + } +$ScaleNodesJob = Start-Job $_ScaleNodesJob -ArgumentList $NodeCount, ` + $ResourceGroup, ` + $AKSClusterName, ` + $NodePoolName +Wait-Job $ScaleNodesJob +Receive-Job $ScaleNodesJob +Write-Host "Scaling Node Pool Finished" -ForegroundColor Green + +Write-Host "Triggering Load Test" -ForegroundColor Green +# Trigger the first run of the load test +RunLoadTest -Profile $Profile -NumOfDocs $NumOfDocs -TestTenantConfig $TestTenantConfig + +# Waits for a specified amount of time after which the load test is re-triggered +Start-Sleep -Seconds (1*60*$RestartTime) + +# End the first run, and re-trigger a second. This is done to tackle the cold-start issue we often encounter +Write-Host "Ending the Load Test" -ForegroundColor Green +$DeleteNamespaceJob = { param($namespace) ` + kubectl delete ns $namespace + } +$DeleteJob = Start-Job $DeleteNamespaceJob -ArgumentList $Namespace +Wait-Job $DeleteJob +Receive-Job $DeleteJob + +# Sets the StartTime environment variable - which will later be used by the queryLoadTestResults script +$StartTime=(Get-Date -Format "yyyy-MM-ddTHH:mm").ToString() +[System.Environment]::SetEnvironmentVariable('LoadTestStartTime', ` + $StartTime, ` + [System.EnvironmentVariableTarget]::User) + +Write-Host "Re-Triggering Load Test" -ForegroundColor Green +# Modify $OutFile as appropriate to point to the location where you want to save the output of `RunLoadTest` +$OutFile = (Get-Location).ToString() +"\out\" + (Get-Date -Format "dddd MM_dd_yyyy HH_mm").ToString() + ".txt" +RunLoadTest -Profile $Profile -NumOfDocs $NumOfDocs -TestTenantConfig $TestTenantConfig ` + | Out-File -FilePath $OutFile \ No newline at end of file diff --git a/packages/test/test-service-load/stopScaleTestAuto.ps1 b/packages/test/test-service-load/stopScaleTestAuto.ps1 new file mode 100644 index 000000000000..f3b9e8d17eb8 --- /dev/null +++ b/packages/test/test-service-load/stopScaleTestAuto.ps1 @@ -0,0 +1,58 @@ +<# + Script to stop the scale test runs: + 1. Deletes the namespace + 2. Scales down the nodes + 3. Powers off the Managed Cluster + + Make sure that all the appropriate environment variables are set. +#> + +Param( + [Parameter(Mandatory = $false, HelpMessage = 'AKS Namespace.')] + [ValidateScript({ ( $NumOfDocs -le 10 ) -or ($_ -eq 'fluid-scale-test' ) })] + [string]$Namespace = 'fluid-scale-test', + + [Parameter(Mandatory = $false, HelpMessage = 'Number of nodes to which the node pool should be scaled')] + [string]$NodeCount = '0', + + [Parameter(Mandatory = $false, HelpMessage = 'Name of the resource group')] + [string]$ResourceGroup= 'Fluid.Load.Test', + + [Parameter(Mandatory = $false, HelpMessage = 'Name of the AKS Cluster')] + [string]$AKSClusterName = 'LoadTestAks1', + + [Parameter(Mandatory = $false, HelpMessage = 'Name of the node pool to be scaled')] + [string]$NodePoolName = 'testpods', + + [Parameter(Mandatory = $false, HelpMessage = 'Load Test Scripts Directory')] + [string]$LoadTestDir = "C:\loadtest" +) + +# Make sure this points to the right directory in your local setup (wherever the load test scripts are present) +cd $LoadTestDir + +Write-Host "Deleting namespace " -ForegroundColor Green +kubectl delete ns $Namespace + +Write-Host "Scaling down the node pool " -ForegroundColor Green +$_ScaleNodesJob = { param($nodecount, $resourcegroup, $aksclustername, $nodepoolname) ` + az aks scale --resource-group $resourcegroup --name $aksclustername ` + --node-count $nodecount --nodepool-name $nodepoolname + } +$ScaleNodesJob = Start-Job $_ScaleNodesJob -ArgumentList $NodeCount, ` + $ResourceGroup, ` + $AKSClusterName, ` + $NodePoolName +Wait-Job $ScaleNodesJob +Receive-Job $ScaleNodesJob +Write-Host "Finished scaling down the node pool " -ForegroundColor Green + +Write-Host "Powering Off Managed Cluster " -ForegroundColor Green +az aks stop -n $AKSClusterName -g $ResourceGroup +Write-Host "Powered Off Managed Cluster " -ForegroundColor Green + +# Sets the StopTime environment variable - which will later be used by the queryLoadTestResults script +$StopTime=(Get-Date -Format "yyyy-MM-ddTHH:mm").ToString() +[System.Environment]::SetEnvironmentVariable('LoadTestStopTime', ` + $StopTime, ` + [System.EnvironmentVariableTarget]::User) \ No newline at end of file From 5ffebbfa08c706e36a9ff66ab20e93f0edfe59eb Mon Sep 17 00:00:00 2001 From: Harivallabha Rangarajan Date: Wed, 26 Jan 2022 21:37:03 +0530 Subject: [PATCH 02/10] Pipe trigger message to output file --- packages/test/test-service-load/runloadtest.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/test/test-service-load/runloadtest.ps1 b/packages/test/test-service-load/runloadtest.ps1 index 320b73fc0705..ef28237d57ad 100644 --- a/packages/test/test-service-load/runloadtest.ps1 +++ b/packages/test/test-service-load/runloadtest.ps1 @@ -32,7 +32,9 @@ function RunLoadTest { # Create and upload configs for pods to trigger tests. CreateAndUploadConfig -TestTenantConfig $TestTenantConfig -TestUid $TestUid - Write-Output "Triggered LoadTest for TestUid: $TestUid TestDocFolder: $TestDocFolder" + $LatestRunSaveFilePath = (Get-Location).ToString() +"\out\latest_run.txt" + Write-Output "Triggered LoadTest for TestUid: $TestUid TestDocFolder: $TestDocFolder" ` + | Out-File -FilePath $LatestRunSaveFilePath } function CreateInfra { From 5a0ed78e3eabb5327fa97f8f3f73e8c9520e8d44 Mon Sep 17 00:00:00 2001 From: Harivallabha Rangarajan Date: Wed, 26 Jan 2022 21:47:56 +0530 Subject: [PATCH 03/10] create output folder if it does not exist --- packages/test/test-service-load/runloadtest.ps1 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/test/test-service-load/runloadtest.ps1 b/packages/test/test-service-load/runloadtest.ps1 index ef28237d57ad..f448d7a4730e 100644 --- a/packages/test/test-service-load/runloadtest.ps1 +++ b/packages/test/test-service-load/runloadtest.ps1 @@ -33,6 +33,11 @@ function RunLoadTest { CreateAndUploadConfig -TestTenantConfig $TestTenantConfig -TestUid $TestUid $LatestRunSaveFilePath = (Get-Location).ToString() +"\out\latest_run.txt" + $OutDir = (Get-Location).ToString() +"\out" + If(!(test-path $OutDir)) + { + New-Item -ItemType Directory -Force -Path $OutDir | Out-Null + } Write-Output "Triggered LoadTest for TestUid: $TestUid TestDocFolder: $TestDocFolder" ` | Out-File -FilePath $LatestRunSaveFilePath } From 3d78eca7b489c2b96004e87328d6aa518afa62b3 Mon Sep 17 00:00:00 2001 From: Harivallabha Rangarajan Date: Mon, 31 Jan 2022 23:05:31 +0530 Subject: [PATCH 04/10] automate the process of merging upstream changes periodically --- .../mergeUpstreamMainAuto.ps1 | 32 +++++++++++++++++++ .../registerScheduledTasks.ps1 | 32 ++++++++++++++++--- 2 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 packages/test/test-service-load/mergeUpstreamMainAuto.ps1 diff --git a/packages/test/test-service-load/mergeUpstreamMainAuto.ps1 b/packages/test/test-service-load/mergeUpstreamMainAuto.ps1 new file mode 100644 index 000000000000..55b125204d74 --- /dev/null +++ b/packages/test/test-service-load/mergeUpstreamMainAuto.ps1 @@ -0,0 +1,32 @@ +<# + Script to merge the changes in main branch with the loadtest branch at the specified time, + to generate updated builds. + + Ensure that remote origin and remote upstream point to appropriate repositories. + + Merge conflicts need to be resolved manually. +#> + +Param( + [Parameter(Mandatory = $false, HelpMessage = 'Fluid Framework CodeBase Directory')] + [string]$LoadTestDir = "C:\FluidFrameworkCodebase\FluidFramework" +) + +# Make sure this points to the right directory in your local setup (wherever the load test scripts are present) +cd $LoadTestDir + +Write-Host "Checkout local main branch" -ForegroundColor "Green" +git checkout main +Write-Host "Pull the changes from upstream main" -ForegroundColor "Green" +git pull upstream main +Write-Host "Push the changes to origin main" -ForegroundColor "Green" +git push origin main +Write-Host "Checkout local loadtest branch" -ForegroundColor "Green" +git checkout loadtest/main + +# Merge conflicts would need to be resolved manually +Write-Host "Merge the changes made in the main branch, with the loadtest branch" -ForegroundColor "Green" +git merge main +Write-Host "Push changes from local loadtest branch to the branch in remote origin" -ForegroundColor "Green" +git push origin loadtest/main +Write-Host "Changes updated successfully" -ForegroundColor "Green" \ No newline at end of file diff --git a/packages/test/test-service-load/registerScheduledTasks.ps1 b/packages/test/test-service-load/registerScheduledTasks.ps1 index d23a8401f2a0..2f87fc0fb9b2 100644 --- a/packages/test/test-service-load/registerScheduledTasks.ps1 +++ b/packages/test/test-service-load/registerScheduledTasks.ps1 @@ -6,6 +6,9 @@ It then fetches the results of the scale test by running the queryResultsAuto.ps1 script at QueryTime. The times are in Coordinated Universal Time (UTC) unless specified otherwise. + + Presently, frequency for the Start-Load-Test, Stop-Load-Test and Fetch-Load-Test-Results tasks + is Daily, and that of the Merge-From-Upstream task is once every 3 days. #> Param( @@ -25,7 +28,13 @@ Param( [string]$QueryTime = "02:00 pm", [Parameter(Mandatory = $false, HelpMessage = 'Location of the fetch results script')] - [string]$QueryScriptPath = "C:\loadtest\queryResultsAuto.ps1" + [string]$QueryScriptPath = "C:\loadtest\queryResultsAuto.ps1", + + [Parameter(Mandatory = $false, HelpMessage = 'Time at which changes from upstream are to be updated')] + [string]$UpdateTime = "03:00 am", + + [Parameter(Mandatory = $false, HelpMessage = 'Location of the merge-from-upstream script')] + [string]$UpdateScriptPath = "C:\loadtest\mergeUpstreamMainAuto.ps1" ) function registerTask { @@ -39,14 +48,20 @@ function registerTask { .PARAMETER ScheduledTime Time at which the Scheduled Task is to be triggered + .PARAMETER Frequency + Frequency (in units of Days) at which the Scheduled Task is to be triggered + .PARAMETER ScriptPath Path to the script that is to be executed as a scheduled task #> Param( + [Parameter()] [string]$TaskName, [Parameter()] [string]$ScheduledTime, [Parameter()] + [string]$Frequency, + [Parameter()] [string]$ScriptPath ) @@ -57,7 +72,7 @@ function registerTask { $Settings = New-ScheduledTaskSettingsSet -Compatibility Win8 # Execute Action at the given Trigger - $Trigger = New-ScheduledTaskTrigger -Daily -At $ScheduledTime + $Trigger = New-ScheduledTaskTrigger -Daily -DaysInterval $Frequency -At $ScheduledTime $Action = New-ScheduledTaskAction -Execute $ExecProg -Argument "-File $ScriptPath" Register-ScheduledTask -TaskName $TaskName -Trigger $Trigger -Action $Action ` -Principal $Principal -Settings $Settings @@ -65,13 +80,20 @@ function registerTask { Write-Host $OutputMessage -ForegroundColor Green } +# Register the Merge-From-Upstream task at the specified start time +registerTask -TaskName "Merge-From-Upstream" -ScheduledTime $UpdateTime ` +-Frequency "3" -ScriptPath $UpdateScriptPath # Register the Start-Load-Test task at the specified start time -registerTask -TaskName "Start-Load-Test" -ScheduledTime $StartTime -ScriptPath $StartScriptPath +registerTask -TaskName "Start-Load-Test" -ScheduledTime $StartTime ` +-Frequency "1" -ScriptPath $StartScriptPath # Register the Stop-Load-Test task at the specified stop time -registerTask -TaskName "Stop-Load-Test" -ScheduledTime $StopTime -ScriptPath $StopScriptPath +registerTask -TaskName "Stop-Load-Test" -ScheduledTime $StopTime ` +-Frequency "1" -ScriptPath $StopScriptPath # Register the fetch results task at the specified time -registerTask -TaskName "Fetch-Load-Test-Results" -ScheduledTime $QueryTime -ScriptPath $QueryScriptPath +registerTask -TaskName "Fetch-Load-Test-Results" -ScheduledTime $QueryTime ` +-Frequency "1" -ScriptPath $QueryScriptPath + From 15eb5e8455f185758abadf6444296ca800db8d79 Mon Sep 17 00:00:00 2001 From: Harivallabha Rangarajan Date: Wed, 2 Feb 2022 06:20:46 +0530 Subject: [PATCH 05/10] Add programmatic mailer for the results --- packages/test/test-service-load/mailAuto.ps1 | 125 ++++++++++++++++++ .../test/test-service-load/mailAutoHelper.ps1 | 14 ++ .../queryLoadTestResults.ps1 | 10 +- .../registerScheduledTasks.ps1 | 20 ++- 4 files changed, 160 insertions(+), 9 deletions(-) create mode 100644 packages/test/test-service-load/mailAuto.ps1 create mode 100644 packages/test/test-service-load/mailAutoHelper.ps1 diff --git a/packages/test/test-service-load/mailAuto.ps1 b/packages/test/test-service-load/mailAuto.ps1 new file mode 100644 index 000000000000..58a395499cb5 --- /dev/null +++ b/packages/test/test-service-load/mailAuto.ps1 @@ -0,0 +1,125 @@ +<# + Script to programmatically mail the results to + the specified users, at the end of a scale test run. + + Make sure that the appropriate environment variables are set. +#> + +Param( + + [Parameter(Mandatory = $false, HelpMessage = 'Mail Address of the Sender')] + [string]$MailFrom = "", + + [Parameter(Mandatory = $false, HelpMessage = 'Mail Address of the Recipient')] + [string]$MailTo = "", + + [Parameter(Mandatory = $false, HelpMessage = 'Mail Address of Additional Recipient')] + [string]$MailCC = "", + + [Parameter(Mandatory = $false, HelpMessage = 'Subject of the Mail')] + [string]$MailSubject = "Scale Test Results", + + [Parameter(Mandatory = $false, HelpMessage = 'Full Path of the results file to be attached')] + [string]$AttachmentPath = [System.Environment]::GetEnvironmentVariable('LoadTestResultsFile') + +) + +function Invoke-SetProperty { + # Reflection to set the SendUsingAccount property + Param( + [__ComObject] $Object, + [String] $Property, + $Value + ) + + [Void] $Object.GetType().InvokeMember($Property,"SetProperty",$NULL,$Object,$Value) +} + +function MailUsingOutlookPowershell { + <# + .SYNOPSIS + Creates an Outlook COM Object, and sends a mail + with the desired parameters. + + You should ensure that the Outlook client + is setup with the appropriate accounts + #> + $SendFromSmtpAddress = $MailFrom + + # Create COM object named Outlook + $Outlook = New-Object -ComObject Outlook.Application + $Account = $Outlook.session.accounts | ? { $_.smtpAddress -eq $SendFromSmtpAddress } + + # Create Outlook MailItem named Mail using CreateItem() method + $Mail = $Outlook.CreateItem(0) + + # Add properties as desired + $Mail.To = $MailTo + $Mail.CC = $MailCC + $Mail.Subject = $MailSubject + $Mail.Body = $MailBody + $Attachment = $AttachmentPath + $Mail.Attachments.Add($Attachment) + + # Send message + Invoke-SetProperty -Object $Mail -Property "SendUsingAccount" -Value $Account + $Mail.Send() + + # Wait for sometime before quitting outlook + Start-Sleep -Seconds (1*30) + + # Quit and Cleanup + $Outlook.Quit() + [System.Runtime.Interopservices.Marshal]::ReleaseComObject($Outlook) | Out-Null +} + +function MailUsingSendMailMessage { + <# + .SYNOPSIS + Sends a mail using the `Send-MailMessage` cmdlet. + Will be deprecated soon, so you should migrate to + `MailUsingOutlookPowershell` above. + + You should ensure that the appropriate environment + variables are set, and that the credentials are available. + #> + $PassLoc = [System.Environment]::GetEnvironmentVariable('MailFromPasswordPath') + $Username = $MailFrom + $Password = Get-Content $PassLoc | ConvertTo-SecureString + $Cred = New-Object -typename System.Management.Automation.PSCredential ` + -argumentlist $Username, $Password + + $MailHash = @{ + To = $MailTo + From = $MailFrom + Subject = $MailSubject + Body = $MailBody + CC = $MailCC + BodyAsHtml = $true + SmtpServer = 'smtp.office365.com' + UseSSL = $true + Credential = $Cred + Port = 587 + Attachments = $AttachmentPath + } + + Send-MailMessage @MailHash +} + +$TestGuid = [System.Environment]::GetEnvironmentVariable('LoadTestGuid') +$MailBody = "Results of the Load Test with GUID: {0}" -f $TestGuid + +<# + TO-DO: Deprecate this shortly, and use the alternative `MailUsingOutlookPowershell` + for the programmatic mailer after setting up the appropriate account(s) on Outlook. +#> +Write-Host "Mailing the results of the load test" -ForegroundColor Green +MailUsingSendMailMessage +#MailUsingOutlookPowershell +Write-Host "Successfully mailed the results of the load test" -ForegroundColor Green + +<# + Acknowledgements: + [1] https://gist.github.com/ClaudioESSilva/dfaf1de2e5da88fca1e59f70edd7f4ae [ClaudioESSilva] [Mail with Outlook Powershell] + [2] https://sid-500.com/2020/08/25/microsoft-365-send-e-mails-with-powershell/ [Patrick Gruenauer] [Mail with Send-MailMessage] +#> \ No newline at end of file diff --git a/packages/test/test-service-load/mailAutoHelper.ps1 b/packages/test/test-service-load/mailAutoHelper.ps1 new file mode 100644 index 000000000000..a16c63ec42a8 --- /dev/null +++ b/packages/test/test-service-load/mailAutoHelper.ps1 @@ -0,0 +1,14 @@ +<# + Helper script to read in the password for the automatic mailer, + and save the path to the password (stored as a secure string), + as an appropriate environment variable. +#> + +$PasswordLocation=(Get-Location).ToString() + "\Upass.txt" +# Enter the password, to be stored as a secure string +Write-Host "Enter the password for the automatic mailer" -ForegroundColor Cyan +read-host -assecurestring | convertfrom-securestring | out-file $PasswordLocation +# Store the location of the password in an environment variable +[System.Environment]::SetEnvironmentVariable('MailFromPasswordPath', ` + $PasswordLocation, ` + [System.EnvironmentVariableTarget]::User) \ No newline at end of file diff --git a/packages/test/test-service-load/queryLoadTestResults.ps1 b/packages/test/test-service-load/queryLoadTestResults.ps1 index 1f26c31a7d03..226d22258d7d 100644 --- a/packages/test/test-service-load/queryLoadTestResults.ps1 +++ b/packages/test/test-service-load/queryLoadTestResults.ps1 @@ -74,9 +74,11 @@ function QueryLoadTestResults{ # CURL command to fetch the results as a JSON object, and pretty print & pipe the JSON into an output file curl.exe "$SiteURL/v1/apps/$AppID/query?timespan=$Timespan&query=$URLQuery" -H "x-api-key: $APIKey" ` | python -m json.tool | Out-File -FilePath $SaveResults - + + [System.Environment]::SetEnvironmentVariable('LoadTestResultsFile', ` + $SaveResults, ` + [System.EnvironmentVariableTarget]::User) + Write-Host "Results of the latest load test, TestGuid: $LoadTestGuid fetched and written to $SaveResults" ` -ForegroundColor Green -} - - +} \ No newline at end of file diff --git a/packages/test/test-service-load/registerScheduledTasks.ps1 b/packages/test/test-service-load/registerScheduledTasks.ps1 index 2f87fc0fb9b2..37ad129034c0 100644 --- a/packages/test/test-service-load/registerScheduledTasks.ps1 +++ b/packages/test/test-service-load/registerScheduledTasks.ps1 @@ -1,14 +1,15 @@ <# - Script to register and run startScaleTestAuto.ps1, stopScaleTestAuto.ps1 and queryResultsAuto.ps1 - as Scheduled Tasks. + Script to register and run startScaleTestAuto.ps1, stopScaleTestAuto.ps1, queryResultsAuto.ps1, + mergeUpstreamMainAuto.ps1, and mailAuto.ps1 as Scheduled Tasks. Starts the Fluid Scale test at the StartTime specified, and stops the test at the specified StopTime. It then fetches the results of the scale test by running the queryResultsAuto.ps1 script at QueryTime. + Finally, the results of the load test are mailed to the appropriate mailing list. The times are in Coordinated Universal Time (UTC) unless specified otherwise. - Presently, frequency for the Start-Load-Test, Stop-Load-Test and Fetch-Load-Test-Results tasks - is Daily, and that of the Merge-From-Upstream task is once every 3 days. + Presently, frequency for the Start-Load-Test, Stop-Load-Test, Fetch-Load-Test-Results and + Mail-Load-Test-Results tasks is Daily, and that of the Merge-From-Upstream task is once every 3 days. #> Param( @@ -30,11 +31,18 @@ Param( [Parameter(Mandatory = $false, HelpMessage = 'Location of the fetch results script')] [string]$QueryScriptPath = "C:\loadtest\queryResultsAuto.ps1", + [Parameter(Mandatory = $false, HelpMessage = 'Time at which load test results are to be mailed')] + [string]$MailTime = "02:05 pm", + + [Parameter(Mandatory = $false, HelpMessage = 'Location of the programmatic results mailer script')] + [string]$MailScriptPath = "C:\loadtest\mailAuto.ps1", + [Parameter(Mandatory = $false, HelpMessage = 'Time at which changes from upstream are to be updated')] [string]$UpdateTime = "03:00 am", [Parameter(Mandatory = $false, HelpMessage = 'Location of the merge-from-upstream script')] [string]$UpdateScriptPath = "C:\loadtest\mergeUpstreamMainAuto.ps1" + ) function registerTask { @@ -96,4 +104,6 @@ registerTask -TaskName "Stop-Load-Test" -ScheduledTime $StopTime ` registerTask -TaskName "Fetch-Load-Test-Results" -ScheduledTime $QueryTime ` -Frequency "1" -ScriptPath $QueryScriptPath - +# Register the mail results task at the specified time +registerTask -TaskName "Mail-Load-Test-Results" -ScheduledTime $MailTime ` +-Frequency "1" -ScriptPath $MailScriptPath \ No newline at end of file From 80c3d0da172222e06c0c20fcdf4cebee919dadc4 Mon Sep 17 00:00:00 2001 From: Harivallabha Rangarajan Date: Wed, 9 Feb 2022 14:22:12 +0530 Subject: [PATCH 06/10] Lint fixes --- packages/test/test-service-load/mailAutoHelper.ps1 | 6 +++--- .../test-service-load/registerScheduledTasks.ps1 | 2 +- .../test/test-service-load/startScaleTestAuto.ps1 | 8 ++++---- .../test/test-service-load/stopScaleTestAuto.ps1 | 13 ++++++------- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/packages/test/test-service-load/mailAutoHelper.ps1 b/packages/test/test-service-load/mailAutoHelper.ps1 index a16c63ec42a8..5ac53ef49ed3 100644 --- a/packages/test/test-service-load/mailAutoHelper.ps1 +++ b/packages/test/test-service-load/mailAutoHelper.ps1 @@ -4,11 +4,11 @@ as an appropriate environment variable. #> -$PasswordLocation=(Get-Location).ToString() + "\Upass.txt" +$PasswordLocation = (Get-Location).ToString() + "\Upass.txt" # Enter the password, to be stored as a secure string Write-Host "Enter the password for the automatic mailer" -ForegroundColor Cyan read-host -assecurestring | convertfrom-securestring | out-file $PasswordLocation # Store the location of the password in an environment variable [System.Environment]::SetEnvironmentVariable('MailFromPasswordPath', ` - $PasswordLocation, ` - [System.EnvironmentVariableTarget]::User) \ No newline at end of file + $PasswordLocation, ` + [System.EnvironmentVariableTarget]::User) \ No newline at end of file diff --git a/packages/test/test-service-load/registerScheduledTasks.ps1 b/packages/test/test-service-load/registerScheduledTasks.ps1 index 37ad129034c0..2cb0e49f46cb 100644 --- a/packages/test/test-service-load/registerScheduledTasks.ps1 +++ b/packages/test/test-service-load/registerScheduledTasks.ps1 @@ -84,7 +84,7 @@ function registerTask { $Action = New-ScheduledTaskAction -Execute $ExecProg -Argument "-File $ScriptPath" Register-ScheduledTask -TaskName $TaskName -Trigger $Trigger -Action $Action ` -Principal $Principal -Settings $Settings - $OutputMessage ="Successfully Registered the {0} Scheduled Task" -f $TaskName + $OutputMessage = "Successfully Registered the {0} Scheduled Task" -f $TaskName Write-Host $OutputMessage -ForegroundColor Green } diff --git a/packages/test/test-service-load/startScaleTestAuto.ps1 b/packages/test/test-service-load/startScaleTestAuto.ps1 index 36f425a0845e..d84049eb9c59 100644 --- a/packages/test/test-service-load/startScaleTestAuto.ps1 +++ b/packages/test/test-service-load/startScaleTestAuto.ps1 @@ -18,7 +18,7 @@ Param( [Parameter(Mandatory = $false, HelpMessage = 'Profile to run test with')] [string]$Profile = 'scale', - [Parameter(Mandatory = $false, HelpMessage = 'AKS Namespace.')] + [Parameter(Mandatory = $false, HelpMessage = 'AKS Namespace')] [ValidateScript({ ( $NumOfDocs -le 10 ) -or ($_ -eq 'fluid-scale-test' ) })] [string]$Namespace = 'fluid-scale-test', @@ -32,7 +32,7 @@ Param( [string]$NodeCount = '5', [Parameter(Mandatory = $false, HelpMessage = 'Name of the resource group')] - [string]$ResourceGroup= 'Fluid.Load.Test', + [string]$ResourceGroup = 'Fluid.Load.Test', [Parameter(Mandatory = $false, HelpMessage = 'Name of the AKS Cluster')] [string]$AKSClusterName = 'LoadTestAks1', @@ -92,13 +92,13 @@ Wait-Job $DeleteJob Receive-Job $DeleteJob # Sets the StartTime environment variable - which will later be used by the queryLoadTestResults script -$StartTime=(Get-Date -Format "yyyy-MM-ddTHH:mm").ToString() +$StartTime = (Get-Date -Format "yyyy-MM-ddTHH:mm").ToString() [System.Environment]::SetEnvironmentVariable('LoadTestStartTime', ` $StartTime, ` [System.EnvironmentVariableTarget]::User) Write-Host "Re-Triggering Load Test" -ForegroundColor Green # Modify $OutFile as appropriate to point to the location where you want to save the output of `RunLoadTest` -$OutFile = (Get-Location).ToString() +"\out\" + (Get-Date -Format "dddd MM_dd_yyyy HH_mm").ToString() + ".txt" +$OutFile = (Get-Location).ToString() +"\out\" + (Get-Date -Format "dddd MM_dd_yyyy HH_mm").ToString() + ".txt" RunLoadTest -Profile $Profile -NumOfDocs $NumOfDocs -TestTenantConfig $TestTenantConfig ` | Out-File -FilePath $OutFile \ No newline at end of file diff --git a/packages/test/test-service-load/stopScaleTestAuto.ps1 b/packages/test/test-service-load/stopScaleTestAuto.ps1 index f3b9e8d17eb8..295b6f3d26db 100644 --- a/packages/test/test-service-load/stopScaleTestAuto.ps1 +++ b/packages/test/test-service-load/stopScaleTestAuto.ps1 @@ -8,15 +8,14 @@ #> Param( - [Parameter(Mandatory = $false, HelpMessage = 'AKS Namespace.')] - [ValidateScript({ ( $NumOfDocs -le 10 ) -or ($_ -eq 'fluid-scale-test' ) })] + [Parameter(Mandatory = $false, HelpMessage = 'AKS Namespace')] [string]$Namespace = 'fluid-scale-test', [Parameter(Mandatory = $false, HelpMessage = 'Number of nodes to which the node pool should be scaled')] [string]$NodeCount = '0', [Parameter(Mandatory = $false, HelpMessage = 'Name of the resource group')] - [string]$ResourceGroup= 'Fluid.Load.Test', + [string]$ResourceGroup = 'Fluid.Load.Test', [Parameter(Mandatory = $false, HelpMessage = 'Name of the AKS Cluster')] [string]$AKSClusterName = 'LoadTestAks1', @@ -45,14 +44,14 @@ $ScaleNodesJob = Start-Job $_ScaleNodesJob -ArgumentList $NodeCount, ` $NodePoolName Wait-Job $ScaleNodesJob Receive-Job $ScaleNodesJob -Write-Host "Finished scaling down the node pool " -ForegroundColor Green +Write-Host "Finished scaling down the node pool" -ForegroundColor Green -Write-Host "Powering Off Managed Cluster " -ForegroundColor Green +Write-Host "Powering Off Managed Cluster" -ForegroundColor Green az aks stop -n $AKSClusterName -g $ResourceGroup -Write-Host "Powered Off Managed Cluster " -ForegroundColor Green +Write-Host "Powered Off Managed Cluster" -ForegroundColor Green # Sets the StopTime environment variable - which will later be used by the queryLoadTestResults script -$StopTime=(Get-Date -Format "yyyy-MM-ddTHH:mm").ToString() +$StopTime = (Get-Date -Format "yyyy-MM-ddTHH:mm").ToString() [System.Environment]::SetEnvironmentVariable('LoadTestStopTime', ` $StopTime, ` [System.EnvironmentVariableTarget]::User) \ No newline at end of file From b019574eb9a704b62b1577159e16637092dce261 Mon Sep 17 00:00:00 2001 From: Harivallabha Rangarajan Date: Wed, 9 Feb 2022 14:48:09 +0530 Subject: [PATCH 07/10] remove references --- packages/test/test-service-load/mailAuto.ps1 | 4 +-- .../test/test-service-load/mailAutoHelper.ps1 | 2 +- .../queryLoadTestResults.ps1 | 10 +++--- .../test-service-load/queryResultsAuto.ps1 | 2 +- .../registerScheduledTasks.ps1 | 32 +++++++++---------- .../test-service-load/startScaleTestAuto.ps1 | 16 +++++----- .../test-service-load/stopScaleTestAuto.ps1 | 12 +++---- 7 files changed, 38 insertions(+), 40 deletions(-) diff --git a/packages/test/test-service-load/mailAuto.ps1 b/packages/test/test-service-load/mailAuto.ps1 index 58a395499cb5..7826073d0606 100644 --- a/packages/test/test-service-load/mailAuto.ps1 +++ b/packages/test/test-service-load/mailAuto.ps1 @@ -5,8 +5,7 @@ Make sure that the appropriate environment variables are set. #> -Param( - +Param( [Parameter(Mandatory = $false, HelpMessage = 'Mail Address of the Sender')] [string]$MailFrom = "", @@ -21,7 +20,6 @@ Param( [Parameter(Mandatory = $false, HelpMessage = 'Full Path of the results file to be attached')] [string]$AttachmentPath = [System.Environment]::GetEnvironmentVariable('LoadTestResultsFile') - ) function Invoke-SetProperty { diff --git a/packages/test/test-service-load/mailAutoHelper.ps1 b/packages/test/test-service-load/mailAutoHelper.ps1 index 5ac53ef49ed3..657c2ff57679 100644 --- a/packages/test/test-service-load/mailAutoHelper.ps1 +++ b/packages/test/test-service-load/mailAutoHelper.ps1 @@ -4,7 +4,7 @@ as an appropriate environment variable. #> -$PasswordLocation = (Get-Location).ToString() + "\Upass.txt" +$PasswordLocation = "Programmatic Mailer Password Path" # Enter the password, to be stored as a secure string Write-Host "Enter the password for the automatic mailer" -ForegroundColor Cyan read-host -assecurestring | convertfrom-securestring | out-file $PasswordLocation diff --git a/packages/test/test-service-load/queryLoadTestResults.ps1 b/packages/test/test-service-load/queryLoadTestResults.ps1 index 226d22258d7d..38257a9f48e8 100644 --- a/packages/test/test-service-load/queryLoadTestResults.ps1 +++ b/packages/test/test-service-load/queryLoadTestResults.ps1 @@ -60,7 +60,7 @@ function QueryLoadTestResults{ if($KustoQueryName -eq "TestRunSummary") { - # KustoQuery to fetch the TestRunSummary results + # KustoQuery to fetch the TestRunSummary results $KustoQuery = 'let searchTestUid = "{0}";let searchTimeStart = datetime({1});let searchTimeEnd = datetime({2});customEvents| where timestamp between(searchTimeStart .. searchTimeEnd)| extend testUid = tostring(customDimensions.testUid)| where testUid == searchTestUid| extend category = tostring(customDimensions.category), error = tostring(customDimensions.error), runId = toint(customDimensions.runId), eventName = tostring(customDimensions.eventName)| where category == "error" and error !has "deprecated" and error !contains "fluid:telemetry:SummaryStatus Behind" and eventName != "fluid:telemetry:SummaryStatus:Behind" and error !has "MaxListenersExceededWarning" and eventName != "Runner Error"| summarize errCnt = count() by error, eventName| summarize errCount = sum(errCnt), errors = make_bag(pack(iif(isnotempty(error), error, "Unknown"), errCnt)) by eventName| order by errCount' -f $LoadTestGuid,$LoadTestStartTime,$LoadTestStopTime } @@ -68,16 +68,16 @@ function QueryLoadTestResults{ $URLQuery = [uri]::EscapeDataString($KustoQuery) $SaveFileSuffix = "_{0}.txt" -f $KustoQueryName - $SaveResults = (Get-Location).ToString() +"\out\" + ` - (Get-Date -Format "dddd MM_dd_yyyy HH_mm").ToString() + $SaveFileSuffix + $SaveResults = (Get-Location).ToString() +"\out\" + ` + (Get-Date -Format "dddd MM_dd_yyyy HH_mm").ToString() + $SaveFileSuffix # CURL command to fetch the results as a JSON object, and pretty print & pipe the JSON into an output file curl.exe "$SiteURL/v1/apps/$AppID/query?timespan=$Timespan&query=$URLQuery" -H "x-api-key: $APIKey" ` | python -m json.tool | Out-File -FilePath $SaveResults [System.Environment]::SetEnvironmentVariable('LoadTestResultsFile', ` - $SaveResults, ` - [System.EnvironmentVariableTarget]::User) + $SaveResults, ` + [System.EnvironmentVariableTarget]::User) Write-Host "Results of the latest load test, TestGuid: $LoadTestGuid fetched and written to $SaveResults" ` -ForegroundColor Green diff --git a/packages/test/test-service-load/queryResultsAuto.ps1 b/packages/test/test-service-load/queryResultsAuto.ps1 index be8e79bfa549..ff5a984ba683 100644 --- a/packages/test/test-service-load/queryResultsAuto.ps1 +++ b/packages/test/test-service-load/queryResultsAuto.ps1 @@ -7,7 +7,7 @@ Param( [Parameter(Mandatory = $false, HelpMessage = 'Load Test Scripts Directory')] - [string]$LoadTestDir = "C:\loadtest" + [string]$LoadTestDir = "" ) # Make sure this points to the right directory in your local setup (wherever the load test scripts are present) diff --git a/packages/test/test-service-load/registerScheduledTasks.ps1 b/packages/test/test-service-load/registerScheduledTasks.ps1 index 2cb0e49f46cb..bb87671d68b8 100644 --- a/packages/test/test-service-load/registerScheduledTasks.ps1 +++ b/packages/test/test-service-load/registerScheduledTasks.ps1 @@ -14,38 +14,38 @@ Param( [Parameter(Mandatory = $false, HelpMessage = 'Start time for the load test')] - [string]$StartTime = "03:30 am", + [string]$StartTime = "", [Parameter(Mandatory = $false, HelpMessage = 'Location of the start script')] - [string]$StartScriptPath = "C:\loadtest\startScaleTestAuto.ps1", + [string]$StartScriptPath = "", [Parameter(Mandatory = $false, HelpMessage = 'Stop time for the load test')] - [string]$StopTime = "01:30 pm", + [string]$StopTime = "", [Parameter(Mandatory = $false, HelpMessage = 'Location of the stop script')] - [string]$StopScriptPath = "C:\loadtest\stopScaleTestAuto.ps1", + [string]$StopScriptPath = "", [Parameter(Mandatory = $false, HelpMessage = 'Time at which load test results are to be fetched')] - [string]$QueryTime = "02:00 pm", + [string]$QueryTime = "", [Parameter(Mandatory = $false, HelpMessage = 'Location of the fetch results script')] - [string]$QueryScriptPath = "C:\loadtest\queryResultsAuto.ps1", + [string]$QueryScriptPath = "", [Parameter(Mandatory = $false, HelpMessage = 'Time at which load test results are to be mailed')] - [string]$MailTime = "02:05 pm", + [string]$MailTime = "", [Parameter(Mandatory = $false, HelpMessage = 'Location of the programmatic results mailer script')] - [string]$MailScriptPath = "C:\loadtest\mailAuto.ps1", + [string]$MailScriptPath = "", [Parameter(Mandatory = $false, HelpMessage = 'Time at which changes from upstream are to be updated')] - [string]$UpdateTime = "03:00 am", + [string]$UpdateTime = "", [Parameter(Mandatory = $false, HelpMessage = 'Location of the merge-from-upstream script')] - [string]$UpdateScriptPath = "C:\loadtest\mergeUpstreamMainAuto.ps1" + [string]$UpdateScriptPath = "" ) -function registerTask { +function RegisterTask { <# .SYNOPSIS Registers the task with the given name, to be triggered at the specified time. @@ -89,21 +89,21 @@ function registerTask { } # Register the Merge-From-Upstream task at the specified start time -registerTask -TaskName "Merge-From-Upstream" -ScheduledTime $UpdateTime ` +RegisterTask -TaskName "Merge-From-Upstream" -ScheduledTime $UpdateTime ` -Frequency "3" -ScriptPath $UpdateScriptPath # Register the Start-Load-Test task at the specified start time -registerTask -TaskName "Start-Load-Test" -ScheduledTime $StartTime ` +RegisterTask -TaskName "Start-Load-Test" -ScheduledTime $StartTime ` -Frequency "1" -ScriptPath $StartScriptPath # Register the Stop-Load-Test task at the specified stop time -registerTask -TaskName "Stop-Load-Test" -ScheduledTime $StopTime ` +RegisterTask -TaskName "Stop-Load-Test" -ScheduledTime $StopTime ` -Frequency "1" -ScriptPath $StopScriptPath # Register the fetch results task at the specified time -registerTask -TaskName "Fetch-Load-Test-Results" -ScheduledTime $QueryTime ` +RegisterTask -TaskName "Fetch-Load-Test-Results" -ScheduledTime $QueryTime ` -Frequency "1" -ScriptPath $QueryScriptPath # Register the mail results task at the specified time -registerTask -TaskName "Mail-Load-Test-Results" -ScheduledTime $MailTime ` +RegisterTask -TaskName "Mail-Load-Test-Results" -ScheduledTime $MailTime ` -Frequency "1" -ScriptPath $MailScriptPath \ No newline at end of file diff --git a/packages/test/test-service-load/startScaleTestAuto.ps1 b/packages/test/test-service-load/startScaleTestAuto.ps1 index d84049eb9c59..bac9f86767d6 100644 --- a/packages/test/test-service-load/startScaleTestAuto.ps1 +++ b/packages/test/test-service-load/startScaleTestAuto.ps1 @@ -16,36 +16,36 @@ Param( [int]$NumOfDocs = 10, [Parameter(Mandatory = $false, HelpMessage = 'Profile to run test with')] - [string]$Profile = 'scale', + [string]$Profile = '', [Parameter(Mandatory = $false, HelpMessage = 'AKS Namespace')] [ValidateScript({ ( $NumOfDocs -le 10 ) -or ($_ -eq 'fluid-scale-test' ) })] - [string]$Namespace = 'fluid-scale-test', + [string]$Namespace = "", [Parameter(Mandatory = $false, HelpMessage = 'Folder to create in Storage for test files')] [string]$TestDocFolder = [Math]::Floor([decimal](Get-Date(Get-Date).ToUniversalTime() -uformat '%s')), [Parameter(Mandatory = $false, HelpMessage = 'File with tenants and users information')] - [string]$TestTenantConfig = '.\testTenantConfig_vmss.json', + [string]$TestTenantConfig = '', [Parameter(Mandatory = $false, HelpMessage = 'Number of nodes to which the node pool should be scaled')] [string]$NodeCount = '5', [Parameter(Mandatory = $false, HelpMessage = 'Name of the resource group')] - [string]$ResourceGroup = 'Fluid.Load.Test', + [string]$ResourceGroup = '', [Parameter(Mandatory = $false, HelpMessage = 'Name of the AKS Cluster')] - [string]$AKSClusterName = 'LoadTestAks1', + [string]$AKSClusterName = '', [Parameter(Mandatory = $false, HelpMessage = 'Name of the node pool to be scaled')] - [string]$NodePoolName = 'testpods', + [string]$NodePoolName = '', # Restart the load test after a specified time to avoid the Cold Start issue [Parameter(Mandatory = $false, HelpMessage = 'Time (in min) after which the scale test is re-triggered')] [int]$RestartTime = 60, [Parameter(Mandatory = $false, HelpMessage = 'Load Test Scripts Directory')] - [string]$LoadTestDir = "C:\loadtest" + [string]$LoadTestDir = "" ) # Make sure this points to the right directory in your local setup (wherever the load test scripts are present) @@ -66,7 +66,7 @@ Write-Host "Scaling the Node Pool" -ForegroundColor Green $_ScaleNodesJob = { param($nodecount, $resourcegroup, $aksclustername, $nodepoolname) ` az aks scale --resource-group $resourcegroup --name $aksclustername ` --node-count $nodecount --nodepool-name $nodepoolname - } + } $ScaleNodesJob = Start-Job $_ScaleNodesJob -ArgumentList $NodeCount, ` $ResourceGroup, ` $AKSClusterName, ` diff --git a/packages/test/test-service-load/stopScaleTestAuto.ps1 b/packages/test/test-service-load/stopScaleTestAuto.ps1 index 295b6f3d26db..4d1448335c58 100644 --- a/packages/test/test-service-load/stopScaleTestAuto.ps1 +++ b/packages/test/test-service-load/stopScaleTestAuto.ps1 @@ -9,22 +9,22 @@ Param( [Parameter(Mandatory = $false, HelpMessage = 'AKS Namespace')] - [string]$Namespace = 'fluid-scale-test', + [string]$Namespace = '', [Parameter(Mandatory = $false, HelpMessage = 'Number of nodes to which the node pool should be scaled')] [string]$NodeCount = '0', [Parameter(Mandatory = $false, HelpMessage = 'Name of the resource group')] - [string]$ResourceGroup = 'Fluid.Load.Test', + [string]$ResourceGroup = '', [Parameter(Mandatory = $false, HelpMessage = 'Name of the AKS Cluster')] - [string]$AKSClusterName = 'LoadTestAks1', + [string]$AKSClusterName = '', [Parameter(Mandatory = $false, HelpMessage = 'Name of the node pool to be scaled')] - [string]$NodePoolName = 'testpods', + [string]$NodePoolName = '', [Parameter(Mandatory = $false, HelpMessage = 'Load Test Scripts Directory')] - [string]$LoadTestDir = "C:\loadtest" + [string]$LoadTestDir = "" ) # Make sure this points to the right directory in your local setup (wherever the load test scripts are present) @@ -37,7 +37,7 @@ Write-Host "Scaling down the node pool " -ForegroundColor Green $_ScaleNodesJob = { param($nodecount, $resourcegroup, $aksclustername, $nodepoolname) ` az aks scale --resource-group $resourcegroup --name $aksclustername ` --node-count $nodecount --nodepool-name $nodepoolname - } + } $ScaleNodesJob = Start-Job $_ScaleNodesJob -ArgumentList $NodeCount, ` $ResourceGroup, ` $AKSClusterName, ` From 9c2cfa40a47bd45fa2bbea2ea82e9d70ebdf0455 Mon Sep 17 00:00:00 2001 From: Harivallabha Rangarajan Date: Wed, 9 Feb 2022 14:55:03 +0530 Subject: [PATCH 08/10] more lint fixes --- .../test/test-service-load/startScaleTestAuto.ps1 | 10 +++++----- .../test/test-service-load/stopScaleTestAuto.ps1 | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/test/test-service-load/startScaleTestAuto.ps1 b/packages/test/test-service-load/startScaleTestAuto.ps1 index bac9f86767d6..cff741bcaaeb 100644 --- a/packages/test/test-service-load/startScaleTestAuto.ps1 +++ b/packages/test/test-service-load/startScaleTestAuto.ps1 @@ -16,7 +16,7 @@ Param( [int]$NumOfDocs = 10, [Parameter(Mandatory = $false, HelpMessage = 'Profile to run test with')] - [string]$Profile = '', + [string]$Profile = "", [Parameter(Mandatory = $false, HelpMessage = 'AKS Namespace')] [ValidateScript({ ( $NumOfDocs -le 10 ) -or ($_ -eq 'fluid-scale-test' ) })] @@ -26,19 +26,19 @@ Param( [string]$TestDocFolder = [Math]::Floor([decimal](Get-Date(Get-Date).ToUniversalTime() -uformat '%s')), [Parameter(Mandatory = $false, HelpMessage = 'File with tenants and users information')] - [string]$TestTenantConfig = '', + [string]$TestTenantConfig = "", [Parameter(Mandatory = $false, HelpMessage = 'Number of nodes to which the node pool should be scaled')] [string]$NodeCount = '5', [Parameter(Mandatory = $false, HelpMessage = 'Name of the resource group')] - [string]$ResourceGroup = '', + [string]$ResourceGroup = "", [Parameter(Mandatory = $false, HelpMessage = 'Name of the AKS Cluster')] - [string]$AKSClusterName = '', + [string]$AKSClusterName = "", [Parameter(Mandatory = $false, HelpMessage = 'Name of the node pool to be scaled')] - [string]$NodePoolName = '', + [string]$NodePoolName = "", # Restart the load test after a specified time to avoid the Cold Start issue [Parameter(Mandatory = $false, HelpMessage = 'Time (in min) after which the scale test is re-triggered')] diff --git a/packages/test/test-service-load/stopScaleTestAuto.ps1 b/packages/test/test-service-load/stopScaleTestAuto.ps1 index 4d1448335c58..a49a2b64a7c7 100644 --- a/packages/test/test-service-load/stopScaleTestAuto.ps1 +++ b/packages/test/test-service-load/stopScaleTestAuto.ps1 @@ -9,19 +9,19 @@ Param( [Parameter(Mandatory = $false, HelpMessage = 'AKS Namespace')] - [string]$Namespace = '', + [string]$Namespace = "", [Parameter(Mandatory = $false, HelpMessage = 'Number of nodes to which the node pool should be scaled')] [string]$NodeCount = '0', [Parameter(Mandatory = $false, HelpMessage = 'Name of the resource group')] - [string]$ResourceGroup = '', + [string]$ResourceGroup = "", [Parameter(Mandatory = $false, HelpMessage = 'Name of the AKS Cluster')] - [string]$AKSClusterName = '', + [string]$AKSClusterName = "", [Parameter(Mandatory = $false, HelpMessage = 'Name of the node pool to be scaled')] - [string]$NodePoolName = '', + [string]$NodePoolName = "", [Parameter(Mandatory = $false, HelpMessage = 'Load Test Scripts Directory')] [string]$LoadTestDir = "" @@ -30,10 +30,10 @@ Param( # Make sure this points to the right directory in your local setup (wherever the load test scripts are present) cd $LoadTestDir -Write-Host "Deleting namespace " -ForegroundColor Green +Write-Host "Deleting namespace" -ForegroundColor Green kubectl delete ns $Namespace -Write-Host "Scaling down the node pool " -ForegroundColor Green +Write-Host "Scaling down the node pool" -ForegroundColor Green $_ScaleNodesJob = { param($nodecount, $resourcegroup, $aksclustername, $nodepoolname) ` az aks scale --resource-group $resourcegroup --name $aksclustername ` --node-count $nodecount --nodepool-name $nodepoolname From 553012e078cedb56c8a5dd33a0b9220caec2f640 Mon Sep 17 00:00:00 2001 From: Harivallabha Rangarajan Date: Wed, 9 Feb 2022 15:24:44 +0530 Subject: [PATCH 09/10] Place query commands in a json separately --- packages/test/test-service-load/QueryCommands.json | 3 +++ packages/test/test-service-load/queryLoadTestResults.ps1 | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 packages/test/test-service-load/QueryCommands.json diff --git a/packages/test/test-service-load/QueryCommands.json b/packages/test/test-service-load/QueryCommands.json new file mode 100644 index 000000000000..0fc7ff1de3b2 --- /dev/null +++ b/packages/test/test-service-load/QueryCommands.json @@ -0,0 +1,3 @@ +{ + "TestRunSummary":"let searchTestUid = \"{0}\";let searchTimeStart = datetime({1});let searchTimeEnd = datetime({2});customEvents| where timestamp between(searchTimeStart .. searchTimeEnd)| extend testUid = tostring(customDimensions.testUid)| where testUid == searchTestUid| extend category = tostring(customDimensions.category), error = tostring(customDimensions.error), runId = toint(customDimensions.runId), eventName = tostring(customDimensions.eventName)| where category == \"error\" and error !has \"deprecated\" and error !contains \"fluid:telemetry:SummaryStatus Behind\" and eventName != \"fluid:telemetry:SummaryStatus:Behind\" and error !has \"MaxListenersExceededWarning\" and eventName != \"Runner Error\"| summarize errCnt = count() by error, eventName| summarize errCount = sum(errCnt), errors = make_bag(pack(iif(isnotempty(error), error, \"Unknown\"), errCnt)) by eventName| order by errCount" +} \ No newline at end of file diff --git a/packages/test/test-service-load/queryLoadTestResults.ps1 b/packages/test/test-service-load/queryLoadTestResults.ps1 index 38257a9f48e8..3b20c6cbaed6 100644 --- a/packages/test/test-service-load/queryLoadTestResults.ps1 +++ b/packages/test/test-service-load/queryLoadTestResults.ps1 @@ -37,7 +37,10 @@ function QueryLoadTestResults{ [string]$Timespan = "PT12H", [Parameter(Mandatory = $false, HelpMessage = 'Kusto Query to get the results of the Load Test')] - [string]$KustoQueryName = "TestRunSummary" + [string]$KustoQueryName = "TestRunSummary", + + [Parameter(Mandatory = $false, HelpMessage = 'Path to JSON file containing query commands')] + [string]$QueryCommandsPath = ".\QueryCommands.json" ) # AppInsights URL @@ -61,7 +64,8 @@ function QueryLoadTestResults{ if($KustoQueryName -eq "TestRunSummary") { # KustoQuery to fetch the TestRunSummary results - $KustoQuery = 'let searchTestUid = "{0}";let searchTimeStart = datetime({1});let searchTimeEnd = datetime({2});customEvents| where timestamp between(searchTimeStart .. searchTimeEnd)| extend testUid = tostring(customDimensions.testUid)| where testUid == searchTestUid| extend category = tostring(customDimensions.category), error = tostring(customDimensions.error), runId = toint(customDimensions.runId), eventName = tostring(customDimensions.eventName)| where category == "error" and error !has "deprecated" and error !contains "fluid:telemetry:SummaryStatus Behind" and eventName != "fluid:telemetry:SummaryStatus:Behind" and error !has "MaxListenersExceededWarning" and eventName != "Runner Error"| summarize errCnt = count() by error, eventName| summarize errCount = sum(errCnt), errors = make_bag(pack(iif(isnotempty(error), error, "Unknown"), errCnt)) by eventName| order by errCount' -f $LoadTestGuid,$LoadTestStartTime,$LoadTestStopTime + $ReadQuery = Get-Content $QueryCommandsPath | ConvertFrom-Json + $KustoQuery = $ReadQuery.${KustoQueryName} -f $LoadTestGuid,$LoadTestStartTime,$LoadTestStopTime } # Encode Kusto Query into the URI space From 187edb184b047e51922196a79434721ad32dbc38 Mon Sep 17 00:00:00 2001 From: Harivallabha Rangarajan Date: Wed, 9 Feb 2022 17:46:54 +0530 Subject: [PATCH 10/10] Cosmetic fix --- packages/test/test-service-load/mailAutoHelper.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/test/test-service-load/mailAutoHelper.ps1 b/packages/test/test-service-load/mailAutoHelper.ps1 index 657c2ff57679..152773afd1a8 100644 --- a/packages/test/test-service-load/mailAutoHelper.ps1 +++ b/packages/test/test-service-load/mailAutoHelper.ps1 @@ -4,7 +4,7 @@ as an appropriate environment variable. #> -$PasswordLocation = "Programmatic Mailer Password Path" +$PasswordLocation = "" # Enter the password, to be stored as a secure string Write-Host "Enter the password for the automatic mailer" -ForegroundColor Cyan read-host -assecurestring | convertfrom-securestring | out-file $PasswordLocation