Merge pull request microsoft#2 from vikagrawal/automate-load-test

Add scripts for scale test automation

packages/test/test-service-load/QueryCommands.json

@@ -0,0 +1,3 @@
+{
+	"TestRunSummary":"let searchTestUid = \"{0}\";let searchTimeStart = datetime({1});let searchTimeEnd = datetime({2});customEvents| where timestamp between(searchTimeStart .. searchTimeEnd)| extend testUid = tostring(customDimensions.testUid)| where testUid == searchTestUid| extend category = tostring(customDimensions.category), error = tostring(customDimensions.error), runId = toint(customDimensions.runId), eventName = tostring(customDimensions.eventName)| where category == \"error\" and error !has \"deprecated\" and error !contains \"fluid:telemetry:SummaryStatus Behind\" and eventName != \"fluid:telemetry:SummaryStatus:Behind\" and error !has \"MaxListenersExceededWarning\" and eventName != \"Runner Error\"| summarize errCnt = count() by error, eventName| summarize errCount = sum(errCnt), errors = make_bag(pack(iif(isnotempty(error), error, \"Unknown\"), errCnt)) by eventName| order by errCount"
+}

packages/test/test-service-load/mailAuto.ps1

@@ -0,0 +1,123 @@
+<#
+    Script to programmatically mail the results to 
+    the specified users, at the end of a scale test run.
+
+    Make sure that the appropriate environment variables are set.
+#>
+
+Param(  
+    [Parameter(Mandatory = $false, HelpMessage = 'Mail Address of the Sender')]
+    [string]$MailFrom = "<Sender Mail Address>",
+
+    [Parameter(Mandatory = $false, HelpMessage = 'Mail Address of the Recipient')]
+    [string]$MailTo = "<Recipient Mail Address>",
+
+    [Parameter(Mandatory = $false, HelpMessage = 'Mail Address of Additional Recipient')]
+    [string]$MailCC = "<Additional Mail Address>",
+
+    [Parameter(Mandatory = $false, HelpMessage = 'Subject of the Mail')]
+    [string]$MailSubject = "Scale Test Results",
+
+    [Parameter(Mandatory = $false, HelpMessage = 'Full Path of the results file to be attached')]
+    [string]$AttachmentPath = [System.Environment]::GetEnvironmentVariable('LoadTestResultsFile')
+)
+
+function Invoke-SetProperty {
+    # Reflection to set the SendUsingAccount property
+    Param(
+        [__ComObject] $Object,
+        [String] $Property,
+        $Value
+    )
+
+    [Void] $Object.GetType().InvokeMember($Property,"SetProperty",$NULL,$Object,$Value)
+}
+
+function MailUsingOutlookPowershell {
+    <#
+        .SYNOPSIS
+            Creates an Outlook COM Object, and sends a mail
+            with the desired parameters.
+
+            You should ensure that the Outlook client 
+            is setup with the appropriate accounts
+    #>
+    $SendFromSmtpAddress = $MailFrom
+
+    # Create COM object named Outlook
+    $Outlook = New-Object -ComObject Outlook.Application
+    $Account = $Outlook.session.accounts | ? { $_.smtpAddress -eq $SendFromSmtpAddress }
+
+    # Create Outlook MailItem named Mail using CreateItem() method
+    $Mail = $Outlook.CreateItem(0)
+
+    # Add properties as desired
+    $Mail.To = $MailTo
+    $Mail.CC = $MailCC
+    $Mail.Subject = $MailSubject
+    $Mail.Body = $MailBody
+    $Attachment = $AttachmentPath
+    $Mail.Attachments.Add($Attachment)
+
+    # Send message
+    Invoke-SetProperty -Object $Mail -Property "SendUsingAccount" -Value $Account
+    $Mail.Send()
+
+    # Wait for sometime before quitting outlook
+    Start-Sleep -Seconds (1*30)
+
+    # Quit and Cleanup
+    $Outlook.Quit() 
+    [System.Runtime.Interopservices.Marshal]::ReleaseComObject($Outlook) | Out-Null
+}
+
+function MailUsingSendMailMessage {
+    <#
+        .SYNOPSIS
+            Sends a mail using the `Send-MailMessage` cmdlet.
+            Will be deprecated soon, so you should migrate to
+            `MailUsingOutlookPowershell` above.
+
+            You should ensure that the appropriate environment
+            variables are set, and that the credentials are available.
+    #>
+    $PassLoc = [System.Environment]::GetEnvironmentVariable('MailFromPasswordPath')
+    $Username = $MailFrom
+    $Password = Get-Content $PassLoc | ConvertTo-SecureString
+    $Cred = New-Object -typename System.Management.Automation.PSCredential `
+            -argumentlist $Username, $Password
+
+    $MailHash = @{
+        To = $MailTo
+        From = $MailFrom
+        Subject = $MailSubject
+        Body = $MailBody
+        CC = $MailCC
+        BodyAsHtml = $true
+        SmtpServer = 'smtp.office365.com'
+        UseSSL = $true
+        Credential = $Cred
+        Port = 587
+        Attachments = $AttachmentPath
+    }
+
+    Send-MailMessage @MailHash
+}
+
+$TestGuid = [System.Environment]::GetEnvironmentVariable('LoadTestGuid')
+$MailBody = "Results of the Load Test with GUID: {0}" -f $TestGuid
+
+<# 
+    TO-DO: Deprecate this shortly, and use the alternative `MailUsingOutlookPowershell` 
+    for the programmatic mailer after setting up the appropriate account(s) on Outlook.
+#>
+Write-Host "Mailing the results of the load test" -ForegroundColor Green
+MailUsingSendMailMessage 
+#MailUsingOutlookPowershell
+Write-Host "Successfully mailed the results of the load test" -ForegroundColor Green
+
+<#
+    Acknowledgements:
+    [1] https://gist.github.com/ClaudioESSilva/dfaf1de2e5da88fca1e59f70edd7f4ae [ClaudioESSilva] [Mail with Outlook Powershell]
+    [2] https://sid-500.com/2020/08/25/microsoft-365-send-e-mails-with-powershell/ [Patrick Gruenauer] [Mail with Send-MailMessage]
+#>

packages/test/test-service-load/mailAutoHelper.ps1

@@ -0,0 +1,14 @@
+<#
+    Helper script to read in the password for the automatic mailer,
+    and save the path to the password (stored as a secure string),
+    as an appropriate environment variable.
+#>
+
+$PasswordLocation = "<Programmatic Mailer Password Path>"
+# Enter the password, to be stored as a secure string
+Write-Host "Enter the password for the automatic mailer" -ForegroundColor Cyan
+read-host -assecurestring | convertfrom-securestring | out-file $PasswordLocation
+# Store the location of the password in an environment variable
+[System.Environment]::SetEnvironmentVariable('MailFromPasswordPath', `
+                                             $PasswordLocation, `
+                                             [System.EnvironmentVariableTarget]::User)

packages/test/test-service-load/mergeUpstreamMainAuto.ps1

@@ -0,0 +1,32 @@
+<#
+    Script to merge the changes in main branch with the loadtest branch at the specified time, 
+    to generate updated builds.
+
+    Ensure that remote origin and remote upstream point to appropriate repositories.
+
+    Merge conflicts need to be resolved manually.
+#>
+
+Param(
+    [Parameter(Mandatory = $false, HelpMessage = 'Fluid Framework CodeBase Directory')]
+    [string]$LoadTestDir = "C:\FluidFrameworkCodebase\FluidFramework"
+)
+
+# Make sure this points to the right directory in your local setup (wherever the load test scripts are present)
+cd $LoadTestDir
+
+Write-Host "Checkout local main branch" -ForegroundColor "Green"
+git checkout main
+Write-Host "Pull the changes from upstream main" -ForegroundColor "Green"
+git pull upstream main
+Write-Host "Push the changes to origin main" -ForegroundColor "Green"
+git push origin main
+Write-Host "Checkout local loadtest branch" -ForegroundColor "Green"
+git checkout loadtest/main
+
+# Merge conflicts would need to be resolved manually
+Write-Host "Merge the changes made in the main branch, with the loadtest branch" -ForegroundColor "Green"
+git merge main
+Write-Host "Push changes from local loadtest branch to the branch in remote origin" -ForegroundColor "Green"
+git push origin loadtest/main
+Write-Host "Changes updated successfully" -ForegroundColor "Green"

packages/test/test-service-load/queryLoadTestResults.ps1

@@ -0,0 +1,88 @@
+<#
+    Make sure that the appropriate environment variables are set.
+    Uses the python json.tool to pretty-print the results JSON object.
+#>
+
+function QueryLoadTestResults{
+    <#
+        .SYNOPSIS
+            Fetches the results of the load test run.
+        
+        .DESCRIPTION
+            Queries the results of the scale test by calling the REST API exposed by AppInsights.
+            Generates the corresponding CURL command given the KustoQuery, and appropriate parameters.
+            Saves the results as a pretty-printed JSON object, to an output file.
+
+        .PARAMETER AppID
+            Application ID for the AppInsights Client
+        
+        .PARAMETER APIKey
+            APIKey to call the AppInsights REST API
+        
+        .PARAMETER Timespan
+            Timespan to look at, in the AppInsight logs
+        
+        .PARAMETER KustoQueryName
+            Name of the KustoQuery to be executed
+    #>
+    [CmdletBinding()]
+    Param(
+        [Parameter(Mandatory = $false, HelpMessage = 'App ID corresponding to the AppInsight')]
+        [string]$AppID = "",
+
+		[Parameter(Mandatory = $false, HelpMessage = 'API Key with the relevant permission, for the AppInsight')]
+        [string]$APIKey = "",
+
+        [Parameter(Mandatory = $false, HelpMessage = 'Timespan for the query')]
+        [string]$Timespan = "PT12H",
+
+        [Parameter(Mandatory = $false, HelpMessage = 'Kusto Query to get the results of the Load Test')]
+        [string]$KustoQueryName = "TestRunSummary",
+
+        [Parameter(Mandatory = $false, HelpMessage = 'Path to JSON file containing query commands')]
+        [string]$QueryCommandsPath = ".\QueryCommands.json"
+    )
+
+    # AppInsights URL 
+    $SiteURL = "https://api.applicationinsights.io"
+
+    # Get the required environment variables
+    $LoadTestGuid = [System.Environment]::GetEnvironmentVariable('LoadTestGuid')  
+    $LoadTestStartTime = [System.Environment]::GetEnvironmentVariable('LoadTestStartTime')
+    $LoadTestStopTime = [System.Environment]::GetEnvironmentVariable('LoadTestStopTime')
+
+    if ($AppID -eq "")
+    {
+        $AppID = [System.Environment]::GetEnvironmentVariable('AppInsightAppID')
+    }
+
+    if ($APIKey -eq "")
+    {
+        $APIKey = [System.Environment]::GetEnvironmentVariable('AppInsightAPIKey')
+    }
+
+    if($KustoQueryName -eq "TestRunSummary")
+    {
+       # KustoQuery to fetch the TestRunSummary results
+       $ReadQuery = Get-Content $QueryCommandsPath | ConvertFrom-Json
+       $KustoQuery = $ReadQuery.${KustoQueryName} -f $LoadTestGuid,$LoadTestStartTime,$LoadTestStopTime
+    }
+
+    # Encode Kusto Query into the URI space
+    $URLQuery = [uri]::EscapeDataString($KustoQuery)		
+
+    $SaveFileSuffix = "_{0}.txt" -f $KustoQueryName
+    $SaveResults = (Get-Location).ToString() +"\out\" + `
+                   (Get-Date -Format "dddd MM_dd_yyyy HH_mm").ToString() + $SaveFileSuffix
+
+    # CURL command to fetch the results as a JSON object, and pretty print & pipe the JSON into an output file
+    curl.exe "$SiteURL/v1/apps/$AppID/query?timespan=$Timespan&query=$URLQuery" -H "x-api-key: $APIKey" `
+             | python -m json.tool | Out-File -FilePath $SaveResults
+
+    [System.Environment]::SetEnvironmentVariable('LoadTestResultsFile', `
+                                                 $SaveResults, `
+                                                 [System.EnvironmentVariableTarget]::User)
+
+    Write-Host "Results of the latest load test, TestGuid: $LoadTestGuid fetched and written to $SaveResults" `
+                -ForegroundColor Green
+}

packages/test/test-service-load/queryResultsAuto.ps1

@@ -0,0 +1,24 @@
+<#
+    Script to fetch the results at the end of a load test run.
+    Triggered as a scheduled task at the specified time.
+
+    Make sure that the appropriate environment variables are set.
+#>
+
+Param(
+    [Parameter(Mandatory = $false, HelpMessage = 'Load Test Scripts Directory')]
+    [string]$LoadTestDir = "<Load Test Codebase Directory>"
+)
+
+# Make sure this points to the right directory in your local setup (wherever the load test scripts are present)
+cd $LoadTestDir
+
+# Register the QueryLoadTestResults cmdlet function
+. .\queryLoadTestResults.ps1
+
+# Fetch the required environment variables 
+$AppID = [System.Environment]::GetEnvironmentVariable('AppInsightAppID')
+$APIKey = [System.Environment]::GetEnvironmentVariable('AppInsightAPIKey')
+
+Write-Host "Fetching the Latest Results" -ForegroundColor Green
+QueryLoadTestResults -KustoQueryName "TestRunSummary" -AppID $AppID -APIKey $APIKey