feat(ci): Upload telemetry to Kusto in test-dds-stress pipeline (#21388)

[AB#2731](https://dev.azure.com/fluidframework/internal/_workitems/edit/2731)

Move notification mechanisms of `Test - DDS Stress` pipeline to use
telemetry alerting rather than ADO notifications.

The updates of YAML file was verified successful on the test branch:
https://dev.azure.com/fluidframework/internal/_build/results?buildId=270864&view=results

Create a new Kusto-to-Metrics query:
https://portal.microsoftgeneva.com/?page=settings&mode=connectors&tab=kusto&activity=kusto-to-metrics&version=1&account=FluidFrameworkMetrics&namespace=PipelineHealth&ruleId=f676f037-c722-4d15-b42a-024b8a23e9da

Set the monitor to track the consecutive failures of this pipeline:
https://portal.microsoftgeneva.com/manage/monitors/monitor?activity=edit-monitor&version=1&action=1&account=FluidFrameworkMetrics&newManageSessionId=176b98d3-eaf7-4a97-bc0e-e976a76d3fb1

---------

Co-authored-by: Alex Villarreal <716334+alexvy86@users.noreply.github.com>

tools/pipelines/templates/include-conditionally-run-stress-tests.yml

@@ -60,7 +60,8 @@ stages:
             AFFECTED_PATHS: ${{ join(';', package.affectedPaths) }}
 
   - ${{ each package in parameters.packages }}:
-    - stage:
+    # The ids for these stages should be kept in sync with the dependency on them in tools/pipelines/test-dds-stress.yml
+    - stage: ${{ replace(replace(package.name, '@fluidframework/', ''), '@fluid-experimental/', 'experimental_') }}_stress_tests
       dependsOn: CheckAffectedPaths
       displayName: Run ${{ package.name }} stress tests
       jobs:

tools/pipelines/test-dds-stress.yml

@@ -22,38 +22,88 @@ resources:
 variables:
 - name: testWorkspace
   value: $(Pipeline.Workspace)/test
+- name: absolutePathToTelemetryGenerator
+  value: $(Build.SourcesDirectory)/tools/telemetry-generator
+  readonly: true
+
+parameters:
+- name: packages
+  type: object
+  default:
+  - name: "@fluidframework/sequence"
+    affectedPaths:
+    - packages/dds/sequence
+    - packages/dds/merge-tree
+    testFileTarName: sequence
+    testCommand: test:stress
+  - name: "@fluid-experimental/tree"
+    affectedPaths:
+    - experimental/dds/tree
+    testFileTarName: experimental-tree
+    testCommand: test:stress
+  - name: "@fluidframework/tree"
+    affectedPaths:
+    - packages/dds/tree
+    testFileTarName: tree
+    testCommand: test:stress
+  - name: "@fluidframework/map"
+    affectedPaths:
+    - packages/dds/map
+    - packages/dds/merge-tree
+    testFileTarName: map
+    testCommand: test:stress
+  - name: "@fluidframework/matrix"
+    affectedPaths:
+    - packages/dds/matrix
+    - packages/dds/merge-tree
+    testFileTarName: matrix
+    testCommand: test:stress
+
 
 stages:
   - template: templates/include-conditionally-run-stress-tests.yml
     parameters:
       artifactBuildId: $(resources.pipeline.client.runID)
-      packages:
-      - name: "@fluidframework/sequence"
-        affectedPaths:
-        - packages/dds/sequence
-        - packages/dds/merge-tree
-        testFileTarName: sequence
-        testCommand: test:stress
-      - name: "@fluid-experimental/tree"
-        affectedPaths:
-        - experimental/dds/tree
-        testFileTarName: experimental-tree
-        testCommand: test:stress
-      - name: "@fluidframework/tree"
-        affectedPaths:
-        - packages/dds/tree
-        testFileTarName: tree
-        testCommand: test:stress
-      - name: "@fluidframework/map"
-        affectedPaths:
-        - packages/dds/map
-        - packages/dds/merge-tree
-        testFileTarName: map
-        testCommand: test:stress
-      - name: "@fluidframework/matrix"
-        affectedPaths:
-        - packages/dds/matrix
-        - packages/dds/merge-tree
-        testFileTarName: matrix
-        testCommand: test:stress
+      packages: ${{ parameters.packages }}
       testWorkspace: ${{ variables.testWorkspace }}
+
+  # Capture telemetry about pipeline stages
+  - stage: upload_run_telemetry
+    displayName: Upload pipeline run telemetry to Kusto
+    condition: succeededOrFailed()
+    dependsOn:
+    - ${{ each package in parameters.packages }}:
+      # This list of stages should be kept in sync with the way we generate their ids in tools/pipelines/templates/include-conditionally-run-stress-tests.yml
+      - ${{ replace(replace(package.name, '@fluidframework/', ''), '@fluid-experimental/', 'experimental_') }}_stress_tests
+    jobs:
+    - job: upload_run_telemetry
+      displayName: Upload pipeline run telemetry to Kusto
+      pool: Small
+      variables:
+      - group: ado-feeds
+
+      steps:
+      - template: templates/include-telemetry-setup.yml
+        parameters:
+          devFeedUrl: $(ado-feeds-dev)
+          officeFeedUrl: $(ado-feeds-office)
+          isCheckoutNeeded: true
+      - task: Bash@3
+        displayName: Retrieve buildId results
+        inputs:
+          targetType: 'inline'
+          workingDirectory: $(absolutePathToTelemetryGenerator)
+          script: |
+            echo "creating output folder"
+            mkdir -p ${{ variables.testWorkspace }}/timingOutput
+            echo "Retrieving pipeline run timeline data command ..."
+            echo 'curl -u ":<REDACTED>" "https://dev.azure.com/fluidframework/internal/_apis/build/builds/$BUILD_ID/timeline"'
+            curl -u ":$ADO_API_TOKEN" "https://dev.azure.com/fluidframework/internal/_apis/build/builds/$BUILD_ID/timeline\?api-version\=6.0-preview.1" > ${{ variables.testWorkspace }}/timingOutput/output.json
+            pwd;
+            ls -laR ${{ variables.testWorkspace }}/timingOutput/output.json;
+            cat ${{ variables.testWorkspace }}/timingOutput/output.json;
+            node --require @ff-internal/aria-logger bin/run --handlerModule $(absolutePathToTelemetryGenerator)/dist/handlers/stageTimingRetriever.js --dir '${{ variables.testWorkspace }}/timingOutput/';
+        env:
+          BUILD_ID: $(Build.BuildId)
+          ADO_API_TOKEN: $(System.AccessToken)
+          PIPELINE: 'DdsStressService'