Processing of execution-time performance test results

experimental/dds/tree/package.json

@@ -33,6 +33,7 @@
     "prettier": "prettier --check \"**/*.{js,json,jsx,md,ts,tsx,yml,yaml}\"",
     "prettier:fix": "prettier --write \"**/*.{js,json,jsx,md,ts,tsx,yml,yaml}\"",
     "test": "nyc npm run test:mocha",
+    "test:benchmark:report": "mocha \"dist/**/*.tests.js\" --node-option unhandled-rejections=strict,expose-gc --exit --perfMode --fgrep @Benchmark -r @fluidframework/mocha-test-setup --reporter @fluid-tools/benchmark/dist/MochaReporter.js --timeout 60000",
     "test:mocha": "mocha \"dist/**/*.tests.js\" --exit -r node_modules/@fluidframework/mocha-test-setup --unhandled-rejections=strict",
     "test:mocha:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:mocha",
     "test:stress": "cross-env FUZZ_TEST_COUNT=10 FUZZ_STRESS_RUN=true npm run test:mocha",
@@ -56,7 +57,7 @@
   },
   "devDependencies": {
     "@fluid-internal/stochastic-test-utils": ">=2.0.0-internal.1.1.0 <2.0.0-internal.2.0.0",
-    "@fluid-tools/benchmark": "^0.40.0",
+    "@fluid-tools/benchmark": "^0.42.0",
     "@fluidframework/build-common": "^0.24.0",
     "@fluidframework/container-loader": ">=2.0.0-internal.1.1.0 <2.0.0-internal.2.0.0",
     "@fluidframework/container-runtime": ">=2.0.0-internal.1.1.0 <2.0.0-internal.2.0.0",

package.json

@@ -51,6 +51,8 @@
     "stop:docker:full": "docker-compose -f server/docker-compose.yml down && docker volume prune --force",
     "test": "npm run test:mocha && npm run test:jest && npm run test:types && npm run test:realsvc",
     "test:bail": "npm run test:mocha:bail && npm run test:jest:bail",
+    "test:benchmark:copyresults": "copyfiles --exclude \"**/node_modules/**\" \"**/benchmarkOutput/**\" benchmarkOutput",
+    "test:benchmark:report": "lerna run test:benchmark:report --stream --no-bail --no-sort -- -- --reporterOptions reportDir=./benchmarkOutput",
     "test:copyresults": "copyfiles --exclude \"**/node_modules/**\" \"**/nyc/**\" nyc",
     "test:coverage": "c8 npm run test:report",
     "test:fromroot": "mocha \"packages/**/dist/test/**/*.spec.js\" --exit",

packages/dds/tree/package.json

@@ -34,6 +34,7 @@
     "lint": "npm run eslint",
     "lint:fix": "npm run eslint:fix",
     "test": "npm run test:mocha",
+    "test:benchmark:report": "mocha \"dist/**/*.bench.js\" --node-option unhandled-rejections=strict,expose-gc --exit --perfMode --fgrep @Benchmark -r @fluidframework/mocha-test-setup --reporter @fluid-tools/benchmark/dist/MochaReporter.js --timeout 60000",
     "test:coverage": "nyc npm test -- --reporter xunit --reporter-option output=nyc/junit-report.xml",
     "test:mocha": "mocha --recursive dist/test --exit -r node_modules/@fluidframework/mocha-test-setup -r source-map-support/register --unhandled-rejections=strict",
     "test:mocha:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:mocha",
@@ -74,7 +75,7 @@
   },
   "devDependencies": {
     "@fluid-internal/stochastic-test-utils": ">=2.0.0-internal.1.1.0 <2.0.0-internal.2.0.0",
-    "@fluid-tools/benchmark": "^0.40.0",
+    "@fluid-tools/benchmark": "^0.42.0",
     "@fluidframework/build-common": "^0.24.0",
     "@fluidframework/build-tools": "^0.3.1000",
     "@fluidframework/common-definitions": "^0.20.1",

packages/test/stochastic-test-utils/package.json

@@ -62,7 +62,7 @@
     "best-random": "^1.0.0"
   },
   "devDependencies": {
-    "@fluid-tools/benchmark": "^0.40.0",
+    "@fluid-tools/benchmark": "^0.42.0",
     "@fluidframework/build-common": "^0.24.0",
     "@fluidframework/eslint-config-fluid": "^0.28.2000",
     "@fluidframework/mocha-test-setup": ">=2.0.0-internal.1.1.0 <2.0.0-internal.2.0.0",

tools/perf-test-processing/.eslintrc.js

@@ -0,0 +1,17 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+module.exports = {
+    "extends": [
+        require.resolve("@fluidframework/eslint-config-fluid")
+    ],
+    "parserOptions": {
+        project: ["./tsconfig.json"],
+    },
+    "rules": {
+        "@typescript-eslint/ban-ts-comment": "off",
+        "@typescript-eslint/no-non-null-assertion": "off",
+    }
+};

tools/perf-test-processing/LICENSE

@@ -0,0 +1,21 @@
+Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tools/perf-test-processing/README.md

@@ -0,0 +1,20 @@
+# @fluid-tools/perf-test-processing
+
+Internal tool to process the output of our performance benchmark tests and write relevant metrics to our monitoring
+systems.
+
+## Running locally
+
+Install dependencies and build the project:
+
+```bash
+npm i
+npm run build
+```
+
+Run the tool, passing as arguments one or more folders that contain performance test output files generated by
+the `@fluid-tools/benchmark` tool, when called with the appropriate options to generate output JSON files. E.g.:
+
+```bash
+node dist/index.js '/workspaces/FluidFramework/experimental/dds/tree/benchOutput' '/workspaces/FluidFramework/packages/dds/tree/benchOutput'
+```