From 328b756fa36508a5ee561afb495eb335362dc473 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 13:47:39 -0300 Subject: [PATCH 01/57] railway script update --- scripts/railway.sh | 43 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/scripts/railway.sh b/scripts/railway.sh index 1b607ccb..90e13b0c 100755 --- a/scripts/railway.sh +++ b/scripts/railway.sh @@ -1,17 +1,50 @@ #!/bin/bash -# Set your Railway token -export RAILWAY_TOKEN="73286d48-1ca0-469e-b9b1-fca1f5011ae8" +# Load environment variables from .env file +if [ -f .env ]; then + echo "Loading environment variables from .env file" + export $(grep -v '^#' .env | xargs) +else + echo "Error: .env file not found" + exit 1 +fi + +# Check if RAILWAY_PROJECT_TOKEN is set +if [ -z "$RAILWAY_PROJECT_TOKEN" ]; then + echo "Error: RAILWAY_PROJECT_TOKEN is not set in .env file" + exit 1 +fi + +# Set Railway token directly +export RAILWAY_TOKEN="$RAILWAY_PROJECT_TOKEN" +echo "Using Railway project token: ${RAILWAY_PROJECT_TOKEN:0:8}..." # Check CLI version echo "Railway CLI version:" railway --version +# Link to the project, environment, and specific service +echo "Linking to project and service..." +railway link --project "$RAILWAY_PROJECT_ID" --environment "$RAILWAY_ENVIRONMENT_ID" --service "xmtp-qa-testing:us-east" + # Check connection status echo "Checking Railway status:" railway status - # Run your performance test -echo "Running performance test on service:" -railway run -s xmtp-qa-testing:us-east yarn test ts_delivery \ No newline at end of file +echo "Running performance test..." + +# Run with proper environment variables +railway run --env XMTP_ENV=production -- \ + bash -c 'echo "Service details:"; + echo "- Project name: $RAILWAY_PROJECT_NAME"; + echo "- Environment: $RAILWAY_ENVIRONMENT_NAME"; + echo "- Service name: $RAILWAY_SERVICE_NAME"; + + # Extract geolocation from service name + export GEOLOCATION=$(echo $RAILWAY_SERVICE_NAME | cut -d":" -f2); + echo "- Extracted geolocation: $GEOLOCATION"; + echo "- XMTP environment: $XMTP_ENV"; + + # Run the test + yarn test ts_performance' \ No newline at end of file From fdee2d586890b9fd887b6019f22f938f10dae224 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 15:51:35 -0300 Subject: [PATCH 02/57] push summary --- Dockerfile.test | 17 ++ ...6.493Z-Core SDK operations performance.csv | 61 ------ ...ts_performance-south-america-production.md | 55 +++++ .../ts_performance-us-east-production.md | 55 +++++ helpers/client.ts | 2 +- helpers/datadog.ts | 206 ++++++++++++++---- helpers/logger.ts | 1 - scripts/railway.sh | 39 ++-- tests/TS_Performance.test.ts | 90 ++++++++ workers/main.ts | 14 +- 10 files changed, 412 insertions(+), 128 deletions(-) create mode 100644 Dockerfile.test delete mode 100644 datadog/extract-2025-03-14T18_10_16.493Z-Core SDK operations performance.csv create mode 100644 datadog/reports/ts_performance-south-america-production.md create mode 100644 datadog/reports/ts_performance-us-east-production.md diff --git a/Dockerfile.test b/Dockerfile.test new file mode 100644 index 00000000..33e5985b --- /dev/null +++ b/Dockerfile.test @@ -0,0 +1,17 @@ +FROM node:16 + +WORKDIR /app + +# Copy package.json and yarn.lock +COPY package.json yarn.lock ./ +RUN yarn install + +# Copy the rest of the code +COPY . . + +# Set the environment variables +ENV XMTP_ENV=production +ENV GEOLOCATION=us-east + +# Run the test +CMD ["yarn", "test", "ts_performance"] diff --git a/datadog/extract-2025-03-14T18_10_16.493Z-Core SDK operations performance.csv b/datadog/extract-2025-03-14T18_10_16.493Z-Core SDK operations performance.csv deleted file mode 100644 index 737a1f70..00000000 --- a/datadog/extract-2025-03-14T18_10_16.493Z-Core SDK operations performance.csv +++ /dev/null @@ -1,61 +0,0 @@ -"OPERATION","MEMBERS","AVG:XMTP.SDK.DURATION{METRIC_TYPE:OPERATION,ENV:PRODUCTION,GEO:US-EAST,TEST:TS_PERFORMANCE}" -creategroupbyidentifiers,400,7293.44 -creategroup,400,7172.52 -creategroupbyidentifiers,350,6502.96 -creategroup,350,6457.56 -creategroupbyidentifiers,300,5726.98 -creategroup,300,5657.68 -creategroupbyidentifiers,250,4526.802857142858 -creategroup,250,4310.269523809524 -creategroupbyidentifiers,200,3161.3219047619045 -creategroup,200,2967.5857142857144 -creategroupbyidentifiers,150,2203.7285714285713 -creategroup,150,1996.9409523809525 -creategroup,100,1273.5847619047618 -creategroupbyidentifiers,100,1200.4352380952382 -creategroup,50,1081.152380952381 -creategroupbyidentifiers,50,681.1723809523809 -removemembers,400,295.98 -removemembers,250,275.2142857142857 -removemembers,350,260.5 -createdm,,238.44285714285715 -updategroupname,400,234.38 -removemembers,300,230.22 -updategroupname,350,227.6 -updategroupname,300,214.68 -updategroupname,250,208.2952380952381 -removemembers,200,187.73142857142858 -receivegroupmessage,400,187.22 -receivegroupmessage,350,174.44 -removemembers,150,165.25333333333336 -receivegroupmessage,300,162.72 -updategroupname,200,157.90571428571428 -syncgroup,400,157.44 -receivegroupmessage,250,155.24571428571429 -syncgroup,250,151.98761904761903 -updategroupname,150,147.18857142857144 -removemembers,100,146.58285714285714 -syncgroup,350,146.38 -receivegroupmessage,200,145.02 -syncgroup,300,138.02 -updategroupname,100,135.80285714285714 -receivegroupmessage,150,133.98857142857142 -removemembers,50,131.87333333333333 -updategroupname,50,129.19714285714286 -receivegroupmessage,100,125.77333333333334 -syncgroup,200,119.56380952380952 -receivegroupmessage,50,118.44380952380952 -sendgm,,115.38857142857142 -sendgroupmessage,400,97.88 -sendgroupmessage,350,92.86 -sendgroupmessage,300,92.32 -syncgroup,150,89.00952380952381 -sendgroupmessage,200,88.7247619047619 -sendgroupmessage,250,87.93523809523809 -sendgroupmessage,150,84.53523809523809 -receivegm,,82.28 -sendgroupmessage,100,81.56761904761905 -sendgroupmessage,50,81.22095238095238 -syncgroup,100,78.81809523809524 -syncgroup,50,74.64190476190475 -inboxstate,,42.72380952380952 diff --git a/datadog/reports/ts_performance-south-america-production.md b/datadog/reports/ts_performance-south-america-production.md new file mode 100644 index 00000000..cf60dfa1 --- /dev/null +++ b/datadog/reports/ts_performance-south-america-production.md @@ -0,0 +1,55 @@ +# METRICS SUMMARY + +| Operation | Members | Samples | Avg (ms) | Min/Max (ms) | Threshold (ms) | Pass Rate | Status | +| ------------------------------------ | ------- | ------- | -------- | ------------ | -------------- | --------- | ------- | +| clientCreate:operation | - | 1 | 4018 | 4018/4018 | 9000 | 100.0% | PASS ✅ | +| inboxState:operation | - | 1 | 192 | 192/192 | 400 | 100.0% | PASS ✅ | +| clientCreate:network | - | 5 | 344 | 54/805 | 400 | 60.0% | PASS ✅ | +| inboxState:network | - | 5 | 307 | 5/764 | 400 | 60.0% | PASS ✅ | +| createDM:operation | - | 1 | 2064 | 2064/2064 | 3600 | 100.0% | PASS ✅ | +| createDM:network | - | 5 | 328 | 3/818 | 400 | 60.0% | PASS ✅ | +| sendGM:operation | - | 1 | 1022 | 1022/1022 | 2400 | 100.0% | PASS ✅ | +| receiveGM:operation | - | 1 | 805 | 805/805 | 2000 | 100.0% | PASS ✅ | +| sendGM:network | - | 5 | 321 | 9/793 | 400 | 60.0% | PASS ✅ | +| receiveGM:network | - | 5 | 310 | 4/770 | 400 | 60.0% | PASS ✅ | +| createGroup:operation | - | 1 | 2113 | 2113/2113 | 4000 | 100.0% | PASS ✅ | +| createGroup:network | - | 5 | 328 | 5/814 | 400 | 60.0% | PASS ✅ | +| createGroupByIdentifiers:operation | - | 1 | 2315 | 2315/2315 | 4000 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network | - | 5 | 318 | 6/788 | 400 | 60.0% | PASS ✅ | +| syncGroup:operation | - | 1 | 854 | 854/854 | 4000 | 100.0% | PASS ✅ | +| updateGroupName:operation | - | 1 | 794 | 794/794 | 4000 | 100.0% | PASS ✅ | +| syncGroup:network | - | 5 | 345 | 10/852 | 400 | 60.0% | PASS ✅ | +| updateGroupName:network | - | 5 | 322 | 3/803 | 400 | 60.0% | PASS ✅ | +| removeMembers:operation | - | 1 | 853 | 853/853 | 2000 | 100.0% | PASS ✅ | +| removeMembers:network | - | 5 | 319 | 5/792 | 400 | 60.0% | PASS ✅ | +| sendGroupMessage:operation | - | 1 | 861 | 861/861 | 4000 | 100.0% | PASS ✅ | +| sendGroupMessage:network | - | 5 | 321 | 4/798 | 400 | 60.0% | PASS ✅ | +| receiveGroupMessage:operation | - | 1 | 1053 | 1053/1053 | 4000 | 100.0% | PASS ✅ | +| receiveGroupMessage:network | - | 5 | 389 | 3/968 | 400 | 60.0% | PASS ✅ | +| createGroup:operation:4 | 4 | 1 | 1828 | 1828/1828 | 4000 | 100.0% | PASS ✅ | +| createGroup:network:4 | 4 | 5 | 322 | 5/800 | 400 | 60.0% | PASS ✅ | +| createGroupByIdentifiers:operation:4 | 4 | 1 | 2370 | 2370/2370 | 4000 | 100.0% | PASS ✅ | +| syncGroup:operation:4 | 4 | 1 | 847 | 847/847 | 4000 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network:4 | 4 | 5 | 333 | 2/829 | 400 | 60.0% | PASS ✅ | +| syncGroup:network:4 | 4 | 5 | 392 | 3/977 | 400 | 60.0% | PASS ✅ | +| updateGroupName:operation:4 | 4 | 1 | 1089 | 1089/1089 | 4000 | 100.0% | PASS ✅ | +| removeMembers:operation:4 | 4 | 1 | 845 | 845/845 | 2000 | 100.0% | PASS ✅ | +| updateGroupName:network:4 | 4 | 5 | 338 | 2/842 | 400 | 60.0% | PASS ✅ | +| sendGroupMessage:operation:4 | 4 | 1 | 605 | 605/605 | 4000 | 100.0% | PASS ✅ | +| removeMembers:network:4 | 4 | 5 | 425 | 4/1059 | 400 | 60.0% | FAIL ❌ | +| sendGroupMessage:network:4 | 4 | 5 | 335 | 10/826 | 400 | 60.0% | PASS ✅ | +| receiveGroupMessage:operation:4 | 4 | 1 | 1283 | 1283/1283 | 4000 | 100.0% | PASS ✅ | +| receiveGroupMessage:network:4 | 4 | 5 | 317 | 3/789 | 400 | 60.0% | PASS ✅ | +| createGroup:operation:8 | 8 | 1 | 1698 | 1698/1698 | 4000 | 100.0% | PASS ✅ | +| createGroup:network:8 | 8 | 5 | 319 | 5/792 | 400 | 60.0% | PASS ✅ | +| createGroupByIdentifiers:operation:8 | 8 | 1 | 2218 | 2218/2218 | 4000 | 100.0% | PASS ✅ | +| syncGroup:operation:8 | 8 | 1 | 762 | 762/762 | 4000 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network:8 | 8 | 5 | 320 | 3/797 | 400 | 60.0% | PASS ✅ | +| updateGroupName:operation:8 | 8 | 1 | 794 | 794/794 | 4000 | 100.0% | PASS ✅ | +| syncGroup:network:8 | 8 | 5 | 387 | 9/958 | 400 | 60.0% | PASS ✅ | +| updateGroupName:network:8 | 8 | 5 | 315 | 4/783 | 400 | 60.0% | PASS ✅ | +| removeMembers:operation:8 | 8 | 1 | 875 | 875/875 | 2000 | 100.0% | PASS ✅ | +| sendGroupMessage:operation:8 | 8 | 1 | 772 | 772/772 | 4000 | 100.0% | PASS ✅ | +| removeMembers:network:8 | 8 | 5 | 315 | 2/786 | 400 | 60.0% | PASS ✅ | +| sendGroupMessage:network:8 | 8 | 5 | 336 | 3/836 | 400 | 60.0% | PASS ✅ | +| receiveGroupMessage:operation:8 | 8 | 1 | 1064 | 1064/1064 | 4000 | 100.0% | PASS ✅ | diff --git a/datadog/reports/ts_performance-us-east-production.md b/datadog/reports/ts_performance-us-east-production.md new file mode 100644 index 00000000..ec12bfc7 --- /dev/null +++ b/datadog/reports/ts_performance-us-east-production.md @@ -0,0 +1,55 @@ +METRICS SUMMARY +=============== + +Operation | Members | Samples | Avg (ms) | Min/Max (ms) | Threshold (ms) | Pass Rate | Status +----------|---------|---------|----------|--------------|----------------|-----------|------- +clientCreate:operation | - | 1 | 4188 | 4188/4188 | 4500 | 100.0% | PASS ✅ +inboxState:operation | - | 1 | 196 | 196/196 | 200 | 100.0% | PASS ✅ +inboxState:network | - | 5 | 332 | 3/826 | 200 | 60.0% | FAIL ❌ +clientCreate:network | - | 5 | 439 | 53/1044 | 200 | 40.0% | FAIL ❌ +createDM:operation | - | 1 | 2066 | 2066/2066 | 1800 | 0.0% | FAIL ❌ +createDM:network | - | 5 | 319 | 7/791 | 200 | 60.0% | FAIL ❌ +sendGM:operation | - | 1 | 1024 | 1024/1024 | 1200 | 100.0% | PASS ✅ +sendGM:network | - | 5 | 321 | 2/799 | 200 | 60.0% | FAIL ❌ +receiveGM:operation | - | 1 | 835 | 835/835 | 1000 | 100.0% | PASS ✅ +receiveGM:network | - | 5 | 324 | 2/809 | 200 | 60.0% | FAIL ❌ +createGroup:operation | - | 1 | 1878 | 1878/1878 | 2000 | 100.0% | PASS ✅ +createGroup:network | - | 5 | 318 | 3/792 | 200 | 60.0% | FAIL ❌ +createGroupByIdentifiers:operation | - | 1 | 2060 | 2060/2060 | 2000 | 0.0% | FAIL ❌ +syncGroup:operation | - | 1 | 765 | 765/765 | 2000 | 100.0% | PASS ✅ +createGroupByIdentifiers:network | - | 5 | 333 | 7/826 | 200 | 60.0% | FAIL ❌ +syncGroup:network | - | 5 | 320 | 5/795 | 200 | 60.0% | FAIL ❌ +updateGroupName:operation | - | 1 | 1055 | 1055/1055 | 2000 | 100.0% | PASS ✅ +updateGroupName:network | - | 5 | 332 | 2/828 | 200 | 40.0% | FAIL ❌ +removeMembers:operation | - | 1 | 1033 | 1033/1033 | 1000 | 0.0% | FAIL ❌ +sendGroupMessage:operation | - | 1 | 600 | 600/600 | 2000 | 100.0% | PASS ✅ +removeMembers:network | - | 5 | 335 | 3/835 | 200 | 40.0% | FAIL ❌ +sendGroupMessage:network | - | 5 | 336 | 3/837 | 200 | 60.0% | FAIL ❌ +receiveGroupMessage:operation | - | 1 | 978 | 978/978 | 2000 | 100.0% | PASS ✅ +receiveGroupMessage:network | - | 5 | 398 | 6/989 | 200 | 40.0% | FAIL ❌ +createGroup:operation:4 | 4 | 1 | 1539 | 1539/1539 | 2000 | 100.0% | PASS ✅ +createGroup:network:4 | 4 | 5 | 318 | 2/792 | 200 | 60.0% | FAIL ❌ +createGroupByIdentifiers:operation:4 | 4 | 1 | 1809 | 1809/1809 | 2000 | 100.0% | PASS ✅ +syncGroup:operation:4 | 4 | 1 | 718 | 718/718 | 2000 | 100.0% | PASS ✅ +createGroupByIdentifiers:network:4 | 4 | 5 | 328 | 3/816 | 200 | 60.0% | FAIL ❌ +updateGroupName:operation:4 | 4 | 1 | 757 | 757/757 | 2000 | 100.0% | PASS ✅ +syncGroup:network:4 | 4 | 5 | 404 | 5/1005 | 200 | 40.0% | FAIL ❌ +updateGroupName:network:4 | 4 | 5 | 321 | 4/797 | 200 | 60.0% | FAIL ❌ +removeMembers:operation:4 | 4 | 1 | 987 | 987/987 | 1000 | 100.0% | PASS ✅ +sendGroupMessage:operation:4 | 4 | 1 | 610 | 610/610 | 2000 | 100.0% | PASS ✅ +removeMembers:network:4 | 4 | 5 | 333 | 3/829 | 200 | 60.0% | FAIL ❌ +receiveGroupMessage:operation:4 | 4 | 1 | 988 | 988/988 | 2000 | 100.0% | PASS ✅ +sendGroupMessage:network:4 | 4 | 5 | 392 | 5/974 | 200 | 40.0% | FAIL ❌ +receiveGroupMessage:network:4 | 4 | 5 | 324 | 2/807 | 200 | 60.0% | FAIL ❌ +createGroup:operation:8 | 8 | 1 | 1604 | 1604/1604 | 2000 | 100.0% | PASS ✅ +createGroup:network:8 | 8 | 5 | 404 | 5/1006 | 200 | 40.0% | FAIL ❌ +createGroupByIdentifiers:operation:8 | 8 | 1 | 2040 | 2040/2040 | 2000 | 0.0% | FAIL ❌ +syncGroup:operation:8 | 8 | 1 | 552 | 552/552 | 2000 | 100.0% | PASS ✅ +createGroupByIdentifiers:network:8 | 8 | 5 | 314 | 2/782 | 200 | 60.0% | FAIL ❌ +syncGroup:network:8 | 8 | 5 | 323 | 2/805 | 200 | 60.0% | FAIL ❌ +updateGroupName:operation:8 | 8 | 1 | 1040 | 1040/1040 | 2000 | 100.0% | PASS ✅ +updateGroupName:network:8 | 8 | 5 | 319 | 6/791 | 200 | 60.0% | FAIL ❌ +removeMembers:operation:8 | 8 | 1 | 1040 | 1040/1040 | 1000 | 0.0% | FAIL ❌ +sendGroupMessage:operation:8 | 8 | 1 | 554 | 554/554 | 2000 | 100.0% | PASS ✅ +removeMembers:network:8 | 8 | 5 | 315 | 2/785 | 200 | 60.0% | FAIL ❌ +receiveGroupMessage:operation:8 | 8 | 1 | 969 | 969/969 | 2000 | 100.0% | PASS ✅ diff --git a/helpers/client.ts b/helpers/client.ts index 2a7f7f52..0906b3b6 100644 --- a/helpers/client.ts +++ b/helpers/client.ts @@ -150,7 +150,7 @@ export function getAddressOfMember(members: GroupMember[], inboxId: string) { export async function closeEnv(testName: string, workers: WorkerManager) { flushLogger(testName); - await flushMetrics(); + await flushMetrics(testName); if (workers && typeof workers.getWorkers === "function") { for (const worker of workers.getWorkers()) { await worker.worker.terminate(); diff --git a/helpers/datadog.ts b/helpers/datadog.ts index 0a804b8a..f9b13215 100644 --- a/helpers/datadog.ts +++ b/helpers/datadog.ts @@ -1,4 +1,6 @@ import { exec } from "child_process"; +import * as fs from "fs"; +import * as path from "path"; import { promisify } from "util"; import type { WorkerManager } from "@workers/manager"; import metrics from "datadog-metrics"; @@ -6,27 +8,33 @@ import metrics from "datadog-metrics"; // Global state variables let isInitialized = false; let currentGeo: string = ""; +let collectedMetrics: Record< + string, + { values: number[]; threshold: number; members?: string } +> = {}; // Refactored thresholds into a single configuration object const THRESHOLDS = { core: { - clientcreate: 3000, - createdm: 500, - sendgm: 200, - receivegm: 200, - receivegroupmessage: 200, - updategroupname: 200, - syncgroup: 200, + clientcreate: 4500, + createdm: 1800, + sendgm: 1200, + receivegm: 1000, + creategroup: 1500, + creategroupbyidentifiers: 2000, + receivegroupmessage: 1300, + updategroupname: 1000, + syncgroup: 1000, addmembers: 500, - removemembers: 300, - inboxstate: 100, + removemembers: 1000, + inboxstate: 200, }, network: { - dns_lookup: 100, - tcp_connection: 200, - tls_handshake: 300, - processing: 100, - server_call: 400, + dns_lookup: 200, + tcp_connection: 250, + tls_handshake: 350, + processing: 200, + server_call: 500, }, group: { createGroup: { @@ -95,7 +103,7 @@ const THRESHOLDS = { "us-west": 1.0, europe: 1.0, asia: 1.5, - "south-america": 2.6, + "south-america": 2, }, GEO_TO_COUNTRY_CODE: { "us-east": "US", @@ -122,12 +130,26 @@ export function getThresholdForOperation( members: string = "", region: string = "us-east", ): number { + // Convert operation to lowercase for consistent lookups + const operationLower = operation.toLowerCase(); + + // Get the region multiplier + const regionMultiplier = + THRESHOLDS.regionMultipliers[ + region as keyof typeof THRESHOLDS.regionMultipliers + ] || 1.0; + if (operationType === "network") { const networkThreshold = - THRESHOLDS.network[ - operation.toLowerCase() as keyof typeof THRESHOLDS.network - ]; - return typeof networkThreshold === "number" ? networkThreshold : 200; + THRESHOLDS.network[operationLower as keyof typeof THRESHOLDS.network]; + // Default to 200 if the specific network operation isn't found + const baseThreshold = + typeof networkThreshold === "number" ? networkThreshold : 200; + + // Apply region multiplier to network thresholds + const finalThreshold = Math.round(baseThreshold * regionMultiplier); + + return finalThreshold; } if (operationType === "group") { @@ -135,7 +157,7 @@ export function getThresholdForOperation( // Get the operation-specific thresholds object const groupThresholds = - THRESHOLDS.group[operation as keyof typeof THRESHOLDS.group]; + THRESHOLDS.group[operationLower as keyof typeof THRESHOLDS.group]; // Safely check if this size exists, defaulting to 2000 if not let baseThreshold = 2000; @@ -155,18 +177,15 @@ export function getThresholdForOperation( ); } - const coreOp = operation.toLowerCase(); + // For core operations, ensure we're using lowercase for lookup const baseThreshold = - coreOp in THRESHOLDS.core - ? THRESHOLDS.core[coreOp as keyof typeof THRESHOLDS.core] + operationLower in THRESHOLDS.core + ? THRESHOLDS.core[operationLower as keyof typeof THRESHOLDS.core] : 300; - return Math.round( - baseThreshold * - (THRESHOLDS.regionMultipliers[ - region as keyof typeof THRESHOLDS.regionMultipliers - ] || 1.0), - ); + const finalThreshold = Math.round(baseThreshold * regionMultiplier); + + return finalThreshold; } export const sendPerformanceResult = ( @@ -226,7 +245,7 @@ export function initDataDog( } } -// Combined metric sending function to reduce duplication +// Modify the sendMetric function to collect metrics for summary export function sendMetric( metricName: string, metricValue: number, @@ -243,10 +262,26 @@ export function sendMetric( if (!allTags.some((tag) => tag.startsWith("env:"))) { allTags.push(`env:${process.env.XMTP_ENV as string}`); } + // Add version tag if not already present + if (!allTags.some((tag) => tag.startsWith("xv:"))) { + allTags.push(`vm:${process.env.RAILWAY_SERVICE_ID || "unknown"}`); + } - if (allTags.includes("success:false")) { - console.debug(fullMetricName, Math.round(metricValue), allTags); + // Track metrics for summary log + // Include members in the operation key if available + const operationKey = tags.operation + ? `${tags.operation}:${tags.metric_type || ""}${tags.members ? `:${tags.members}` : ""}` + : metricName; + + if (!collectedMetrics[operationKey]) { + collectedMetrics[operationKey] = { + values: [], + threshold: tags.threshold || 0, + members: tags.members || "-", // Store members info separately + }; } + collectedMetrics[operationKey].values.push(metricValue); + metrics.gauge(fullMetricName, Math.round(metricValue), allTags); } catch (error) { console.error( @@ -294,9 +329,10 @@ export async function sendPerformanceMetric( const operationName = operationParts[1]?.split("-")[0] || ""; const members = operationParts[1]?.split("-")[1] || ""; - const operationType = operationName.toLowerCase().includes("group") - ? "group" - : "core"; + // Use a more reliable approach to determine if this is a group operation + const isGroupOperation = operationName.toLowerCase().includes("group"); + const operationType = isGroupOperation ? "group" : "core"; + const threshold = getThresholdForOperation( operationName, operationType, @@ -320,6 +356,7 @@ export async function sendPerformanceMetric( // Network stats handling if (!skipNetworkStats) { + //ignore group operations const networkStats = await getNetworkStats(); const countryCode = getCountryCodeFromGeo(currentGeo); @@ -329,6 +366,8 @@ export async function sendPerformanceMetric( const networkThreshold = getThresholdForOperation( networkPhase, "network", + members, + currentGeo, ); sendMetric("duration", networkMetricValue, { @@ -354,20 +393,107 @@ export async function sendPerformanceMetric( } /** - * Explicitly flush all buffered metrics to DataDog - * Call this at the end of your test suite + * Logs a summary of all collected metrics against their thresholds + * Call this at the end of test execution */ -export function flushMetrics(): Promise { +export function logMetricsSummary(testName: string): void { + if (!isInitialized || Object.keys(collectedMetrics).length === 0) { + console.log("No metrics collected to summarize"); + return; + } + + console.log("\n📊 Creating metrics summary report"); + + // Create a simple text summary for the console + const passedMetrics = Object.entries(collectedMetrics).filter( + ([operation, data]) => { + // Skip workflow metrics when counting passed metrics + if (operation === "workflow") return false; + + if (data.values.length === 0) return false; + const average = + data.values.reduce((sum, val) => sum + val, 0) / data.values.length; + return average <= data.threshold; + }, + ).length; + + // Count only non-workflow metrics for total + const totalMetrics = Object.entries(collectedMetrics).filter( + ([operation]) => operation !== "workflow", + ).length; + + console.log( + `✅ Passed: ${passedMetrics}/${totalMetrics} metrics (${Math.round((passedMetrics / totalMetrics) * 100)}%)`, + ); + + // Create a directory for reports if it doesn't exist + const reportsDir = path.join(process.cwd(), "datadog/reports"); + if (!fs.existsSync(reportsDir)) { + fs.mkdirSync(reportsDir, { recursive: true }); + } + + // Create filename with timestamp + const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); + const filename = path.join( + reportsDir, + `${testName}-${currentGeo}-${process.env.XMTP_ENV}.md`, + ); + + // Build the table content + let fileContent = "METRICS SUMMARY\n===============\n\n"; + fileContent += + "Operation | Members | Samples | Avg (ms) | Min/Max (ms) | Threshold (ms) | Pass Rate | Status\n"; + fileContent += + "----------|---------|---------|----------|--------------|----------------|-----------|-------\n"; + + for (const [operation, data] of Object.entries(collectedMetrics)) { + // Skip workflow metrics in the summary table + if (operation === "workflow") continue; + + if (data.values.length === 0) continue; + + // Extract member count from operationKey + const parts = operation.split(":"); + // The member count should be the last segment if it exists + const memberCount = + parts.length > 2 ? parts[parts.length - 1] : data.members || "-"; + + const average = + data.values.reduce((sum, val) => sum + val, 0) / data.values.length; + const min = Math.min(...data.values); + const max = Math.max(...data.values); + const passRate = + (data.values.filter((v) => v <= data.threshold).length / + data.values.length) * + 100; + const status = average <= data.threshold ? "PASS ✅" : "FAIL ❌"; + fileContent += `${operation} | ${memberCount} | ${data.values.length} | ${Math.round(average)} | ${Math.round(min)}/${Math.round(max)} | ${data.threshold} | ${passRate.toFixed(1)}% | ${status}\n`; + } + + // Write to file + try { + fs.writeFileSync(filename, fileContent); + console.log(`📝 Metrics summary written to: ${filename}`); + } catch (error) { + console.error(`❌ Error writing metrics summary to file:`, error); + } + + // Reset metrics collection for next test run + collectedMetrics = {}; +} + +// Modify flushMetrics to include the summary log +export function flushMetrics(testName: string): Promise { return new Promise((resolve) => { if (!isInitialized) { resolve(); return; } - //console.log("🔄 Flushing DataDog metrics..."); + // Log metrics summary before flushing + logMetricsSummary(testName); void metrics.flush().then(() => { - //console.log("✅ DataDog metrics flushed successfully"); resolve(); }); }); diff --git a/helpers/logger.ts b/helpers/logger.ts index 351097b0..45132582 100644 --- a/helpers/logger.ts +++ b/helpers/logger.ts @@ -145,7 +145,6 @@ function filterLog(args: any[]): string { const getLogFilePath = (testName: string): string => { const env = process.env.XMTP_ENV as string; const logName = testName + "_" + env; - console.log("logName", logName); const sanitizedName = logName.replace(/[^a-zA-Z0-9-_]/g, "_"); const fileName = `${sanitizedName}.log`; diff --git a/scripts/railway.sh b/scripts/railway.sh index 90e13b0c..0d3d4374 100755 --- a/scripts/railway.sh +++ b/scripts/railway.sh @@ -23,28 +23,31 @@ echo "Using Railway project token: ${RAILWAY_PROJECT_TOKEN:0:8}..." echo "Railway CLI version:" railway --version -# Link to the project, environment, and specific service -echo "Linking to project and service..." -railway link --project "$RAILWAY_PROJECT_ID" --environment "$RAILWAY_ENVIRONMENT_ID" --service "xmtp-qa-testing:us-east" - # Check connection status echo "Checking Railway status:" railway status -# Run your performance test -echo "Running performance test..." - -# Run with proper environment variables -railway run --env XMTP_ENV=production -- \ - bash -c 'echo "Service details:"; - echo "- Project name: $RAILWAY_PROJECT_NAME"; - echo "- Environment: $RAILWAY_ENVIRONMENT_NAME"; - echo "- Service name: $RAILWAY_SERVICE_NAME"; +# Important: The key issue is understanding what 'railway run' does +# It executes a command locally but with Railway environment variables +# To truly run on Railway infrastructure, we need to use railway exec + +echo "Running test on Railway remote infrastructure..." +# Use railway exec to run a command on the remote service +railway run --service "xmtp-qa-testing:us-east" -- bash -c ' + echo "Running on Railway remote infrastructure" + echo "Service details:" + echo "- Project name: $RAILWAY_PROJECT_NAME" + echo "- Environment: $RAILWAY_ENVIRONMENT_NAME" + echo "- Service name: $RAILWAY_SERVICE_NAME" + echo "- Service ID: $RAILWAY_SERVICE_ID" + + # Set XMTP_ENV to production manually + export XMTP_ENV=production + export GEOLOCATION=us-east - # Extract geolocation from service name - export GEOLOCATION=$(echo $RAILWAY_SERVICE_NAME | cut -d":" -f2); - echo "- Extracted geolocation: $GEOLOCATION"; - echo "- XMTP environment: $XMTP_ENV"; + echo "- XMTP environment: $XMTP_ENV" + echo "- Geolocation: $GEOLOCATION" # Run the test - yarn test ts_performance' \ No newline at end of file + yarn test ts_performance +' \ No newline at end of file diff --git a/tests/TS_Performance.test.ts b/tests/TS_Performance.test.ts index 1f8f2b06..c064534e 100644 --- a/tests/TS_Performance.test.ts +++ b/tests/TS_Performance.test.ts @@ -147,6 +147,96 @@ describe(testName, () => { throw e; } }); + let i = 4; + let newGroup: Conversation; + it(`createGroup: should create a large group of ${i} participants ${i}`, async () => { + try { + const sliced = generatedInboxes.slice(0, i); + newGroup = await workers + .get("henry")! + .client.conversations.newGroup(sliced.map((inbox) => inbox.inboxId)); + expect(newGroup.id).toBeDefined(); + } catch (e) { + hasFailures = logError(e, expect); + throw e; + } + }); + it(`createGroupByIdentifiers: should create a large group of ${i} participants ${i}`, async () => { + try { + const sliced = generatedInboxes.slice(0, i); + const newGroupByIdentifier = await workers + .get("henry")! + .client.conversations.newGroupWithIdentifiers( + sliced.map((inbox) => ({ + identifier: inbox.accountAddress, + identifierKind: IdentifierKind.Ethereum, + })), + ); + expect(newGroupByIdentifier.id).toBeDefined(); + } catch (e) { + hasFailures = logError(e, expect); + throw e; + } + }); + it(`syncGroup: should sync a large group of ${i} participants ${i}`, async () => { + try { + await newGroup.sync(); + const members = await newGroup.members(); + expect(members.length).toBe(i + 1); + } catch (e) { + hasFailures = logError(e, expect); + throw e; + } + }); + it(`updateGroupName: should update the group name`, async () => { + try { + const newName = "Large Group"; + await (newGroup as Group).updateName(newName); + await newGroup.sync(); + const name = (newGroup as Group).name; + expect(name).toBe(newName); + } catch (e) { + hasFailures = logError(e, expect); + throw e; + } + }); + it(`removeMembers: should remove a participant from a group`, async () => { + try { + const previousMembers = await newGroup.members(); + await (newGroup as Group).removeMembers([ + previousMembers.filter( + (member) => member.inboxId !== (newGroup as Group).addedByInboxId, + )[0].inboxId, + ]); + + const members = await newGroup.members(); + expect(members.length).toBe(previousMembers.length - 1); + } catch (e) { + hasFailures = logError(e, expect); + throw e; + } + }); + it(`sendGroupMessage: should measure sending a gm in a group of ${i} participants`, async () => { + try { + const groupMessage = "gm-" + Math.random().toString(36).substring(2, 15); + + await newGroup.send(groupMessage); + console.log("GM Message sent in group", groupMessage); + expect(groupMessage).toBeDefined(); + } catch (e) { + hasFailures = logError(e, expect); + throw e; + } + }); + it(`receiveGroupMessage: should create a group and measure all streams`, async () => { + try { + const verifyResult = await verifyStreamAll(newGroup, workers); + expect(verifyResult.allReceived).toBe(true); + } catch (e) { + hasFailures = logError(e, expect); + throw e; + } + }); for (let i = batchSize; i <= total; i += batchSize) { let newGroup: Conversation; diff --git a/workers/main.ts b/workers/main.ts index 8cdd4e69..9cbab733 100644 --- a/workers/main.ts +++ b/workers/main.ts @@ -210,13 +210,13 @@ export class WorkerClient extends Worker { const installationId = this.client.installationId; - console.debug({ - inboxId: this.client.inboxId, - dbPath, - version, - address: this.address, - installationId, - }); + // console.debug({ + // inboxId: this.client.inboxId, + // dbPath, + // version, + // address: this.address, + // installationId, + // }); return { client: this.client, dbPath, From 3cb11532e55c720b926a88a6dc3360ff185f0828 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 16:01:14 -0300 Subject: [PATCH 03/57] fix lint --- helpers/logger.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/helpers/logger.ts b/helpers/logger.ts index 45132582..64f2e12a 100644 --- a/helpers/logger.ts +++ b/helpers/logger.ts @@ -189,12 +189,12 @@ export const overrideConsole = (logger: winston.Logger) => { logger.log("error", message); } }; + console.debug = (...args: any[]) => { - // Only show debug logs when not in CI //if (!process.env.CI) { - // Use the original console.debug function + // Using a specific function type for console.debug const originalConsoleDebug = Function.prototype.bind.call( - console.constructor.prototype.debug, + console.constructor.prototype.debug as (...args: any[]) => void, console, ); originalConsoleDebug(...args); From 205328bc99928e8978a7458d2b19a4270dded568 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 16:04:33 -0300 Subject: [PATCH 04/57] updload summary --- .github/workflows/TS_Performance.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/TS_Performance.yml b/.github/workflows/TS_Performance.yml index 16355068..05a4fe2c 100644 --- a/.github/workflows/TS_Performance.yml +++ b/.github/workflows/TS_Performance.yml @@ -37,6 +37,11 @@ jobs: with: name: performance_logs_dev path: logs/ + - name: Upload datadog reports + uses: actions/upload-artifact@v4 + with: + name: datadog_reports_dev + path: datadog/reports/ production: runs-on: ubuntu-latest env: @@ -65,3 +70,8 @@ jobs: with: name: performance_logs_production path: logs/ + - name: Upload datadog reports + uses: actions/upload-artifact@v4 + with: + name: datadog_reports_production + path: datadog/reports/ From b007aa32c3e4ec8469cb0c07026d875dd9197b4c Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 16:14:35 -0300 Subject: [PATCH 05/57] us-east tressholds --- .../ts_performance-us-east-production.md | 192 +++++++++++++----- helpers/datadog.ts | 130 ++++++------ 2 files changed, 203 insertions(+), 119 deletions(-) diff --git a/datadog/reports/ts_performance-us-east-production.md b/datadog/reports/ts_performance-us-east-production.md index ec12bfc7..29f0c927 100644 --- a/datadog/reports/ts_performance-us-east-production.md +++ b/datadog/reports/ts_performance-us-east-production.md @@ -1,55 +1,139 @@ -METRICS SUMMARY -=============== +# METRICS SUMMARY -Operation | Members | Samples | Avg (ms) | Min/Max (ms) | Threshold (ms) | Pass Rate | Status -----------|---------|---------|----------|--------------|----------------|-----------|------- -clientCreate:operation | - | 1 | 4188 | 4188/4188 | 4500 | 100.0% | PASS ✅ -inboxState:operation | - | 1 | 196 | 196/196 | 200 | 100.0% | PASS ✅ -inboxState:network | - | 5 | 332 | 3/826 | 200 | 60.0% | FAIL ❌ -clientCreate:network | - | 5 | 439 | 53/1044 | 200 | 40.0% | FAIL ❌ -createDM:operation | - | 1 | 2066 | 2066/2066 | 1800 | 0.0% | FAIL ❌ -createDM:network | - | 5 | 319 | 7/791 | 200 | 60.0% | FAIL ❌ -sendGM:operation | - | 1 | 1024 | 1024/1024 | 1200 | 100.0% | PASS ✅ -sendGM:network | - | 5 | 321 | 2/799 | 200 | 60.0% | FAIL ❌ -receiveGM:operation | - | 1 | 835 | 835/835 | 1000 | 100.0% | PASS ✅ -receiveGM:network | - | 5 | 324 | 2/809 | 200 | 60.0% | FAIL ❌ -createGroup:operation | - | 1 | 1878 | 1878/1878 | 2000 | 100.0% | PASS ✅ -createGroup:network | - | 5 | 318 | 3/792 | 200 | 60.0% | FAIL ❌ -createGroupByIdentifiers:operation | - | 1 | 2060 | 2060/2060 | 2000 | 0.0% | FAIL ❌ -syncGroup:operation | - | 1 | 765 | 765/765 | 2000 | 100.0% | PASS ✅ -createGroupByIdentifiers:network | - | 5 | 333 | 7/826 | 200 | 60.0% | FAIL ❌ -syncGroup:network | - | 5 | 320 | 5/795 | 200 | 60.0% | FAIL ❌ -updateGroupName:operation | - | 1 | 1055 | 1055/1055 | 2000 | 100.0% | PASS ✅ -updateGroupName:network | - | 5 | 332 | 2/828 | 200 | 40.0% | FAIL ❌ -removeMembers:operation | - | 1 | 1033 | 1033/1033 | 1000 | 0.0% | FAIL ❌ -sendGroupMessage:operation | - | 1 | 600 | 600/600 | 2000 | 100.0% | PASS ✅ -removeMembers:network | - | 5 | 335 | 3/835 | 200 | 40.0% | FAIL ❌ -sendGroupMessage:network | - | 5 | 336 | 3/837 | 200 | 60.0% | FAIL ❌ -receiveGroupMessage:operation | - | 1 | 978 | 978/978 | 2000 | 100.0% | PASS ✅ -receiveGroupMessage:network | - | 5 | 398 | 6/989 | 200 | 40.0% | FAIL ❌ -createGroup:operation:4 | 4 | 1 | 1539 | 1539/1539 | 2000 | 100.0% | PASS ✅ -createGroup:network:4 | 4 | 5 | 318 | 2/792 | 200 | 60.0% | FAIL ❌ -createGroupByIdentifiers:operation:4 | 4 | 1 | 1809 | 1809/1809 | 2000 | 100.0% | PASS ✅ -syncGroup:operation:4 | 4 | 1 | 718 | 718/718 | 2000 | 100.0% | PASS ✅ -createGroupByIdentifiers:network:4 | 4 | 5 | 328 | 3/816 | 200 | 60.0% | FAIL ❌ -updateGroupName:operation:4 | 4 | 1 | 757 | 757/757 | 2000 | 100.0% | PASS ✅ -syncGroup:network:4 | 4 | 5 | 404 | 5/1005 | 200 | 40.0% | FAIL ❌ -updateGroupName:network:4 | 4 | 5 | 321 | 4/797 | 200 | 60.0% | FAIL ❌ -removeMembers:operation:4 | 4 | 1 | 987 | 987/987 | 1000 | 100.0% | PASS ✅ -sendGroupMessage:operation:4 | 4 | 1 | 610 | 610/610 | 2000 | 100.0% | PASS ✅ -removeMembers:network:4 | 4 | 5 | 333 | 3/829 | 200 | 60.0% | FAIL ❌ -receiveGroupMessage:operation:4 | 4 | 1 | 988 | 988/988 | 2000 | 100.0% | PASS ✅ -sendGroupMessage:network:4 | 4 | 5 | 392 | 5/974 | 200 | 40.0% | FAIL ❌ -receiveGroupMessage:network:4 | 4 | 5 | 324 | 2/807 | 200 | 60.0% | FAIL ❌ -createGroup:operation:8 | 8 | 1 | 1604 | 1604/1604 | 2000 | 100.0% | PASS ✅ -createGroup:network:8 | 8 | 5 | 404 | 5/1006 | 200 | 40.0% | FAIL ❌ -createGroupByIdentifiers:operation:8 | 8 | 1 | 2040 | 2040/2040 | 2000 | 0.0% | FAIL ❌ -syncGroup:operation:8 | 8 | 1 | 552 | 552/552 | 2000 | 100.0% | PASS ✅ -createGroupByIdentifiers:network:8 | 8 | 5 | 314 | 2/782 | 200 | 60.0% | FAIL ❌ -syncGroup:network:8 | 8 | 5 | 323 | 2/805 | 200 | 60.0% | FAIL ❌ -updateGroupName:operation:8 | 8 | 1 | 1040 | 1040/1040 | 2000 | 100.0% | PASS ✅ -updateGroupName:network:8 | 8 | 5 | 319 | 6/791 | 200 | 60.0% | FAIL ❌ -removeMembers:operation:8 | 8 | 1 | 1040 | 1040/1040 | 1000 | 0.0% | FAIL ❌ -sendGroupMessage:operation:8 | 8 | 1 | 554 | 554/554 | 2000 | 100.0% | PASS ✅ -removeMembers:network:8 | 8 | 5 | 315 | 2/785 | 200 | 60.0% | FAIL ❌ -receiveGroupMessage:operation:8 | 8 | 1 | 969 | 969/969 | 2000 | 100.0% | PASS ✅ +| Operation | Members | Samples | Avg (ms) | Min/Max (ms) | Threshold (ms) | Pass Rate | Status | +| -------------------------------------- | ------- | ------- | -------- | ------------ | -------------- | --------- | ------- | +| clientCreate:operation | - | 1 | 288 | 288/288 | 4500 | 100.0% | PASS ✅ | +| inboxState:operation | - | 1 | 13 | 13/13 | 200 | 100.0% | PASS ✅ | +| createDM:operation | - | 1 | 116 | 116/116 | 1800 | 100.0% | PASS ✅ | +| sendGM:operation | - | 1 | 44 | 44/44 | 1200 | 100.0% | PASS ✅ | +| receiveGM:operation | - | 1 | 37 | 37/37 | 1000 | 100.0% | PASS ✅ | +| clientCreate:network | - | 5 | 208 | 10/298 | 200 | 20.0% | FAIL ❌ | +| inboxState:network | - | 5 | 194 | 10/280 | 200 | 40.0% | PASS ✅ | +| sendGM:network | - | 5 | 54 | 10/107 | 200 | 100.0% | PASS ✅ | +| createDM:network | - | 5 | 99 | 11/167 | 200 | 100.0% | PASS ✅ | +| receiveGM:network | - | 5 | 61 | 11/102 | 200 | 100.0% | PASS ✅ | +| createGroup:operation | - | 1 | 148 | 148/148 | 2000 | 100.0% | PASS ✅ | +| createGroup:network | - | 5 | 67 | 11/109 | 200 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:operation | - | 1 | 125 | 125/125 | 2000 | 100.0% | PASS ✅ | +| syncGroup:operation | - | 1 | 27 | 27/27 | 2000 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network | - | 5 | 29 | 3/65 | 200 | 100.0% | PASS ✅ | +| updateGroupName:operation | - | 1 | 53 | 53/53 | 2000 | 100.0% | PASS ✅ | +| removeMembers:operation | - | 1 | 53 | 53/53 | 1000 | 100.0% | PASS ✅ | +| syncGroup:network | - | 5 | 67 | 11/109 | 200 | 100.0% | PASS ✅ | +| updateGroupName:network | - | 5 | 27 | 3/58 | 200 | 100.0% | PASS ✅ | +| sendGroupMessage:operation | - | 1 | 33 | 33/33 | 2000 | 100.0% | PASS ✅ | +| receiveGroupMessage:operation | - | 1 | 46 | 46/46 | 2000 | 100.0% | PASS ✅ | +| removeMembers:network | - | 5 | 49 | 11/87 | 200 | 100.0% | PASS ✅ | +| sendGroupMessage:network | - | 5 | 27 | 2/59 | 200 | 100.0% | PASS ✅ | +| receiveGroupMessage:network | - | 5 | 30 | 6/65 | 200 | 100.0% | PASS ✅ | +| createGroup:operation:50 | 50 | 1 | 602 | 602/602 | 2000 | 100.0% | PASS ✅ | +| createGroup:network:50 | 50 | 5 | 45 | 11/81 | 200 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:operation:50 | 50 | 1 | 427 | 427/427 | 2000 | 100.0% | PASS ✅ | +| syncGroup:operation:50 | 50 | 1 | 31 | 31/31 | 2000 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network:50 | 50 | 5 | 29 | 3/64 | 200 | 100.0% | PASS ✅ | +| updateGroupName:operation:50 | 50 | 1 | 58 | 58/58 | 2000 | 100.0% | PASS ✅ | +| syncGroup:network:50 | 50 | 5 | 28 | 3/62 | 200 | 100.0% | PASS ✅ | +| removeMembers:operation:50 | 50 | 1 | 65 | 65/65 | 1000 | 100.0% | PASS ✅ | +| updateGroupName:network:50 | 50 | 5 | 47 | 10/87 | 200 | 100.0% | PASS ✅ | +| sendGroupMessage:operation:50 | 50 | 1 | 34 | 34/34 | 2000 | 100.0% | PASS ✅ | +| removeMembers:network:50 | 50 | 5 | 28 | 3/62 | 200 | 100.0% | PASS ✅ | +| receiveGroupMessage:operation:50 | 50 | 1 | 52 | 52/52 | 2000 | 100.0% | PASS ✅ | +| sendGroupMessage:network:50 | 50 | 5 | 28 | 3/62 | 200 | 100.0% | PASS ✅ | +| receiveGroupMessage:network:50 | 50 | 5 | 28 | 3/63 | 200 | 100.0% | PASS ✅ | +| createGroup:operation:100 | 100 | 1 | 945 | 945/945 | 2000 | 100.0% | PASS ✅ | +| createGroup:network:100 | 100 | 5 | 27 | 2/60 | 200 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:operation:100 | 100 | 1 | 967 | 967/967 | 2000 | 100.0% | PASS ✅ | +| syncGroup:operation:100 | 100 | 1 | 36 | 36/36 | 2000 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network:100 | 100 | 5 | 29 | 2/63 | 200 | 100.0% | PASS ✅ | +| updateGroupName:operation:100 | 100 | 1 | 67 | 67/67 | 2000 | 100.0% | PASS ✅ | +| syncGroup:network:100 | 100 | 5 | 27 | 3/59 | 200 | 100.0% | PASS ✅ | +| updateGroupName:network:100 | 100 | 5 | 31 | 5/68 | 200 | 100.0% | PASS ✅ | +| removeMembers:operation:100 | 100 | 1 | 83 | 83/83 | 1000 | 100.0% | PASS ✅ | +| sendGroupMessage:operation:100 | 100 | 1 | 38 | 38/38 | 2000 | 100.0% | PASS ✅ | +| removeMembers:network:100 | 100 | 5 | 27 | 2/59 | 200 | 100.0% | PASS ✅ | +| receiveGroupMessage:operation:100 | 100 | 1 | 57 | 57/57 | 2000 | 100.0% | PASS ✅ | +| sendGroupMessage:network:100 | 100 | 5 | 29 | 3/63 | 200 | 100.0% | PASS ✅ | +| receiveGroupMessage:network:100 | 100 | 5 | 29 | 3/65 | 200 | 100.0% | PASS ✅ | +| createGroup:operation:150 | 150 | 1 | 1537 | 1537/1537 | 2000 | 100.0% | PASS ✅ | +| createGroup:network:150 | 150 | 5 | 27 | 3/60 | 200 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:operation:150 | 150 | 1 | 1516 | 1516/1516 | 2000 | 100.0% | PASS ✅ | +| syncGroup:operation:150 | 150 | 1 | 40 | 40/40 | 2000 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network:150 | 150 | 5 | 27 | 3/60 | 200 | 100.0% | PASS ✅ | +| syncGroup:network:150 | 150 | 5 | 30 | 7/63 | 200 | 100.0% | PASS ✅ | +| updateGroupName:operation:150 | 150 | 1 | 78 | 78/78 | 2000 | 100.0% | PASS ✅ | +| updateGroupName:network:150 | 150 | 5 | 29 | 3/65 | 200 | 100.0% | PASS ✅ | +| removeMembers:operation:150 | 150 | 1 | 100 | 100/100 | 1000 | 100.0% | PASS ✅ | +| sendGroupMessage:operation:150 | 150 | 1 | 42 | 42/42 | 2000 | 100.0% | PASS ✅ | +| removeMembers:network:150 | 150 | 5 | 27 | 3/60 | 200 | 100.0% | PASS ✅ | +| receiveGroupMessage:operation:150 | 150 | 1 | 65 | 65/65 | 2000 | 100.0% | PASS ✅ | +| sendGroupMessage:network:150 | 150 | 5 | 27 | 3/59 | 200 | 100.0% | PASS ✅ | +| receiveGroupMessage:network:150 | 150 | 5 | 27 | 2/61 | 200 | 100.0% | PASS ✅ | +| createGroup:operation:200 | 200 | 1 | 2340 | 2340/2340 | 2000 | 0.0% | FAIL ❌ | +| createGroup:network:200 | 200 | 5 | 27 | 3/60 | 200 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:operation:200 | 200 | 1 | 2307 | 2307/2307 | 2000 | 0.0% | FAIL ❌ | +| syncGroup:operation:200 | 200 | 1 | 57 | 57/57 | 2000 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network:200 | 200 | 5 | 27 | 3/59 | 200 | 100.0% | PASS ✅ | +| syncGroup:network:200 | 200 | 5 | 29 | 5/63 | 200 | 100.0% | PASS ✅ | +| updateGroupName:operation:200 | 200 | 1 | 86 | 86/86 | 2000 | 100.0% | PASS ✅ | +| updateGroupName:network:200 | 200 | 5 | 29 | 2/64 | 200 | 100.0% | PASS ✅ | +| removeMembers:operation:200 | 200 | 1 | 117 | 117/117 | 1000 | 100.0% | PASS ✅ | +| sendGroupMessage:operation:200 | 200 | 1 | 48 | 48/48 | 2000 | 100.0% | PASS ✅ | +| removeMembers:network:200 | 200 | 5 | 29 | 5/62 | 200 | 100.0% | PASS ✅ | +| sendGroupMessage:network:200 | 200 | 5 | 27 | 3/59 | 200 | 100.0% | PASS ✅ | +| receiveGroupMessage:operation:200 | 200 | 1 | 72 | 72/72 | 2000 | 100.0% | PASS ✅ | +| receiveGroupMessage:network:200 | 200 | 5 | 27 | 2/59 | 200 | 100.0% | PASS ✅ | +| createGroup:operation:250 | 250 | 1 | 3272 | 3272/3272 | 2000 | 0.0% | FAIL ❌ | +| createGroup:network:250 | 250 | 5 | 27 | 3/60 | 200 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:operation:250 | 250 | 1 | 3312 | 3312/3312 | 2000 | 0.0% | FAIL ❌ | +| createGroupByIdentifiers:network:250 | 250 | 5 | 26 | 2/58 | 200 | 100.0% | PASS ✅ | +| syncGroup:operation:250 | 250 | 1 | 68 | 68/68 | 2000 | 100.0% | PASS ✅ | +| syncGroup:network:250 | 250 | 5 | 30 | 5/65 | 200 | 100.0% | PASS ✅ | +| updateGroupName:operation:250 | 250 | 1 | 102 | 102/102 | 2000 | 100.0% | PASS ✅ | +| updateGroupName:network:250 | 250 | 5 | 29 | 3/64 | 200 | 100.0% | PASS ✅ | +| removeMembers:operation:250 | 250 | 1 | 152 | 152/152 | 1000 | 100.0% | PASS ✅ | +| sendGroupMessage:operation:250 | 250 | 1 | 41 | 41/41 | 2000 | 100.0% | PASS ✅ | +| removeMembers:network:250 | 250 | 5 | 26 | 2/59 | 200 | 100.0% | PASS ✅ | +| sendGroupMessage:network:250 | 250 | 5 | 26 | 2/59 | 200 | 100.0% | PASS ✅ | +| receiveGroupMessage:operation:250 | 250 | 1 | 78 | 78/78 | 2000 | 100.0% | PASS ✅ | +| receiveGroupMessage:network:250 | 250 | 5 | 33 | 3/74 | 200 | 100.0% | PASS ✅ | +| createGroup:operation:300 | 300 | 1 | 4218 | 4218/4218 | 2000 | 0.0% | FAIL ❌ | +| createGroup:network:300 | 300 | 5 | 26 | 3/57 | 200 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:operation:300 | 300 | 1 | 5683 | 5683/5683 | 2000 | 0.0% | FAIL ❌ | +| createGroupByIdentifiers:network:300 | 300 | 5 | 28 | 5/61 | 200 | 100.0% | PASS ✅ | +| syncGroup:operation:300 | 300 | 1 | 71 | 71/71 | 2000 | 100.0% | PASS ✅ | +| syncGroup:network:300 | 300 | 5 | 31 | 3/68 | 200 | 100.0% | PASS ✅ | +| updateGroupName:operation:300 | 300 | 1 | 118 | 118/118 | 2000 | 100.0% | PASS ✅ | +| updateGroupName:network:300 | 300 | 5 | 26 | 3/58 | 200 | 100.0% | PASS ✅ | +| removeMembers:operation:300 | 300 | 1 | 161 | 161/161 | 1000 | 100.0% | PASS ✅ | +| sendGroupMessage:operation:300 | 300 | 1 | 48 | 48/48 | 2000 | 100.0% | PASS ✅ | +| removeMembers:network:300 | 300 | 5 | 27 | 3/60 | 200 | 100.0% | PASS ✅ | +| sendGroupMessage:network:300 | 300 | 5 | 28 | 4/60 | 200 | 100.0% | PASS ✅ | +| receiveGroupMessage:operation:300 | 300 | 1 | 95 | 95/95 | 2000 | 100.0% | PASS ✅ | +| receiveGroupMessage:network:300 | 300 | 5 | 27 | 2/61 | 200 | 100.0% | PASS ✅ | +| createGroup:operation:350 | 350 | 1 | 4764 | 4764/4764 | 2000 | 0.0% | FAIL ❌ | +| createGroup:network:350 | 350 | 5 | 28 | 2/62 | 200 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:operation:350 | 350 | 1 | 4778 | 4778/4778 | 2000 | 0.0% | FAIL ❌ | +| createGroupByIdentifiers:network:350 | 350 | 5 | 26 | 3/58 | 200 | 100.0% | PASS ✅ | +| syncGroup:operation:350 | 350 | 1 | 82 | 82/82 | 2000 | 100.0% | PASS ✅ | +| syncGroup:network:350 | 350 | 5 | 35 | 3/80 | 200 | 100.0% | PASS ✅ | +| updateGroupName:operation:350 | 350 | 1 | 134 | 134/134 | 2000 | 100.0% | PASS ✅ | +| updateGroupName:network:350 | 350 | 5 | 26 | 2/57 | 200 | 100.0% | PASS ✅ | +| removeMembers:operation:350 | 350 | 1 | 188 | 188/188 | 1000 | 100.0% | PASS ✅ | +| sendGroupMessage:operation:350 | 350 | 1 | 48 | 48/48 | 2000 | 100.0% | PASS ✅ | +| removeMembers:network:350 | 350 | 5 | 27 | 3/59 | 200 | 100.0% | PASS ✅ | +| sendGroupMessage:network:350 | 350 | 5 | 26 | 2/58 | 200 | 100.0% | PASS ✅ | +| receiveGroupMessage:operation:350 | 350 | 1 | 103 | 103/103 | 2000 | 100.0% | PASS ✅ | +| receiveGroupMessage:network:350 | 350 | 5 | 29 | 3/65 | 200 | 100.0% | PASS ✅ | +| createGroup:operation:400 | 400 | 1 | 5537 | 5537/5537 | 2000 | 0.0% | FAIL ❌ | +| createGroup:network:400 | 400 | 5 | 27 | 3/59 | 200 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:operation:400 | 400 | 1 | 5163 | 5163/5163 | 2000 | 0.0% | FAIL ❌ | +| createGroupByIdentifiers:network:400 | 400 | 5 | 29 | 3/65 | 200 | 100.0% | PASS ✅ | +| syncGroup:operation:400 | 400 | 1 | 96 | 96/96 | 2000 | 100.0% | PASS ✅ | +| syncGroup:network:400 | 400 | 5 | 43 | 11/86 | 200 | 100.0% | PASS ✅ | +| updateGroupName:operation:400 | 400 | 1 | 155 | 155/155 | 2000 | 100.0% | PASS ✅ | +| updateGroupName:network:400 | 400 | 5 | 26 | 3/58 | 200 | 100.0% | PASS ✅ | +| removeMembers:operation:400 | 400 | 1 | 224 | 224/224 | 1000 | 100.0% | PASS ✅ | +| sendGroupMessage:operation:400 | 400 | 1 | 52 | 52/52 | 2000 | 100.0% | PASS ✅ | +| removeMembers:network:400 | 400 | 5 | 29 | 2/66 | 200 | 100.0% | PASS ✅ | +| sendGroupMessage:network:400 | 400 | 5 | 34 | 3/76 | 200 | 100.0% | PASS ✅ | +| receiveGroupMessage:operation:400 | 400 | 1 | 119 | 119/119 | 2000 | 100.0% | PASS ✅ | diff --git a/helpers/datadog.ts b/helpers/datadog.ts index f9b13215..528bef97 100644 --- a/helpers/datadog.ts +++ b/helpers/datadog.ts @@ -16,86 +16,86 @@ let collectedMetrics: Record< // Refactored thresholds into a single configuration object const THRESHOLDS = { core: { - clientcreate: 4500, - createdm: 1800, - sendgm: 1200, - receivegm: 1000, - creategroup: 1500, - creategroupbyidentifiers: 2000, - receivegroupmessage: 1300, - updategroupname: 1000, - syncgroup: 1000, - addmembers: 500, - removemembers: 1000, - inboxstate: 200, + clientcreate: 350, + createdm: 150, + sendgm: 100, + receivegm: 100, + creategroup: 200, + creategroupbyidentifiers: 150, + receivegroupmessage: 150, + updategroupname: 100, + syncgroup: 100, + addmembers: 250, + removemembers: 100, + inboxstate: 50, }, network: { - dns_lookup: 200, - tcp_connection: 250, - tls_handshake: 350, - processing: 200, - server_call: 500, + dns_lookup: 50, + tcp_connection: 50, + tls_handshake: 100, + processing: 50, + server_call: 100, }, group: { createGroup: { - "50": 2000, - "100": 2000, - "150": 4000, - "200": 5000, - "250": 7000, - "300": 9000, - "350": 11000, - "400": 13000, + "50": 650, + "100": 1000, + "150": 1600, + "200": 2500, + "250": 3500, + "300": 4500, + "350": 5000, + "400": 6000, }, createGroupByIdentifiers: { - "50": 2300, - "100": 2500, - "150": 4500, - "200": 5500, - "250": 7500, - "300": 9500, - "350": 11500, - "400": 15000, + "50": 500, + "100": 1100, + "150": 1700, + "200": 2500, + "250": 3500, + "300": 6000, + "350": 5000, + "400": 5500, }, sendGroupMessage: { - "50": 100, - "100": 100, - "150": 100, - "200": 150, - "250": 200, - "300": 300, - "350": 350, - "400": 500, + "50": 50, + "100": 50, + "150": 50, + "200": 70, + "250": 50, + "300": 60, + "350": 60, + "400": 70, }, syncGroup: { - "50": 100, - "100": 100, - "150": 100, - "200": 150, - "250": 200, - "300": 350, - "350": 350, - "400": 500, + "50": 50, + "100": 50, + "150": 50, + "200": 70, + "250": 80, + "300": 80, + "350": 100, + "400": 120, }, updateGroupName: { - "50": 300, - "100": 300, - "150": 300, - "200": 300, - "250": 300, - "300": 300, - "350": 1500, - "400": 2000, + "50": 75, + "100": 80, + "150": 100, + "200": 100, + "250": 120, + "300": 150, + "350": 150, + "400": 200, }, removeMembers: { - "50": 300, - "100": 300, - "150": 300, - "200": 300, - "250": 300, - "300": 300, - "350": 300, - "400": 300, + "50": 80, + "100": 100, + "150": 120, + "200": 150, + "250": 175, + "300": 200, + "350": 225, + "400": 250, }, }, regionMultipliers: { From 50465469e84ffd6f6b0831e6bc1d6b326699687f Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 16:25:01 -0300 Subject: [PATCH 06/57] improved artifacts --- .github/workflows/TS_Performance.yml | 36 ++++-- README.md | 2 +- ...ts_performance-south-america-production.md | 109 +++++++++--------- 3 files changed, 84 insertions(+), 63 deletions(-) diff --git a/.github/workflows/TS_Performance.yml b/.github/workflows/TS_Performance.yml index 05a4fe2c..36cf054f 100644 --- a/.github/workflows/TS_Performance.yml +++ b/.github/workflows/TS_Performance.yml @@ -32,16 +32,26 @@ jobs: run: chmod +x ./scripts/run-test.sh - name: Run tests with retry run: ./scripts/run-test.sh TS_Performance + - name: Rename logs with run ID prefix + run: | + mkdir -p renamed_logs + cp -r logs/* renamed_logs/ + find renamed_logs -type f -exec mv {} {}.run-${{ github.run_id }}-dev \; + - name: Rename datadog reports with run ID prefix + run: | + mkdir -p renamed_reports + cp -r datadog/reports/* renamed_reports/ + find renamed_reports -type f -exec mv {} {}.run-${{ github.run_id }}-dev \; - name: Upload log files uses: actions/upload-artifact@v4 with: - name: performance_logs_dev - path: logs/ + name: performance_logs + path: renamed_logs/ - name: Upload datadog reports uses: actions/upload-artifact@v4 with: - name: datadog_reports_dev - path: datadog/reports/ + name: datadog_reports + path: renamed_reports/ production: runs-on: ubuntu-latest env: @@ -65,13 +75,23 @@ jobs: run: chmod +x ./scripts/run-test.sh - name: Run tests with retry run: ./scripts/run-test.sh TS_Performance + - name: Rename logs with run ID prefix + run: | + mkdir -p renamed_logs + cp -r logs/* renamed_logs/ + find renamed_logs -type f -exec mv {} {}.run-${{ github.run_id }}-production \; + - name: Rename datadog reports with run ID prefix + run: | + mkdir -p renamed_reports + cp -r datadog/reports/* renamed_reports/ + find renamed_reports -type f -exec mv {} {}.run-${{ github.run_id }}-production \; - name: Upload log files uses: actions/upload-artifact@v4 with: - name: performance_logs_production - path: logs/ + name: performance_logs + path: renamed_logs/ - name: Upload datadog reports uses: actions/upload-artifact@v4 with: - name: datadog_reports_production - path: datadog/reports/ + name: datadog_reports + path: renamed_reports/ diff --git a/README.md b/README.md index 3bb8cdaa..c1c2647e 100644 --- a/README.md +++ b/README.md @@ -246,7 +246,7 @@ _Note: Cross-SDK was tested using the `operations` describe above and is not cov - **Repository:** [xmtp-qa-testing](https://github.com/xmtp/xmtp-qa-testing): This monorepo contains multiple tools for testing and monitoring - **Test bot:** Bot for testing with multiple agents - [see section](https://github.com/xmtp/xmtp-qa-testing/tree/main/bots/test/) - **Workflows:** See our CI/CD pipeline configuration - [see section](https://github.com/xmtp/xmtp-qa-testing/tree/main/.github/workflows) -- **Vitest:** We use Vitest for running tests with an interactive UI - [see section](https://xmtp-qa-testing.up.railway.app/__vitest__/#/) +- **Vitest:** We use Vitest for running tests with an interactive UI - [see section](https://xmtp-qa-testingus-east-production.up.railway.app/__vitest__/#/) - **Railway:** Visit our Railway project with all our services - [see section](https://railway.com/project/cc97c743-1be5-4ca3-a41d-0109e41ca1fd) - **Gm bot:** Bot for testing with older version of the protocol - [see section](https://github.com/xmtp/gm-bot) diff --git a/datadog/reports/ts_performance-south-america-production.md b/datadog/reports/ts_performance-south-america-production.md index cf60dfa1..6c105cf0 100644 --- a/datadog/reports/ts_performance-south-america-production.md +++ b/datadog/reports/ts_performance-south-america-production.md @@ -1,55 +1,56 @@ -# METRICS SUMMARY +METRICS SUMMARY +=============== -| Operation | Members | Samples | Avg (ms) | Min/Max (ms) | Threshold (ms) | Pass Rate | Status | -| ------------------------------------ | ------- | ------- | -------- | ------------ | -------------- | --------- | ------- | -| clientCreate:operation | - | 1 | 4018 | 4018/4018 | 9000 | 100.0% | PASS ✅ | -| inboxState:operation | - | 1 | 192 | 192/192 | 400 | 100.0% | PASS ✅ | -| clientCreate:network | - | 5 | 344 | 54/805 | 400 | 60.0% | PASS ✅ | -| inboxState:network | - | 5 | 307 | 5/764 | 400 | 60.0% | PASS ✅ | -| createDM:operation | - | 1 | 2064 | 2064/2064 | 3600 | 100.0% | PASS ✅ | -| createDM:network | - | 5 | 328 | 3/818 | 400 | 60.0% | PASS ✅ | -| sendGM:operation | - | 1 | 1022 | 1022/1022 | 2400 | 100.0% | PASS ✅ | -| receiveGM:operation | - | 1 | 805 | 805/805 | 2000 | 100.0% | PASS ✅ | -| sendGM:network | - | 5 | 321 | 9/793 | 400 | 60.0% | PASS ✅ | -| receiveGM:network | - | 5 | 310 | 4/770 | 400 | 60.0% | PASS ✅ | -| createGroup:operation | - | 1 | 2113 | 2113/2113 | 4000 | 100.0% | PASS ✅ | -| createGroup:network | - | 5 | 328 | 5/814 | 400 | 60.0% | PASS ✅ | -| createGroupByIdentifiers:operation | - | 1 | 2315 | 2315/2315 | 4000 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network | - | 5 | 318 | 6/788 | 400 | 60.0% | PASS ✅ | -| syncGroup:operation | - | 1 | 854 | 854/854 | 4000 | 100.0% | PASS ✅ | -| updateGroupName:operation | - | 1 | 794 | 794/794 | 4000 | 100.0% | PASS ✅ | -| syncGroup:network | - | 5 | 345 | 10/852 | 400 | 60.0% | PASS ✅ | -| updateGroupName:network | - | 5 | 322 | 3/803 | 400 | 60.0% | PASS ✅ | -| removeMembers:operation | - | 1 | 853 | 853/853 | 2000 | 100.0% | PASS ✅ | -| removeMembers:network | - | 5 | 319 | 5/792 | 400 | 60.0% | PASS ✅ | -| sendGroupMessage:operation | - | 1 | 861 | 861/861 | 4000 | 100.0% | PASS ✅ | -| sendGroupMessage:network | - | 5 | 321 | 4/798 | 400 | 60.0% | PASS ✅ | -| receiveGroupMessage:operation | - | 1 | 1053 | 1053/1053 | 4000 | 100.0% | PASS ✅ | -| receiveGroupMessage:network | - | 5 | 389 | 3/968 | 400 | 60.0% | PASS ✅ | -| createGroup:operation:4 | 4 | 1 | 1828 | 1828/1828 | 4000 | 100.0% | PASS ✅ | -| createGroup:network:4 | 4 | 5 | 322 | 5/800 | 400 | 60.0% | PASS ✅ | -| createGroupByIdentifiers:operation:4 | 4 | 1 | 2370 | 2370/2370 | 4000 | 100.0% | PASS ✅ | -| syncGroup:operation:4 | 4 | 1 | 847 | 847/847 | 4000 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network:4 | 4 | 5 | 333 | 2/829 | 400 | 60.0% | PASS ✅ | -| syncGroup:network:4 | 4 | 5 | 392 | 3/977 | 400 | 60.0% | PASS ✅ | -| updateGroupName:operation:4 | 4 | 1 | 1089 | 1089/1089 | 4000 | 100.0% | PASS ✅ | -| removeMembers:operation:4 | 4 | 1 | 845 | 845/845 | 2000 | 100.0% | PASS ✅ | -| updateGroupName:network:4 | 4 | 5 | 338 | 2/842 | 400 | 60.0% | PASS ✅ | -| sendGroupMessage:operation:4 | 4 | 1 | 605 | 605/605 | 4000 | 100.0% | PASS ✅ | -| removeMembers:network:4 | 4 | 5 | 425 | 4/1059 | 400 | 60.0% | FAIL ❌ | -| sendGroupMessage:network:4 | 4 | 5 | 335 | 10/826 | 400 | 60.0% | PASS ✅ | -| receiveGroupMessage:operation:4 | 4 | 1 | 1283 | 1283/1283 | 4000 | 100.0% | PASS ✅ | -| receiveGroupMessage:network:4 | 4 | 5 | 317 | 3/789 | 400 | 60.0% | PASS ✅ | -| createGroup:operation:8 | 8 | 1 | 1698 | 1698/1698 | 4000 | 100.0% | PASS ✅ | -| createGroup:network:8 | 8 | 5 | 319 | 5/792 | 400 | 60.0% | PASS ✅ | -| createGroupByIdentifiers:operation:8 | 8 | 1 | 2218 | 2218/2218 | 4000 | 100.0% | PASS ✅ | -| syncGroup:operation:8 | 8 | 1 | 762 | 762/762 | 4000 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network:8 | 8 | 5 | 320 | 3/797 | 400 | 60.0% | PASS ✅ | -| updateGroupName:operation:8 | 8 | 1 | 794 | 794/794 | 4000 | 100.0% | PASS ✅ | -| syncGroup:network:8 | 8 | 5 | 387 | 9/958 | 400 | 60.0% | PASS ✅ | -| updateGroupName:network:8 | 8 | 5 | 315 | 4/783 | 400 | 60.0% | PASS ✅ | -| removeMembers:operation:8 | 8 | 1 | 875 | 875/875 | 2000 | 100.0% | PASS ✅ | -| sendGroupMessage:operation:8 | 8 | 1 | 772 | 772/772 | 4000 | 100.0% | PASS ✅ | -| removeMembers:network:8 | 8 | 5 | 315 | 2/786 | 400 | 60.0% | PASS ✅ | -| sendGroupMessage:network:8 | 8 | 5 | 336 | 3/836 | 400 | 60.0% | PASS ✅ | -| receiveGroupMessage:operation:8 | 8 | 1 | 1064 | 1064/1064 | 4000 | 100.0% | PASS ✅ | +Operation | Members | Samples | Avg (ms) | Min/Max (ms) | Threshold (ms) | Pass Rate | Status +----------|---------|---------|----------|--------------|----------------|-----------|------- +clientCreate:operation | - | 1 | 4173 | 4173/4173 | 700 | 0.0% | FAIL ❌ +inboxState:operation | - | 1 | 198 | 198/198 | 100 | 0.0% | FAIL ❌ +clientCreate:network | - | 5 | 424 | 51/1008 | 100 | 40.0% | FAIL ❌ +inboxState:network | - | 5 | 326 | 2/812 | 100 | 40.0% | FAIL ❌ +createDM:operation | - | 1 | 2131 | 2131/2131 | 300 | 0.0% | FAIL ❌ +createDM:network | - | 5 | 330 | 2/823 | 100 | 40.0% | FAIL ❌ +sendGM:operation | - | 1 | 1210 | 1210/1210 | 200 | 0.0% | FAIL ❌ +receiveGM:operation | - | 1 | 615 | 615/615 | 200 | 0.0% | FAIL ❌ +sendGM:network | - | 5 | 326 | 3/813 | 100 | 40.0% | FAIL ❌ +receiveGM:network | - | 5 | 319 | 3/795 | 100 | 40.0% | FAIL ❌ +createGroup:operation | - | 1 | 1873 | 1873/1873 | 4000 | 100.0% | PASS ✅ +createGroup:network | - | 5 | 378 | 2/944 | 100 | 40.0% | FAIL ❌ +createGroupByIdentifiers:operation | - | 1 | 1911 | 1911/1911 | 4000 | 100.0% | PASS ✅ +syncGroup:operation | - | 1 | 729 | 729/729 | 4000 | 100.0% | PASS ✅ +createGroupByIdentifiers:network | - | 5 | 319 | 11/785 | 100 | 40.0% | FAIL ❌ +syncGroup:network | - | 5 | 299 | 3/745 | 100 | 40.0% | FAIL ❌ +updateGroupName:operation | - | 1 | 1020 | 1020/1020 | 4000 | 100.0% | PASS ✅ +updateGroupName:network | - | 5 | 386 | 4/962 | 100 | 40.0% | FAIL ❌ +removeMembers:operation | - | 1 | 1035 | 1035/1035 | 200 | 0.0% | FAIL ❌ +sendGroupMessage:operation | - | 1 | 621 | 621/621 | 4000 | 100.0% | PASS ✅ +removeMembers:network | - | 5 | 323 | 6/802 | 100 | 40.0% | FAIL ❌ +sendGroupMessage:network | - | 5 | 391 | 2/975 | 100 | 40.0% | FAIL ❌ +receiveGroupMessage:operation | - | 1 | 1213 | 1213/1213 | 4000 | 100.0% | PASS ✅ +receiveGroupMessage:network | - | 5 | 382 | 2/953 | 100 | 40.0% | FAIL ❌ +createGroup:operation:4 | 4 | 1 | 1879 | 1879/1879 | 4000 | 100.0% | PASS ✅ +createGroup:network:4 | 4 | 5 | 384 | 4/955 | 100 | 40.0% | FAIL ❌ +createGroupByIdentifiers:operation:4 | 4 | 1 | 2052 | 2052/2052 | 4000 | 100.0% | PASS ✅ +createGroupByIdentifiers:network:4 | 4 | 5 | 316 | 2/789 | 100 | 40.0% | FAIL ❌ +syncGroup:operation:4 | 4 | 1 | 811 | 811/811 | 4000 | 100.0% | PASS ✅ +syncGroup:network:4 | 4 | 5 | 306 | 4/760 | 100 | 40.0% | FAIL ❌ +updateGroupName:operation:4 | 4 | 1 | 1041 | 1041/1041 | 4000 | 100.0% | PASS ✅ +updateGroupName:network:4 | 4 | 5 | 312 | 2/779 | 100 | 40.0% | FAIL ❌ +removeMembers:operation:4 | 4 | 1 | 822 | 822/822 | 200 | 0.0% | FAIL ❌ +sendGroupMessage:operation:4 | 4 | 1 | 795 | 795/795 | 4000 | 100.0% | PASS ✅ +removeMembers:network:4 | 4 | 5 | 378 | 2/943 | 100 | 40.0% | FAIL ❌ +sendGroupMessage:network:4 | 4 | 5 | 310 | 3/773 | 100 | 40.0% | FAIL ❌ +receiveGroupMessage:operation:4 | 4 | 1 | 1211 | 1211/1211 | 4000 | 100.0% | PASS ✅ +receiveGroupMessage:network:4 | 4 | 5 | 307 | 2/766 | 100 | 40.0% | FAIL ❌ +createGroup:operation:8 | 8 | 1 | 1846 | 1846/1846 | 4000 | 100.0% | PASS ✅ +createGroup:network:8 | 8 | 5 | 307 | 6/761 | 100 | 40.0% | FAIL ❌ +createGroupByIdentifiers:operation:8 | 8 | 1 | 1903 | 1903/1903 | 4000 | 100.0% | PASS ✅ +syncGroup:operation:8 | 8 | 1 | 736 | 736/736 | 4000 | 100.0% | PASS ✅ +createGroupByIdentifiers:network:8 | 8 | 5 | 318 | 2/794 | 100 | 40.0% | FAIL ❌ +syncGroup:network:8 | 8 | 5 | 301 | 2/750 | 100 | 40.0% | FAIL ❌ +updateGroupName:operation:8 | 8 | 1 | 1056 | 1056/1056 | 4000 | 100.0% | PASS ✅ +updateGroupName:network:8 | 8 | 5 | 381 | 2/951 | 100 | 40.0% | FAIL ❌ +removeMembers:operation:8 | 8 | 1 | 1047 | 1047/1047 | 200 | 0.0% | FAIL ❌ +removeMembers:network:8 | 8 | 5 | 305 | 2/761 | 100 | 40.0% | FAIL ❌ +sendGroupMessage:operation:8 | 8 | 1 | 824 | 824/824 | 4000 | 100.0% | PASS ✅ +sendGroupMessage:network:8 | 8 | 5 | 303 | 2/757 | 100 | 40.0% | FAIL ❌ +receiveGroupMessage:operation:8 | 8 | 1 | 1038 | 1038/1038 | 4000 | 100.0% | PASS ✅ From 12e0ae53757289bdb520feda5010c6b1e8e7c5c5 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 16:29:16 -0300 Subject: [PATCH 07/57] fix log --- .github/workflows/TS_Performance.yml | 8 ++++---- helpers/datadog.ts | 13 ------------- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/.github/workflows/TS_Performance.yml b/.github/workflows/TS_Performance.yml index 36cf054f..b4ab8f6d 100644 --- a/.github/workflows/TS_Performance.yml +++ b/.github/workflows/TS_Performance.yml @@ -45,12 +45,12 @@ jobs: - name: Upload log files uses: actions/upload-artifact@v4 with: - name: performance_logs + name: performance_logs_${{ env.XMTP_ENV }} path: renamed_logs/ - name: Upload datadog reports uses: actions/upload-artifact@v4 with: - name: datadog_reports + name: datadog_reports_${{ env.XMTP_ENV }} path: renamed_reports/ production: runs-on: ubuntu-latest @@ -88,10 +88,10 @@ jobs: - name: Upload log files uses: actions/upload-artifact@v4 with: - name: performance_logs + name: performance_logs_${{ env.XMTP_ENV }} path: renamed_logs/ - name: Upload datadog reports uses: actions/upload-artifact@v4 with: - name: datadog_reports + name: datadog_reports_${{ env.XMTP_ENV }} path: renamed_reports/ diff --git a/helpers/datadog.ts b/helpers/datadog.ts index 528bef97..f5229304 100644 --- a/helpers/datadog.ts +++ b/helpers/datadog.ts @@ -560,19 +560,6 @@ export async function getNetworkStats( stats["Processing"] = 0; } - if ( - stats["Processing"] * 1000 > 300 || - stats["TLS Handshake"] * 1000 > 300 || - stats["Server Call"] * 1000 > 300 - ) { - if (!firstLogShared) { - firstLogShared = true; - console.warn( - `Slow connection detected - total: ${stats["Server Call"] * 1000}ms, TLS: ${stats["TLS Handshake"] * 1000}ms, processing: ${stats["Processing"] * 1000}ms`, - ); - } - } - return stats; } From 8a46c8f24174d8357b354e9ef215190fd48754c2 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 16:30:20 -0300 Subject: [PATCH 08/57] dont log time --- helpers/logger.ts | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/helpers/logger.ts b/helpers/logger.ts index 64f2e12a..92dc499c 100644 --- a/helpers/logger.ts +++ b/helpers/logger.ts @@ -103,21 +103,23 @@ function filterLog(args: any[]): string { // Check for the console.time/timeEnd pattern: where args[0] is "%s: %s" if (args.length >= 2 && args[0] === "%s: %s") { - // Join the remaining parts into one message - const message = args.slice(1).join(" "); - const timePattern = /(\d+(\.\d+)?)(ms|s)\b/; - const match = message.match(timePattern); - if (match) { - const timeValue = parseFloat(match[1]); - const unit = match[3]; - const timeInMs = unit === "ms" ? timeValue : timeValue * 1000; - // Skip logs for durations less than or equal to 300ms - if (timeInMs <= 300) { - return ""; - } - } - // Remove any "%s" placeholders from the message. - return message.replace(/%s/g, "").trim(); + //dont log time + return ""; + // // Join the remaining parts into one message + // const message = args.slice(1).join(" "); + // const timePattern = /(\d+(\.\d+)?)(ms|s)\b/; + // const match = message.match(timePattern); + // if (match) { + // const timeValue = parseFloat(match[1]); + // const unit = match[3]; + // const timeInMs = unit === "ms" ? timeValue : timeValue * 1000; + // // Skip logs for durations less than or equal to 300ms + // if (timeInMs <= 300) { + // return ""; + // } + // } + // // Remove any "%s" placeholders from the message. + // return message.replace(/%s/g, "").trim(); } return ( From 5c40c9b7d3ea6067c5fc3b9ed245e2dba1ab6303 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 16:31:09 -0300 Subject: [PATCH 09/57] dont log time --- helpers/logger.ts | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/helpers/logger.ts b/helpers/logger.ts index 92dc499c..00e27ea6 100644 --- a/helpers/logger.ts +++ b/helpers/logger.ts @@ -102,24 +102,25 @@ function filterLog(args: any[]): string { } // Check for the console.time/timeEnd pattern: where args[0] is "%s: %s" - if (args.length >= 2 && args[0] === "%s: %s") { - //dont log time - return ""; - // // Join the remaining parts into one message - // const message = args.slice(1).join(" "); - // const timePattern = /(\d+(\.\d+)?)(ms|s)\b/; - // const match = message.match(timePattern); - // if (match) { - // const timeValue = parseFloat(match[1]); - // const unit = match[3]; - // const timeInMs = unit === "ms" ? timeValue : timeValue * 1000; - // // Skip logs for durations less than or equal to 300ms - // if (timeInMs <= 300) { - // return ""; - // } - // } - // // Remove any "%s" placeholders from the message. - // return message.replace(/%s/g, "").trim(); + + if (!process.env.CI) { + if (args.length >= 2 && args[0] === "%s: %s") { + // Join the remaining parts into one message + const message = args.slice(1).join(" "); + const timePattern = /(\d+(\.\d+)?)(ms|s)\b/; + const match = message.match(timePattern); + if (match) { + const timeValue = parseFloat(match[1]); + const unit = match[3]; + const timeInMs = unit === "ms" ? timeValue : timeValue * 1000; + // Skip logs for durations less than or equal to 300ms + if (timeInMs <= 300) { + return ""; + } + } + // Remove any "%s" placeholders from the message. + return message.replace(/%s/g, "").trim(); + } } return ( From f05ea721e356aa2246b587a113a37965743c5c03 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 16:33:13 -0300 Subject: [PATCH 10/57] dont log time --- helpers/logger.ts | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/helpers/logger.ts b/helpers/logger.ts index 00e27ea6..58d492da 100644 --- a/helpers/logger.ts +++ b/helpers/logger.ts @@ -121,6 +121,10 @@ function filterLog(args: any[]): string { // Remove any "%s" placeholders from the message. return message.replace(/%s/g, "").trim(); } + } else { + if (args.length >= 2 && args[0] === "%s: %s") { + return ""; + } } return ( @@ -194,14 +198,14 @@ export const overrideConsole = (logger: winston.Logger) => { }; console.debug = (...args: any[]) => { - //if (!process.env.CI) { - // Using a specific function type for console.debug - const originalConsoleDebug = Function.prototype.bind.call( - console.constructor.prototype.debug as (...args: any[]) => void, - console, - ); - originalConsoleDebug(...args); - //} + if (!process.env.CI) { + // Using a specific function type for console.debug + const originalConsoleDebug = Function.prototype.bind.call( + console.constructor.prototype.debug as (...args: any[]) => void, + console, + ); + originalConsoleDebug(...args); + } }; } catch (error) { console.error("Error overriding console", error); From e9e8eb9567493d40648330bbcd2c6128a34e06d4 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 16:38:23 -0300 Subject: [PATCH 11/57] new tessolds --- ...ts_performance-south-america-production.md | 56 ------- ...roduction.md.run-13931915774-production.md | 55 +++++++ .../ts_performance-us-east-production.md | 139 ------------------ ...roduction.md.run-13931915774-production.md | 139 ++++++++++++++++++ helpers/datadog.ts | 52 +++++-- 5 files changed, 230 insertions(+), 211 deletions(-) delete mode 100644 datadog/reports/ts_performance-south-america-production.md create mode 100644 datadog/reports/ts_performance-south-america-production.md.run-13931915774-production.md delete mode 100644 datadog/reports/ts_performance-us-east-production.md create mode 100644 datadog/reports/ts_performance-us-east-production.md.run-13931915774-production.md diff --git a/datadog/reports/ts_performance-south-america-production.md b/datadog/reports/ts_performance-south-america-production.md deleted file mode 100644 index 6c105cf0..00000000 --- a/datadog/reports/ts_performance-south-america-production.md +++ /dev/null @@ -1,56 +0,0 @@ -METRICS SUMMARY -=============== - -Operation | Members | Samples | Avg (ms) | Min/Max (ms) | Threshold (ms) | Pass Rate | Status -----------|---------|---------|----------|--------------|----------------|-----------|------- -clientCreate:operation | - | 1 | 4173 | 4173/4173 | 700 | 0.0% | FAIL ❌ -inboxState:operation | - | 1 | 198 | 198/198 | 100 | 0.0% | FAIL ❌ -clientCreate:network | - | 5 | 424 | 51/1008 | 100 | 40.0% | FAIL ❌ -inboxState:network | - | 5 | 326 | 2/812 | 100 | 40.0% | FAIL ❌ -createDM:operation | - | 1 | 2131 | 2131/2131 | 300 | 0.0% | FAIL ❌ -createDM:network | - | 5 | 330 | 2/823 | 100 | 40.0% | FAIL ❌ -sendGM:operation | - | 1 | 1210 | 1210/1210 | 200 | 0.0% | FAIL ❌ -receiveGM:operation | - | 1 | 615 | 615/615 | 200 | 0.0% | FAIL ❌ -sendGM:network | - | 5 | 326 | 3/813 | 100 | 40.0% | FAIL ❌ -receiveGM:network | - | 5 | 319 | 3/795 | 100 | 40.0% | FAIL ❌ -createGroup:operation | - | 1 | 1873 | 1873/1873 | 4000 | 100.0% | PASS ✅ -createGroup:network | - | 5 | 378 | 2/944 | 100 | 40.0% | FAIL ❌ -createGroupByIdentifiers:operation | - | 1 | 1911 | 1911/1911 | 4000 | 100.0% | PASS ✅ -syncGroup:operation | - | 1 | 729 | 729/729 | 4000 | 100.0% | PASS ✅ -createGroupByIdentifiers:network | - | 5 | 319 | 11/785 | 100 | 40.0% | FAIL ❌ -syncGroup:network | - | 5 | 299 | 3/745 | 100 | 40.0% | FAIL ❌ -updateGroupName:operation | - | 1 | 1020 | 1020/1020 | 4000 | 100.0% | PASS ✅ -updateGroupName:network | - | 5 | 386 | 4/962 | 100 | 40.0% | FAIL ❌ -removeMembers:operation | - | 1 | 1035 | 1035/1035 | 200 | 0.0% | FAIL ❌ -sendGroupMessage:operation | - | 1 | 621 | 621/621 | 4000 | 100.0% | PASS ✅ -removeMembers:network | - | 5 | 323 | 6/802 | 100 | 40.0% | FAIL ❌ -sendGroupMessage:network | - | 5 | 391 | 2/975 | 100 | 40.0% | FAIL ❌ -receiveGroupMessage:operation | - | 1 | 1213 | 1213/1213 | 4000 | 100.0% | PASS ✅ -receiveGroupMessage:network | - | 5 | 382 | 2/953 | 100 | 40.0% | FAIL ❌ -createGroup:operation:4 | 4 | 1 | 1879 | 1879/1879 | 4000 | 100.0% | PASS ✅ -createGroup:network:4 | 4 | 5 | 384 | 4/955 | 100 | 40.0% | FAIL ❌ -createGroupByIdentifiers:operation:4 | 4 | 1 | 2052 | 2052/2052 | 4000 | 100.0% | PASS ✅ -createGroupByIdentifiers:network:4 | 4 | 5 | 316 | 2/789 | 100 | 40.0% | FAIL ❌ -syncGroup:operation:4 | 4 | 1 | 811 | 811/811 | 4000 | 100.0% | PASS ✅ -syncGroup:network:4 | 4 | 5 | 306 | 4/760 | 100 | 40.0% | FAIL ❌ -updateGroupName:operation:4 | 4 | 1 | 1041 | 1041/1041 | 4000 | 100.0% | PASS ✅ -updateGroupName:network:4 | 4 | 5 | 312 | 2/779 | 100 | 40.0% | FAIL ❌ -removeMembers:operation:4 | 4 | 1 | 822 | 822/822 | 200 | 0.0% | FAIL ❌ -sendGroupMessage:operation:4 | 4 | 1 | 795 | 795/795 | 4000 | 100.0% | PASS ✅ -removeMembers:network:4 | 4 | 5 | 378 | 2/943 | 100 | 40.0% | FAIL ❌ -sendGroupMessage:network:4 | 4 | 5 | 310 | 3/773 | 100 | 40.0% | FAIL ❌ -receiveGroupMessage:operation:4 | 4 | 1 | 1211 | 1211/1211 | 4000 | 100.0% | PASS ✅ -receiveGroupMessage:network:4 | 4 | 5 | 307 | 2/766 | 100 | 40.0% | FAIL ❌ -createGroup:operation:8 | 8 | 1 | 1846 | 1846/1846 | 4000 | 100.0% | PASS ✅ -createGroup:network:8 | 8 | 5 | 307 | 6/761 | 100 | 40.0% | FAIL ❌ -createGroupByIdentifiers:operation:8 | 8 | 1 | 1903 | 1903/1903 | 4000 | 100.0% | PASS ✅ -syncGroup:operation:8 | 8 | 1 | 736 | 736/736 | 4000 | 100.0% | PASS ✅ -createGroupByIdentifiers:network:8 | 8 | 5 | 318 | 2/794 | 100 | 40.0% | FAIL ❌ -syncGroup:network:8 | 8 | 5 | 301 | 2/750 | 100 | 40.0% | FAIL ❌ -updateGroupName:operation:8 | 8 | 1 | 1056 | 1056/1056 | 4000 | 100.0% | PASS ✅ -updateGroupName:network:8 | 8 | 5 | 381 | 2/951 | 100 | 40.0% | FAIL ❌ -removeMembers:operation:8 | 8 | 1 | 1047 | 1047/1047 | 200 | 0.0% | FAIL ❌ -removeMembers:network:8 | 8 | 5 | 305 | 2/761 | 100 | 40.0% | FAIL ❌ -sendGroupMessage:operation:8 | 8 | 1 | 824 | 824/824 | 4000 | 100.0% | PASS ✅ -sendGroupMessage:network:8 | 8 | 5 | 303 | 2/757 | 100 | 40.0% | FAIL ❌ -receiveGroupMessage:operation:8 | 8 | 1 | 1038 | 1038/1038 | 4000 | 100.0% | PASS ✅ diff --git a/datadog/reports/ts_performance-south-america-production.md.run-13931915774-production.md b/datadog/reports/ts_performance-south-america-production.md.run-13931915774-production.md new file mode 100644 index 00000000..2189345f --- /dev/null +++ b/datadog/reports/ts_performance-south-america-production.md.run-13931915774-production.md @@ -0,0 +1,55 @@ +# METRICS SUMMARY + +| Operation | Members | Samples | Avg (ms) | Min/Max (ms) | Threshold (ms) | Pass Rate | Status | +| ------------------------------------ | ------- | ------- | -------- | ------------ | -------------- | --------- | ------- | +| clientCreate:operation | - | 1 | 4173 | 4173/4173 | 700 | 0.0% | FAIL ❌ | +| inboxState:operation | - | 1 | 198 | 198/198 | 100 | 0.0% | FAIL ❌ | +| clientCreate:network | - | 5 | 424 | 51/1008 | 100 | 40.0% | FAIL ❌ | +| inboxState:network | - | 5 | 326 | 2/812 | 100 | 40.0% | FAIL ❌ | +| createDM:operation | - | 1 | 2131 | 2131/2131 | 300 | 0.0% | FAIL ❌ | +| createDM:network | - | 5 | 330 | 2/823 | 100 | 40.0% | FAIL ❌ | +| sendGM:operation | - | 1 | 1210 | 1210/1210 | 200 | 0.0% | FAIL ❌ | +| receiveGM:operation | - | 1 | 615 | 615/615 | 200 | 0.0% | FAIL ❌ | +| sendGM:network | - | 5 | 326 | 3/813 | 100 | 40.0% | FAIL ❌ | +| receiveGM:network | - | 5 | 319 | 3/795 | 100 | 40.0% | FAIL ❌ | +| createGroup:operation | - | 1 | 1873 | 1873/1873 | 4000 | 100.0% | PASS ✅ | +| createGroup:network | - | 5 | 378 | 2/944 | 100 | 40.0% | FAIL ❌ | +| createGroupByIdentifiers:operation | - | 1 | 1911 | 1911/1911 | 4000 | 100.0% | PASS ✅ | +| syncGroup:operation | - | 1 | 729 | 729/729 | 4000 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network | - | 5 | 319 | 11/785 | 100 | 40.0% | FAIL ❌ | +| syncGroup:network | - | 5 | 299 | 3/745 | 100 | 40.0% | FAIL ❌ | +| updateGroupName:operation | - | 1 | 1020 | 1020/1020 | 4000 | 100.0% | PASS ✅ | +| updateGroupName:network | - | 5 | 386 | 4/962 | 100 | 40.0% | FAIL ❌ | +| removeMembers:operation | - | 1 | 1035 | 1035/1035 | 200 | 0.0% | FAIL ❌ | +| sendGroupMessage:operation | - | 1 | 621 | 621/621 | 4000 | 100.0% | PASS ✅ | +| removeMembers:network | - | 5 | 323 | 6/802 | 100 | 40.0% | FAIL ❌ | +| sendGroupMessage:network | - | 5 | 391 | 2/975 | 100 | 40.0% | FAIL ❌ | +| receiveGroupMessage:operation | - | 1 | 1213 | 1213/1213 | 4000 | 100.0% | PASS ✅ | +| receiveGroupMessage:network | - | 5 | 382 | 2/953 | 100 | 40.0% | FAIL ❌ | +| createGroup:operation:4 | 4 | 1 | 1879 | 1879/1879 | 4000 | 100.0% | PASS ✅ | +| createGroup:network:4 | 4 | 5 | 384 | 4/955 | 100 | 40.0% | FAIL ❌ | +| createGroupByIdentifiers:operation:4 | 4 | 1 | 2052 | 2052/2052 | 4000 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network:4 | 4 | 5 | 316 | 2/789 | 100 | 40.0% | FAIL ❌ | +| syncGroup:operation:4 | 4 | 1 | 811 | 811/811 | 4000 | 100.0% | PASS ✅ | +| syncGroup:network:4 | 4 | 5 | 306 | 4/760 | 100 | 40.0% | FAIL ❌ | +| updateGroupName:operation:4 | 4 | 1 | 1041 | 1041/1041 | 4000 | 100.0% | PASS ✅ | +| updateGroupName:network:4 | 4 | 5 | 312 | 2/779 | 100 | 40.0% | FAIL ❌ | +| removeMembers:operation:4 | 4 | 1 | 822 | 822/822 | 200 | 0.0% | FAIL ❌ | +| sendGroupMessage:operation:4 | 4 | 1 | 795 | 795/795 | 4000 | 100.0% | PASS ✅ | +| removeMembers:network:4 | 4 | 5 | 378 | 2/943 | 100 | 40.0% | FAIL ❌ | +| sendGroupMessage:network:4 | 4 | 5 | 310 | 3/773 | 100 | 40.0% | FAIL ❌ | +| receiveGroupMessage:operation:4 | 4 | 1 | 1211 | 1211/1211 | 4000 | 100.0% | PASS ✅ | +| receiveGroupMessage:network:4 | 4 | 5 | 307 | 2/766 | 100 | 40.0% | FAIL ❌ | +| createGroup:operation:8 | 8 | 1 | 1846 | 1846/1846 | 4000 | 100.0% | PASS ✅ | +| createGroup:network:8 | 8 | 5 | 307 | 6/761 | 100 | 40.0% | FAIL ❌ | +| createGroupByIdentifiers:operation:8 | 8 | 1 | 1903 | 1903/1903 | 4000 | 100.0% | PASS ✅ | +| syncGroup:operation:8 | 8 | 1 | 736 | 736/736 | 4000 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network:8 | 8 | 5 | 318 | 2/794 | 100 | 40.0% | FAIL ❌ | +| syncGroup:network:8 | 8 | 5 | 301 | 2/750 | 100 | 40.0% | FAIL ❌ | +| updateGroupName:operation:8 | 8 | 1 | 1056 | 1056/1056 | 4000 | 100.0% | PASS ✅ | +| updateGroupName:network:8 | 8 | 5 | 381 | 2/951 | 100 | 40.0% | FAIL ❌ | +| removeMembers:operation:8 | 8 | 1 | 1047 | 1047/1047 | 200 | 0.0% | FAIL ❌ | +| removeMembers:network:8 | 8 | 5 | 305 | 2/761 | 100 | 40.0% | FAIL ❌ | +| sendGroupMessage:operation:8 | 8 | 1 | 824 | 824/824 | 4000 | 100.0% | PASS ✅ | +| sendGroupMessage:network:8 | 8 | 5 | 303 | 2/757 | 100 | 40.0% | FAIL ❌ | +| receiveGroupMessage:operation:8 | 8 | 1 | 1038 | 1038/1038 | 4000 | 100.0% | PASS ✅ | diff --git a/datadog/reports/ts_performance-us-east-production.md b/datadog/reports/ts_performance-us-east-production.md deleted file mode 100644 index 29f0c927..00000000 --- a/datadog/reports/ts_performance-us-east-production.md +++ /dev/null @@ -1,139 +0,0 @@ -# METRICS SUMMARY - -| Operation | Members | Samples | Avg (ms) | Min/Max (ms) | Threshold (ms) | Pass Rate | Status | -| -------------------------------------- | ------- | ------- | -------- | ------------ | -------------- | --------- | ------- | -| clientCreate:operation | - | 1 | 288 | 288/288 | 4500 | 100.0% | PASS ✅ | -| inboxState:operation | - | 1 | 13 | 13/13 | 200 | 100.0% | PASS ✅ | -| createDM:operation | - | 1 | 116 | 116/116 | 1800 | 100.0% | PASS ✅ | -| sendGM:operation | - | 1 | 44 | 44/44 | 1200 | 100.0% | PASS ✅ | -| receiveGM:operation | - | 1 | 37 | 37/37 | 1000 | 100.0% | PASS ✅ | -| clientCreate:network | - | 5 | 208 | 10/298 | 200 | 20.0% | FAIL ❌ | -| inboxState:network | - | 5 | 194 | 10/280 | 200 | 40.0% | PASS ✅ | -| sendGM:network | - | 5 | 54 | 10/107 | 200 | 100.0% | PASS ✅ | -| createDM:network | - | 5 | 99 | 11/167 | 200 | 100.0% | PASS ✅ | -| receiveGM:network | - | 5 | 61 | 11/102 | 200 | 100.0% | PASS ✅ | -| createGroup:operation | - | 1 | 148 | 148/148 | 2000 | 100.0% | PASS ✅ | -| createGroup:network | - | 5 | 67 | 11/109 | 200 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:operation | - | 1 | 125 | 125/125 | 2000 | 100.0% | PASS ✅ | -| syncGroup:operation | - | 1 | 27 | 27/27 | 2000 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network | - | 5 | 29 | 3/65 | 200 | 100.0% | PASS ✅ | -| updateGroupName:operation | - | 1 | 53 | 53/53 | 2000 | 100.0% | PASS ✅ | -| removeMembers:operation | - | 1 | 53 | 53/53 | 1000 | 100.0% | PASS ✅ | -| syncGroup:network | - | 5 | 67 | 11/109 | 200 | 100.0% | PASS ✅ | -| updateGroupName:network | - | 5 | 27 | 3/58 | 200 | 100.0% | PASS ✅ | -| sendGroupMessage:operation | - | 1 | 33 | 33/33 | 2000 | 100.0% | PASS ✅ | -| receiveGroupMessage:operation | - | 1 | 46 | 46/46 | 2000 | 100.0% | PASS ✅ | -| removeMembers:network | - | 5 | 49 | 11/87 | 200 | 100.0% | PASS ✅ | -| sendGroupMessage:network | - | 5 | 27 | 2/59 | 200 | 100.0% | PASS ✅ | -| receiveGroupMessage:network | - | 5 | 30 | 6/65 | 200 | 100.0% | PASS ✅ | -| createGroup:operation:50 | 50 | 1 | 602 | 602/602 | 2000 | 100.0% | PASS ✅ | -| createGroup:network:50 | 50 | 5 | 45 | 11/81 | 200 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:operation:50 | 50 | 1 | 427 | 427/427 | 2000 | 100.0% | PASS ✅ | -| syncGroup:operation:50 | 50 | 1 | 31 | 31/31 | 2000 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network:50 | 50 | 5 | 29 | 3/64 | 200 | 100.0% | PASS ✅ | -| updateGroupName:operation:50 | 50 | 1 | 58 | 58/58 | 2000 | 100.0% | PASS ✅ | -| syncGroup:network:50 | 50 | 5 | 28 | 3/62 | 200 | 100.0% | PASS ✅ | -| removeMembers:operation:50 | 50 | 1 | 65 | 65/65 | 1000 | 100.0% | PASS ✅ | -| updateGroupName:network:50 | 50 | 5 | 47 | 10/87 | 200 | 100.0% | PASS ✅ | -| sendGroupMessage:operation:50 | 50 | 1 | 34 | 34/34 | 2000 | 100.0% | PASS ✅ | -| removeMembers:network:50 | 50 | 5 | 28 | 3/62 | 200 | 100.0% | PASS ✅ | -| receiveGroupMessage:operation:50 | 50 | 1 | 52 | 52/52 | 2000 | 100.0% | PASS ✅ | -| sendGroupMessage:network:50 | 50 | 5 | 28 | 3/62 | 200 | 100.0% | PASS ✅ | -| receiveGroupMessage:network:50 | 50 | 5 | 28 | 3/63 | 200 | 100.0% | PASS ✅ | -| createGroup:operation:100 | 100 | 1 | 945 | 945/945 | 2000 | 100.0% | PASS ✅ | -| createGroup:network:100 | 100 | 5 | 27 | 2/60 | 200 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:operation:100 | 100 | 1 | 967 | 967/967 | 2000 | 100.0% | PASS ✅ | -| syncGroup:operation:100 | 100 | 1 | 36 | 36/36 | 2000 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network:100 | 100 | 5 | 29 | 2/63 | 200 | 100.0% | PASS ✅ | -| updateGroupName:operation:100 | 100 | 1 | 67 | 67/67 | 2000 | 100.0% | PASS ✅ | -| syncGroup:network:100 | 100 | 5 | 27 | 3/59 | 200 | 100.0% | PASS ✅ | -| updateGroupName:network:100 | 100 | 5 | 31 | 5/68 | 200 | 100.0% | PASS ✅ | -| removeMembers:operation:100 | 100 | 1 | 83 | 83/83 | 1000 | 100.0% | PASS ✅ | -| sendGroupMessage:operation:100 | 100 | 1 | 38 | 38/38 | 2000 | 100.0% | PASS ✅ | -| removeMembers:network:100 | 100 | 5 | 27 | 2/59 | 200 | 100.0% | PASS ✅ | -| receiveGroupMessage:operation:100 | 100 | 1 | 57 | 57/57 | 2000 | 100.0% | PASS ✅ | -| sendGroupMessage:network:100 | 100 | 5 | 29 | 3/63 | 200 | 100.0% | PASS ✅ | -| receiveGroupMessage:network:100 | 100 | 5 | 29 | 3/65 | 200 | 100.0% | PASS ✅ | -| createGroup:operation:150 | 150 | 1 | 1537 | 1537/1537 | 2000 | 100.0% | PASS ✅ | -| createGroup:network:150 | 150 | 5 | 27 | 3/60 | 200 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:operation:150 | 150 | 1 | 1516 | 1516/1516 | 2000 | 100.0% | PASS ✅ | -| syncGroup:operation:150 | 150 | 1 | 40 | 40/40 | 2000 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network:150 | 150 | 5 | 27 | 3/60 | 200 | 100.0% | PASS ✅ | -| syncGroup:network:150 | 150 | 5 | 30 | 7/63 | 200 | 100.0% | PASS ✅ | -| updateGroupName:operation:150 | 150 | 1 | 78 | 78/78 | 2000 | 100.0% | PASS ✅ | -| updateGroupName:network:150 | 150 | 5 | 29 | 3/65 | 200 | 100.0% | PASS ✅ | -| removeMembers:operation:150 | 150 | 1 | 100 | 100/100 | 1000 | 100.0% | PASS ✅ | -| sendGroupMessage:operation:150 | 150 | 1 | 42 | 42/42 | 2000 | 100.0% | PASS ✅ | -| removeMembers:network:150 | 150 | 5 | 27 | 3/60 | 200 | 100.0% | PASS ✅ | -| receiveGroupMessage:operation:150 | 150 | 1 | 65 | 65/65 | 2000 | 100.0% | PASS ✅ | -| sendGroupMessage:network:150 | 150 | 5 | 27 | 3/59 | 200 | 100.0% | PASS ✅ | -| receiveGroupMessage:network:150 | 150 | 5 | 27 | 2/61 | 200 | 100.0% | PASS ✅ | -| createGroup:operation:200 | 200 | 1 | 2340 | 2340/2340 | 2000 | 0.0% | FAIL ❌ | -| createGroup:network:200 | 200 | 5 | 27 | 3/60 | 200 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:operation:200 | 200 | 1 | 2307 | 2307/2307 | 2000 | 0.0% | FAIL ❌ | -| syncGroup:operation:200 | 200 | 1 | 57 | 57/57 | 2000 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network:200 | 200 | 5 | 27 | 3/59 | 200 | 100.0% | PASS ✅ | -| syncGroup:network:200 | 200 | 5 | 29 | 5/63 | 200 | 100.0% | PASS ✅ | -| updateGroupName:operation:200 | 200 | 1 | 86 | 86/86 | 2000 | 100.0% | PASS ✅ | -| updateGroupName:network:200 | 200 | 5 | 29 | 2/64 | 200 | 100.0% | PASS ✅ | -| removeMembers:operation:200 | 200 | 1 | 117 | 117/117 | 1000 | 100.0% | PASS ✅ | -| sendGroupMessage:operation:200 | 200 | 1 | 48 | 48/48 | 2000 | 100.0% | PASS ✅ | -| removeMembers:network:200 | 200 | 5 | 29 | 5/62 | 200 | 100.0% | PASS ✅ | -| sendGroupMessage:network:200 | 200 | 5 | 27 | 3/59 | 200 | 100.0% | PASS ✅ | -| receiveGroupMessage:operation:200 | 200 | 1 | 72 | 72/72 | 2000 | 100.0% | PASS ✅ | -| receiveGroupMessage:network:200 | 200 | 5 | 27 | 2/59 | 200 | 100.0% | PASS ✅ | -| createGroup:operation:250 | 250 | 1 | 3272 | 3272/3272 | 2000 | 0.0% | FAIL ❌ | -| createGroup:network:250 | 250 | 5 | 27 | 3/60 | 200 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:operation:250 | 250 | 1 | 3312 | 3312/3312 | 2000 | 0.0% | FAIL ❌ | -| createGroupByIdentifiers:network:250 | 250 | 5 | 26 | 2/58 | 200 | 100.0% | PASS ✅ | -| syncGroup:operation:250 | 250 | 1 | 68 | 68/68 | 2000 | 100.0% | PASS ✅ | -| syncGroup:network:250 | 250 | 5 | 30 | 5/65 | 200 | 100.0% | PASS ✅ | -| updateGroupName:operation:250 | 250 | 1 | 102 | 102/102 | 2000 | 100.0% | PASS ✅ | -| updateGroupName:network:250 | 250 | 5 | 29 | 3/64 | 200 | 100.0% | PASS ✅ | -| removeMembers:operation:250 | 250 | 1 | 152 | 152/152 | 1000 | 100.0% | PASS ✅ | -| sendGroupMessage:operation:250 | 250 | 1 | 41 | 41/41 | 2000 | 100.0% | PASS ✅ | -| removeMembers:network:250 | 250 | 5 | 26 | 2/59 | 200 | 100.0% | PASS ✅ | -| sendGroupMessage:network:250 | 250 | 5 | 26 | 2/59 | 200 | 100.0% | PASS ✅ | -| receiveGroupMessage:operation:250 | 250 | 1 | 78 | 78/78 | 2000 | 100.0% | PASS ✅ | -| receiveGroupMessage:network:250 | 250 | 5 | 33 | 3/74 | 200 | 100.0% | PASS ✅ | -| createGroup:operation:300 | 300 | 1 | 4218 | 4218/4218 | 2000 | 0.0% | FAIL ❌ | -| createGroup:network:300 | 300 | 5 | 26 | 3/57 | 200 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:operation:300 | 300 | 1 | 5683 | 5683/5683 | 2000 | 0.0% | FAIL ❌ | -| createGroupByIdentifiers:network:300 | 300 | 5 | 28 | 5/61 | 200 | 100.0% | PASS ✅ | -| syncGroup:operation:300 | 300 | 1 | 71 | 71/71 | 2000 | 100.0% | PASS ✅ | -| syncGroup:network:300 | 300 | 5 | 31 | 3/68 | 200 | 100.0% | PASS ✅ | -| updateGroupName:operation:300 | 300 | 1 | 118 | 118/118 | 2000 | 100.0% | PASS ✅ | -| updateGroupName:network:300 | 300 | 5 | 26 | 3/58 | 200 | 100.0% | PASS ✅ | -| removeMembers:operation:300 | 300 | 1 | 161 | 161/161 | 1000 | 100.0% | PASS ✅ | -| sendGroupMessage:operation:300 | 300 | 1 | 48 | 48/48 | 2000 | 100.0% | PASS ✅ | -| removeMembers:network:300 | 300 | 5 | 27 | 3/60 | 200 | 100.0% | PASS ✅ | -| sendGroupMessage:network:300 | 300 | 5 | 28 | 4/60 | 200 | 100.0% | PASS ✅ | -| receiveGroupMessage:operation:300 | 300 | 1 | 95 | 95/95 | 2000 | 100.0% | PASS ✅ | -| receiveGroupMessage:network:300 | 300 | 5 | 27 | 2/61 | 200 | 100.0% | PASS ✅ | -| createGroup:operation:350 | 350 | 1 | 4764 | 4764/4764 | 2000 | 0.0% | FAIL ❌ | -| createGroup:network:350 | 350 | 5 | 28 | 2/62 | 200 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:operation:350 | 350 | 1 | 4778 | 4778/4778 | 2000 | 0.0% | FAIL ❌ | -| createGroupByIdentifiers:network:350 | 350 | 5 | 26 | 3/58 | 200 | 100.0% | PASS ✅ | -| syncGroup:operation:350 | 350 | 1 | 82 | 82/82 | 2000 | 100.0% | PASS ✅ | -| syncGroup:network:350 | 350 | 5 | 35 | 3/80 | 200 | 100.0% | PASS ✅ | -| updateGroupName:operation:350 | 350 | 1 | 134 | 134/134 | 2000 | 100.0% | PASS ✅ | -| updateGroupName:network:350 | 350 | 5 | 26 | 2/57 | 200 | 100.0% | PASS ✅ | -| removeMembers:operation:350 | 350 | 1 | 188 | 188/188 | 1000 | 100.0% | PASS ✅ | -| sendGroupMessage:operation:350 | 350 | 1 | 48 | 48/48 | 2000 | 100.0% | PASS ✅ | -| removeMembers:network:350 | 350 | 5 | 27 | 3/59 | 200 | 100.0% | PASS ✅ | -| sendGroupMessage:network:350 | 350 | 5 | 26 | 2/58 | 200 | 100.0% | PASS ✅ | -| receiveGroupMessage:operation:350 | 350 | 1 | 103 | 103/103 | 2000 | 100.0% | PASS ✅ | -| receiveGroupMessage:network:350 | 350 | 5 | 29 | 3/65 | 200 | 100.0% | PASS ✅ | -| createGroup:operation:400 | 400 | 1 | 5537 | 5537/5537 | 2000 | 0.0% | FAIL ❌ | -| createGroup:network:400 | 400 | 5 | 27 | 3/59 | 200 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:operation:400 | 400 | 1 | 5163 | 5163/5163 | 2000 | 0.0% | FAIL ❌ | -| createGroupByIdentifiers:network:400 | 400 | 5 | 29 | 3/65 | 200 | 100.0% | PASS ✅ | -| syncGroup:operation:400 | 400 | 1 | 96 | 96/96 | 2000 | 100.0% | PASS ✅ | -| syncGroup:network:400 | 400 | 5 | 43 | 11/86 | 200 | 100.0% | PASS ✅ | -| updateGroupName:operation:400 | 400 | 1 | 155 | 155/155 | 2000 | 100.0% | PASS ✅ | -| updateGroupName:network:400 | 400 | 5 | 26 | 3/58 | 200 | 100.0% | PASS ✅ | -| removeMembers:operation:400 | 400 | 1 | 224 | 224/224 | 1000 | 100.0% | PASS ✅ | -| sendGroupMessage:operation:400 | 400 | 1 | 52 | 52/52 | 2000 | 100.0% | PASS ✅ | -| removeMembers:network:400 | 400 | 5 | 29 | 2/66 | 200 | 100.0% | PASS ✅ | -| sendGroupMessage:network:400 | 400 | 5 | 34 | 3/76 | 200 | 100.0% | PASS ✅ | -| receiveGroupMessage:operation:400 | 400 | 1 | 119 | 119/119 | 2000 | 100.0% | PASS ✅ | diff --git a/datadog/reports/ts_performance-us-east-production.md.run-13931915774-production.md b/datadog/reports/ts_performance-us-east-production.md.run-13931915774-production.md new file mode 100644 index 00000000..0ca2f5c1 --- /dev/null +++ b/datadog/reports/ts_performance-us-east-production.md.run-13931915774-production.md @@ -0,0 +1,139 @@ +# METRICS SUMMARY + +| Operation | Members | Samples | Avg (ms) | Min/Max (ms) | Threshold (ms) | Pass Rate | Status | +| -------------------------------------- | ------- | ------- | -------- | ------------ | -------------- | --------- | ------- | +| clientCreate:operation | - | 1 | 502 | 502/502 | 350 | 0.0% | FAIL ❌ | +| inboxState:operation | - | 1 | 27 | 27/27 | 50 | 100.0% | PASS ✅ | +| inboxState:network | - | 5 | 102 | 22/174 | 50 | 20.0% | FAIL ❌ | +| clientCreate:network | - | 5 | 131 | 22/210 | 50 | 20.0% | FAIL ❌ | +| createDM:operation | - | 1 | 195 | 195/195 | 150 | 0.0% | FAIL ❌ | +| sendGM:operation | - | 1 | 87 | 87/87 | 100 | 100.0% | PASS ✅ | +| receiveGM:operation | - | 1 | 62 | 62/62 | 100 | 100.0% | PASS ✅ | +| createDM:network | - | 5 | 98 | 22/163 | 50 | 20.0% | FAIL ❌ | +| sendGM:network | - | 5 | 43 | 2/95 | 50 | 60.0% | PASS ✅ | +| receiveGM:network | - | 5 | 95 | 22/159 | 50 | 20.0% | FAIL ❌ | +| createGroup:operation | - | 1 | 225 | 225/225 | 2000 | 100.0% | PASS ✅ | +| createGroup:network | - | 5 | 95 | 23/159 | 50 | 20.0% | FAIL ❌ | +| createGroupByIdentifiers:operation | - | 1 | 232 | 232/232 | 2000 | 100.0% | PASS ✅ | +| syncGroup:operation | - | 1 | 54 | 54/54 | 2000 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network | - | 5 | 66 | 22/124 | 50 | 40.0% | FAIL ❌ | +| updateGroupName:operation | - | 1 | 102 | 102/102 | 2000 | 100.0% | PASS ✅ | +| syncGroup:network | - | 5 | 98 | 23/164 | 50 | 20.0% | FAIL ❌ | +| removeMembers:operation | - | 1 | 93 | 93/93 | 100 | 100.0% | PASS ✅ | +| updateGroupName:network | - | 5 | 52 | 14/106 | 50 | 60.0% | FAIL ❌ | +| sendGroupMessage:operation | - | 1 | 64 | 64/64 | 2000 | 100.0% | PASS ✅ | +| removeMembers:network | - | 5 | 65 | 22/124 | 50 | 40.0% | FAIL ❌ | +| receiveGroupMessage:operation | - | 1 | 91 | 91/91 | 2000 | 100.0% | PASS ✅ | +| sendGroupMessage:network | - | 5 | 51 | 13/103 | 50 | 60.0% | FAIL ❌ | +| receiveGroupMessage:network | - | 5 | 69 | 22/132 | 50 | 40.0% | FAIL ❌ | +| createGroup:operation:50 | 50 | 1 | 837 | 837/837 | 2000 | 100.0% | PASS ✅ | +| createGroup:network:50 | 50 | 5 | 44 | 2/97 | 50 | 60.0% | PASS ✅ | +| createGroupByIdentifiers:operation:50 | 50 | 1 | 558 | 558/558 | 2000 | 100.0% | PASS ✅ | +| syncGroup:operation:50 | 50 | 1 | 63 | 63/63 | 2000 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network:50 | 50 | 5 | 52 | 14/106 | 50 | 60.0% | FAIL ❌ | +| updateGroupName:operation:50 | 50 | 1 | 102 | 102/102 | 2000 | 100.0% | PASS ✅ | +| syncGroup:network:50 | 50 | 5 | 51 | 13/104 | 50 | 60.0% | FAIL ❌ | +| updateGroupName:network:50 | 50 | 5 | 45 | 2/99 | 50 | 60.0% | PASS ✅ | +| removeMembers:operation:50 | 50 | 1 | 122 | 122/122 | 100 | 0.0% | FAIL ❌ | +| sendGroupMessage:operation:50 | 50 | 1 | 63 | 63/63 | 2000 | 100.0% | PASS ✅ | +| removeMembers:network:50 | 50 | 5 | 51 | 13/104 | 50 | 60.0% | FAIL ❌ | +| receiveGroupMessage:operation:50 | 50 | 1 | 98 | 98/98 | 2000 | 100.0% | PASS ✅ | +| sendGroupMessage:network:50 | 50 | 5 | 43 | 2/94 | 50 | 60.0% | PASS ✅ | +| receiveGroupMessage:network:50 | 50 | 5 | 56 | 13/116 | 50 | 60.0% | FAIL ❌ | +| createGroup:operation:100 | 100 | 1 | 1577 | 1577/1577 | 2000 | 100.0% | PASS ✅ | +| createGroup:network:100 | 100 | 5 | 91 | 22/153 | 50 | 20.0% | FAIL ❌ | +| createGroupByIdentifiers:operation:100 | 100 | 1 | 1399 | 1399/1399 | 2000 | 100.0% | PASS ✅ | +| syncGroup:operation:100 | 100 | 1 | 65 | 65/65 | 2000 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network:100 | 100 | 5 | 54 | 14/109 | 50 | 60.0% | FAIL ❌ | +| syncGroup:network:100 | 100 | 5 | 45 | 4/97 | 50 | 60.0% | PASS ✅ | +| updateGroupName:operation:100 | 100 | 1 | 109 | 109/109 | 2000 | 100.0% | PASS ✅ | +| updateGroupName:network:100 | 100 | 5 | 45 | 2/98 | 50 | 60.0% | PASS ✅ | +| removeMembers:operation:100 | 100 | 1 | 120 | 120/120 | 100 | 0.0% | FAIL ❌ | +| sendGroupMessage:operation:100 | 100 | 1 | 62 | 62/62 | 2000 | 100.0% | PASS ✅ | +| receiveGroupMessage:operation:100 | 100 | 1 | 102 | 102/102 | 2000 | 100.0% | PASS ✅ | +| sendGroupMessage:network:100 | 100 | 5 | 45 | 2/101 | 50 | 60.0% | PASS ✅ | +| removeMembers:network:100 | 100 | 5 | 95 | 23/164 | 50 | 20.0% | FAIL ❌ | +| receiveGroupMessage:network:100 | 100 | 5 | 52 | 13/105 | 50 | 60.0% | FAIL ❌ | +| createGroup:operation:150 | 150 | 1 | 2266 | 2266/2266 | 2000 | 0.0% | FAIL ❌ | +| createGroup:network:150 | 150 | 5 | 55 | 16/110 | 50 | 60.0% | FAIL ❌ | +| createGroupByIdentifiers:operation:150 | 150 | 1 | 2562 | 2562/2562 | 2000 | 0.0% | FAIL ❌ | +| syncGroup:operation:150 | 150 | 1 | 88 | 88/88 | 2000 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network:150 | 150 | 5 | 43 | 2/94 | 50 | 60.0% | PASS ✅ | +| syncGroup:network:150 | 150 | 5 | 44 | 4/94 | 50 | 60.0% | PASS ✅ | +| updateGroupName:operation:150 | 150 | 1 | 122 | 122/122 | 2000 | 100.0% | PASS ✅ | +| updateGroupName:network:150 | 150 | 5 | 46 | 3/101 | 50 | 60.0% | PASS ✅ | +| removeMembers:operation:150 | 150 | 1 | 139 | 139/139 | 100 | 0.0% | FAIL ❌ | +| sendGroupMessage:operation:150 | 150 | 1 | 64 | 64/64 | 2000 | 100.0% | PASS ✅ | +| removeMembers:network:150 | 150 | 5 | 94 | 22/162 | 50 | 20.0% | FAIL ❌ | +| sendGroupMessage:network:150 | 150 | 5 | 43 | 2/95 | 50 | 60.0% | PASS ✅ | +| receiveGroupMessage:operation:150 | 150 | 1 | 113 | 113/113 | 2000 | 100.0% | PASS ✅ | +| receiveGroupMessage:network:150 | 150 | 5 | 45 | 2/98 | 50 | 60.0% | PASS ✅ | +| createGroup:operation:200 | 200 | 1 | 3306 | 3306/3306 | 2000 | 0.0% | FAIL ❌ | +| createGroup:network:200 | 200 | 5 | 44 | 3/96 | 50 | 60.0% | PASS ✅ | +| createGroupByIdentifiers:operation:200 | 200 | 1 | 3394 | 3394/3394 | 2000 | 0.0% | FAIL ❌ | +| syncGroup:operation:200 | 200 | 1 | 97 | 97/97 | 2000 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network:200 | 200 | 5 | 50 | 10/103 | 50 | 60.0% | PASS ✅ | +| updateGroupName:operation:200 | 200 | 1 | 137 | 137/137 | 2000 | 100.0% | PASS ✅ | +| syncGroup:network:200 | 200 | 5 | 93 | 23/159 | 50 | 20.0% | FAIL ❌ | +| removeMembers:operation:200 | 200 | 1 | 161 | 161/161 | 100 | 0.0% | FAIL ❌ | +| updateGroupName:network:200 | 200 | 5 | 98 | 22/164 | 50 | 20.0% | FAIL ❌ | +| sendGroupMessage:operation:200 | 200 | 1 | 67 | 67/67 | 2000 | 100.0% | PASS ✅ | +| removeMembers:network:200 | 200 | 5 | 43 | 2/94 | 50 | 60.0% | PASS ✅ | +| sendGroupMessage:network:200 | 200 | 5 | 44 | 2/97 | 50 | 60.0% | PASS ✅ | +| receiveGroupMessage:operation:200 | 200 | 1 | 129 | 129/129 | 2000 | 100.0% | PASS ✅ | +| receiveGroupMessage:network:200 | 200 | 5 | 43 | 3/94 | 50 | 60.0% | PASS ✅ | +| createGroup:operation:250 | 250 | 1 | 5425 | 5425/5425 | 2000 | 0.0% | FAIL ❌ | +| createGroup:network:250 | 250 | 5 | 43 | 3/94 | 50 | 60.0% | PASS ✅ | +| createGroupByIdentifiers:operation:250 | 250 | 1 | 5085 | 5085/5085 | 2000 | 0.0% | FAIL ❌ | +| syncGroup:operation:250 | 250 | 1 | 102 | 102/102 | 2000 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network:250 | 250 | 5 | 43 | 2/95 | 50 | 60.0% | PASS ✅ | +| syncGroup:network:250 | 250 | 5 | 42 | 2/93 | 50 | 60.0% | PASS ✅ | +| updateGroupName:operation:250 | 250 | 1 | 161 | 161/161 | 2000 | 100.0% | PASS ✅ | +| updateGroupName:network:250 | 250 | 5 | 46 | 3/100 | 50 | 60.0% | PASS ✅ | +| removeMembers:operation:250 | 250 | 1 | 200 | 200/200 | 100 | 0.0% | FAIL ❌ | +| sendGroupMessage:operation:250 | 250 | 1 | 72 | 72/72 | 2000 | 100.0% | PASS ✅ | +| removeMembers:network:250 | 250 | 5 | 44 | 2/98 | 50 | 60.0% | PASS ✅ | +| sendGroupMessage:network:250 | 250 | 5 | 46 | 2/102 | 50 | 60.0% | PASS ✅ | +| receiveGroupMessage:operation:250 | 250 | 1 | 132 | 132/132 | 2000 | 100.0% | PASS ✅ | +| receiveGroupMessage:network:250 | 250 | 5 | 43 | 2/94 | 50 | 60.0% | PASS ✅ | +| createGroup:operation:300 | 300 | 1 | 5392 | 5392/5392 | 2000 | 0.0% | FAIL ❌ | +| createGroup:network:300 | 300 | 5 | 45 | 3/97 | 50 | 60.0% | PASS ✅ | +| createGroupByIdentifiers:operation:300 | 300 | 1 | 6466 | 6466/6466 | 2000 | 0.0% | FAIL ❌ | +| createGroupByIdentifiers:network:300 | 300 | 5 | 44 | 3/95 | 50 | 60.0% | PASS ✅ | +| syncGroup:operation:300 | 300 | 1 | 114 | 114/114 | 2000 | 100.0% | PASS ✅ | +| syncGroup:network:300 | 300 | 5 | 55 | 2/124 | 50 | 60.0% | FAIL ❌ | +| updateGroupName:operation:300 | 300 | 1 | 177 | 177/177 | 2000 | 100.0% | PASS ✅ | +| updateGroupName:network:300 | 300 | 5 | 43 | 2/95 | 50 | 60.0% | PASS ✅ | +| removeMembers:operation:300 | 300 | 1 | 200 | 200/200 | 100 | 0.0% | FAIL ❌ | +| sendGroupMessage:operation:300 | 300 | 1 | 72 | 72/72 | 2000 | 100.0% | PASS ✅ | +| removeMembers:network:300 | 300 | 5 | 44 | 3/96 | 50 | 60.0% | PASS ✅ | +| sendGroupMessage:network:300 | 300 | 5 | 43 | 2/94 | 50 | 60.0% | PASS ✅ | +| receiveGroupMessage:operation:300 | 300 | 1 | 136 | 136/136 | 2000 | 100.0% | PASS ✅ | +| receiveGroupMessage:network:300 | 300 | 5 | 56 | 13/116 | 50 | 60.0% | FAIL ❌ | +| createGroup:operation:350 | 350 | 1 | 7734 | 7734/7734 | 2000 | 0.0% | FAIL ❌ | +| createGroup:network:350 | 350 | 5 | 44 | 2/96 | 50 | 60.0% | PASS ✅ | +| createGroupByIdentifiers:operation:350 | 350 | 1 | 6067 | 6067/6067 | 2000 | 0.0% | FAIL ❌ | +| createGroupByIdentifiers:network:350 | 350 | 5 | 43 | 3/95 | 50 | 60.0% | PASS ✅ | +| syncGroup:operation:350 | 350 | 1 | 121 | 121/121 | 2000 | 100.0% | PASS ✅ | +| syncGroup:network:350 | 350 | 5 | 49 | 10/102 | 50 | 60.0% | PASS ✅ | +| updateGroupName:operation:350 | 350 | 1 | 175 | 175/175 | 2000 | 100.0% | PASS ✅ | +| updateGroupName:network:350 | 350 | 5 | 46 | 2/102 | 50 | 60.0% | PASS ✅ | +| removeMembers:operation:350 | 350 | 1 | 234 | 234/234 | 100 | 0.0% | FAIL ❌ | +| sendGroupMessage:operation:350 | 350 | 1 | 76 | 76/76 | 2000 | 100.0% | PASS ✅ | +| removeMembers:network:350 | 350 | 5 | 43 | 2/95 | 50 | 60.0% | PASS ✅ | +| sendGroupMessage:network:350 | 350 | 5 | 45 | 2/99 | 50 | 60.0% | PASS ✅ | +| receiveGroupMessage:operation:350 | 350 | 1 | 150 | 150/150 | 2000 | 100.0% | PASS ✅ | +| receiveGroupMessage:network:350 | 350 | 5 | 44 | 3/95 | 50 | 60.0% | PASS ✅ | +| createGroup:operation:400 | 400 | 1 | 6742 | 6742/6742 | 2000 | 0.0% | FAIL ❌ | +| createGroup:network:400 | 400 | 5 | 163 | 23/245 | 50 | 20.0% | FAIL ❌ | +| createGroupByIdentifiers:operation:400 | 400 | 1 | 6513 | 6513/6513 | 2000 | 0.0% | FAIL ❌ | +| createGroupByIdentifiers:network:400 | 400 | 5 | 44 | 3/95 | 50 | 60.0% | PASS ✅ | +| syncGroup:operation:400 | 400 | 1 | 131 | 131/131 | 2000 | 100.0% | PASS ✅ | +| syncGroup:network:400 | 400 | 5 | 56 | 17/112 | 50 | 60.0% | FAIL ❌ | +| updateGroupName:operation:400 | 400 | 1 | 191 | 191/191 | 2000 | 100.0% | PASS ✅ | +| updateGroupName:network:400 | 400 | 5 | 59 | 22/114 | 50 | 60.0% | FAIL ❌ | +| removeMembers:operation:400 | 400 | 1 | 269 | 269/269 | 100 | 0.0% | FAIL ❌ | +| sendGroupMessage:operation:400 | 400 | 1 | 82 | 82/82 | 2000 | 100.0% | PASS ✅ | +| removeMembers:network:400 | 400 | 5 | 98 | 22/164 | 50 | 20.0% | FAIL ❌ | +| sendGroupMessage:network:400 | 400 | 5 | 43 | 3/95 | 50 | 60.0% | PASS ✅ | +| receiveGroupMessage:operation:400 | 400 | 1 | 158 | 158/158 | 2000 | 100.0% | PASS ✅ | diff --git a/helpers/datadog.ts b/helpers/datadog.ts index f5229304..f721f205 100644 --- a/helpers/datadog.ts +++ b/helpers/datadog.ts @@ -16,28 +16,48 @@ let collectedMetrics: Record< // Refactored thresholds into a single configuration object const THRESHOLDS = { core: { - clientcreate: 350, - createdm: 150, + clientcreate: 550, + createdm: 200, sendgm: 100, receivegm: 100, - creategroup: 200, - creategroupbyidentifiers: 150, + creategroup: 2000, + creategroupbyidentifiers: 2000, receivegroupmessage: 150, updategroupname: 100, syncgroup: 100, addmembers: 250, - removemembers: 100, + removemembers: 250, inboxstate: 50, }, network: { - dns_lookup: 50, - tcp_connection: 50, - tls_handshake: 100, - processing: 50, - server_call: 100, + dns_lookup: 175, + tcp_connection: 175, + tls_handshake: 175, + processing: 175, + server_call: 175, }, group: { - createGroup: { + creategroup: { + "50": 900, + "100": 1600, + "150": 2300, + "200": 3500, + "250": 5500, + "300": 5500, + "350": 8000, + "400": 9000, + }, + creategroupbyidentifiers: { + "50": 600, + "100": 1500, + "150": 2600, + "200": 3500, + "250": 5500, + "300": 6500, + "350": 6500, + "400": 7500, + }, + sendgroupmessage: { "50": 650, "100": 1000, "150": 1600, @@ -47,7 +67,7 @@ const THRESHOLDS = { "350": 5000, "400": 6000, }, - createGroupByIdentifiers: { + creategroupbyidentifiers: { "50": 500, "100": 1100, "150": 1700, @@ -57,7 +77,7 @@ const THRESHOLDS = { "350": 5000, "400": 5500, }, - sendGroupMessage: { + sendgroupmessage: { "50": 50, "100": 50, "150": 50, @@ -67,7 +87,7 @@ const THRESHOLDS = { "350": 60, "400": 70, }, - syncGroup: { + syncgroup: { "50": 50, "100": 50, "150": 50, @@ -77,7 +97,7 @@ const THRESHOLDS = { "350": 100, "400": 120, }, - updateGroupName: { + updategroupname: { "50": 75, "100": 80, "150": 100, @@ -87,7 +107,7 @@ const THRESHOLDS = { "350": 150, "400": 200, }, - removeMembers: { + removemembers: { "50": 80, "100": 100, "150": 120, From 4159bf9af4d00a349135dba37b5e2c4709fda3c2 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 16:39:36 -0300 Subject: [PATCH 12/57] artifcacts --- .github/workflows/TS_Performance.yml | 46 +++++++++++----------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/.github/workflows/TS_Performance.yml b/.github/workflows/TS_Performance.yml index b4ab8f6d..3270a883 100644 --- a/.github/workflows/TS_Performance.yml +++ b/.github/workflows/TS_Performance.yml @@ -34,24 +34,19 @@ jobs: run: ./scripts/run-test.sh TS_Performance - name: Rename logs with run ID prefix run: | - mkdir -p renamed_logs - cp -r logs/* renamed_logs/ - find renamed_logs -type f -exec mv {} {}.run-${{ github.run_id }}-dev \; + mkdir -p artifacts/logs + cp -r logs/* artifacts/logs/ + find artifacts/logs -type f -exec mv {} {}.run-${{ github.run_id }}-dev \; - name: Rename datadog reports with run ID prefix run: | - mkdir -p renamed_reports - cp -r datadog/reports/* renamed_reports/ - find renamed_reports -type f -exec mv {} {}.run-${{ github.run_id }}-dev \; - - name: Upload log files + mkdir -p artifacts/reports + cp -r datadog/reports/* artifacts/reports/ + find artifacts/reports -type f -exec mv {} {}.run-${{ github.run_id }}-dev.md \; + - name: Upload all artifacts uses: actions/upload-artifact@v4 with: - name: performance_logs_${{ env.XMTP_ENV }} - path: renamed_logs/ - - name: Upload datadog reports - uses: actions/upload-artifact@v4 - with: - name: datadog_reports_${{ env.XMTP_ENV }} - path: renamed_reports/ + name: performance_data_${{ env.XMTP_ENV }} + path: artifacts/ production: runs-on: ubuntu-latest env: @@ -77,21 +72,16 @@ jobs: run: ./scripts/run-test.sh TS_Performance - name: Rename logs with run ID prefix run: | - mkdir -p renamed_logs - cp -r logs/* renamed_logs/ - find renamed_logs -type f -exec mv {} {}.run-${{ github.run_id }}-production \; + mkdir -p artifacts/logs + cp -r logs/* artifacts/logs/ + find artifacts/logs -type f -exec mv {} {}.run-${{ github.run_id }}-production \; - name: Rename datadog reports with run ID prefix run: | - mkdir -p renamed_reports - cp -r datadog/reports/* renamed_reports/ - find renamed_reports -type f -exec mv {} {}.run-${{ github.run_id }}-production \; - - name: Upload log files - uses: actions/upload-artifact@v4 - with: - name: performance_logs_${{ env.XMTP_ENV }} - path: renamed_logs/ - - name: Upload datadog reports + mkdir -p artifacts/reports + cp -r datadog/reports/* artifacts/reports/ + find artifacts/reports -type f -exec mv {} {}.run-${{ github.run_id }}-production.md \; + - name: Upload all artifacts uses: actions/upload-artifact@v4 with: - name: datadog_reports_${{ env.XMTP_ENV }} - path: renamed_reports/ + name: performance_data_${{ env.XMTP_ENV }} + path: artifacts/ From 025ab0671231c2debb49928ec384b8c0399297d4 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 16:40:54 -0300 Subject: [PATCH 13/57] fix lint --- helpers/datadog.ts | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/helpers/datadog.ts b/helpers/datadog.ts index f721f205..4f1048df 100644 --- a/helpers/datadog.ts +++ b/helpers/datadog.ts @@ -67,26 +67,6 @@ const THRESHOLDS = { "350": 5000, "400": 6000, }, - creategroupbyidentifiers: { - "50": 500, - "100": 1100, - "150": 1700, - "200": 2500, - "250": 3500, - "300": 6000, - "350": 5000, - "400": 5500, - }, - sendgroupmessage: { - "50": 50, - "100": 50, - "150": 50, - "200": 70, - "250": 50, - "300": 60, - "350": 60, - "400": 70, - }, syncgroup: { "50": 50, "100": 50, From 47c7094af7f421703bb5b198ff840cccbae8c909 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 16:45:43 -0300 Subject: [PATCH 14/57] fix lint --- .github/workflows/TS_Performance.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/TS_Performance.yml b/.github/workflows/TS_Performance.yml index 3270a883..cd5e2129 100644 --- a/.github/workflows/TS_Performance.yml +++ b/.github/workflows/TS_Performance.yml @@ -36,16 +36,18 @@ jobs: run: | mkdir -p artifacts/logs cp -r logs/* artifacts/logs/ - find artifacts/logs -type f -exec mv {} {}.run-${{ github.run_id }}-dev \; + find artifacts/logs -type f -name "*.log" -exec bash -c 'mv "$0" "${0%.log}.run-${{ github.run_id }}-dev.txt"' {} \; + find artifacts/logs -type f ! -name "*.run-*" -exec bash -c 'mv "$0" "$0.run-${{ github.run_id }}-dev.txt"' {} \; - name: Rename datadog reports with run ID prefix run: | mkdir -p artifacts/reports cp -r datadog/reports/* artifacts/reports/ - find artifacts/reports -type f -exec mv {} {}.run-${{ github.run_id }}-dev.md \; + find artifacts/reports -type f -name "*.md" -exec bash -c 'mv "$0" "${0%.md}.run-${{ github.run_id }}-dev.md"' {} \; + find artifacts/reports -type f ! -name "*.run-*" -exec bash -c 'mv "$0" "$0.run-${{ github.run_id }}-dev.md"' {} \; - name: Upload all artifacts uses: actions/upload-artifact@v4 with: - name: performance_data_${{ env.XMTP_ENV }} + name: ts_performance_${{ env.XMTP_ENV }} path: artifacts/ production: runs-on: ubuntu-latest @@ -74,14 +76,16 @@ jobs: run: | mkdir -p artifacts/logs cp -r logs/* artifacts/logs/ - find artifacts/logs -type f -exec mv {} {}.run-${{ github.run_id }}-production \; + find artifacts/logs -type f -name "*.log" -exec bash -c 'mv "$0" "${0%.log}.run-${{ github.run_id }}-production.txt"' {} \; + find artifacts/logs -type f ! -name "*.run-*" -exec bash -c 'mv "$0" "$0.run-${{ github.run_id }}-production.txt"' {} \; - name: Rename datadog reports with run ID prefix run: | mkdir -p artifacts/reports cp -r datadog/reports/* artifacts/reports/ - find artifacts/reports -type f -exec mv {} {}.run-${{ github.run_id }}-production.md \; + find artifacts/reports -type f -name "*.md" -exec bash -c 'mv "$0" "${0%.md}.run-${{ github.run_id }}-production.md"' {} \; + find artifacts/reports -type f ! -name "*.run-*" -exec bash -c 'mv "$0" "$0.run-${{ github.run_id }}-production.md"' {} \; - name: Upload all artifacts uses: actions/upload-artifact@v4 with: - name: performance_data_${{ env.XMTP_ENV }} + name: ts_performance_${{ env.XMTP_ENV }} path: artifacts/ From 5644834618c8fe9d12fe5cc388ced28fcd7a3884 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 16:47:48 -0300 Subject: [PATCH 15/57] variance --- ...roduction.md.run-13932078204-production.md | 139 ++++++++++++++++++ helpers/datadog.ts | 38 +++-- 2 files changed, 161 insertions(+), 16 deletions(-) create mode 100644 datadog/reports/ts_performance-us-east-production.md.run-13932078204-production.md diff --git a/datadog/reports/ts_performance-us-east-production.md.run-13932078204-production.md b/datadog/reports/ts_performance-us-east-production.md.run-13932078204-production.md new file mode 100644 index 00000000..4c3d2938 --- /dev/null +++ b/datadog/reports/ts_performance-us-east-production.md.run-13932078204-production.md @@ -0,0 +1,139 @@ +# METRICS SUMMARY + +| Operation | Members | Samples | Avg (ms) | Min/Max (ms) | Threshold (ms) | Pass Rate | Status | +| -------------------------------------- | ------- | ------- | -------- | ------------ | -------------- | --------- | ------- | +| clientCreate:operation | - | 1 | 355 | 355/355 | 550 | 100.0% | PASS ✅ | +| inboxState:operation | - | 1 | 19 | 19/19 | 50 | 100.0% | PASS ✅ | +| createDM:operation | - | 1 | 131 | 131/131 | 200 | 100.0% | PASS ✅ | +| inboxState:network | - | 5 | 82 | 16/135 | 175 | 100.0% | PASS ✅ | +| clientCreate:network | - | 5 | 106 | 16/166 | 175 | 100.0% | PASS ✅ | +| sendGM:operation | - | 1 | 59 | 59/59 | 100 | 100.0% | PASS ✅ | +| createDM:network | - | 5 | 33 | 2/73 | 175 | 100.0% | PASS ✅ | +| receiveGM:operation | - | 1 | 47 | 47/47 | 100 | 100.0% | PASS ✅ | +| sendGM:network | - | 5 | 55 | 15/102 | 175 | 100.0% | PASS ✅ | +| receiveGM:network | - | 5 | 56 | 15/102 | 175 | 100.0% | PASS ✅ | +| createGroup:operation | - | 1 | 149 | 149/149 | 900 | 100.0% | PASS ✅ | +| createGroup:network | - | 5 | 48 | 15/96 | 175 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:operation | - | 1 | 162 | 162/162 | 600 | 100.0% | PASS ✅ | +| syncGroup:operation | - | 1 | 36 | 36/36 | 50 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network | - | 5 | 58 | 15/104 | 175 | 100.0% | PASS ✅ | +| updateGroupName:operation | - | 1 | 67 | 67/67 | 75 | 100.0% | PASS ✅ | +| syncGroup:network | - | 5 | 89 | 16/144 | 175 | 100.0% | PASS ✅ | +| removeMembers:operation | - | 1 | 79 | 79/79 | 250 | 100.0% | PASS ✅ | +| updateGroupName:network | - | 5 | 33 | 2/74 | 175 | 100.0% | PASS ✅ | +| sendGroupMessage:operation | - | 1 | 79 | 79/79 | 650 | 100.0% | PASS ✅ | +| removeMembers:network | - | 5 | 43 | 9/90 | 175 | 100.0% | PASS ✅ | +| receiveGroupMessage:operation | - | 1 | 70 | 70/70 | 2000 | 100.0% | PASS ✅ | +| sendGroupMessage:network | - | 5 | 48 | 13/99 | 175 | 100.0% | PASS ✅ | +| receiveGroupMessage:network | - | 5 | 59 | 16/110 | 175 | 100.0% | PASS ✅ | +| createGroup:operation:50 | 50 | 1 | 2148 | 2148/2148 | 900 | 0.0% | FAIL ❌ | +| createGroup:network:50 | 50 | 5 | 38 | 9/79 | 175 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:operation:50 | 50 | 1 | 499 | 499/499 | 600 | 100.0% | PASS ✅ | +| syncGroup:operation:50 | 50 | 1 | 41 | 41/41 | 50 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network:50 | 50 | 5 | 35 | 3/76 | 175 | 100.0% | PASS ✅ | +| updateGroupName:operation:50 | 50 | 1 | 82 | 82/82 | 75 | 0.0% | FAIL ❌ | +| syncGroup:network:50 | 50 | 5 | 59 | 16/106 | 175 | 100.0% | PASS ✅ | +| removeMembers:operation:50 | 50 | 1 | 77 | 77/77 | 250 | 100.0% | PASS ✅ | +| updateGroupName:network:50 | 50 | 5 | 41 | 13/82 | 175 | 100.0% | PASS ✅ | +| sendGroupMessage:operation:50 | 50 | 1 | 43 | 43/43 | 650 | 100.0% | PASS ✅ | +| removeMembers:network:50 | 50 | 5 | 35 | 2/77 | 175 | 100.0% | PASS ✅ | +| receiveGroupMessage:operation:50 | 50 | 1 | 69 | 69/69 | 2000 | 100.0% | PASS ✅ | +| sendGroupMessage:network:50 | 50 | 5 | 61 | 15/110 | 175 | 100.0% | PASS ✅ | +| receiveGroupMessage:network:50 | 50 | 5 | 46 | 13/94 | 175 | 100.0% | PASS ✅ | +| createGroup:operation:100 | 100 | 1 | 1317 | 1317/1317 | 1600 | 100.0% | PASS ✅ | +| createGroup:network:100 | 100 | 5 | 34 | 3/74 | 175 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:operation:100 | 100 | 1 | 977 | 977/977 | 1500 | 100.0% | PASS ✅ | +| syncGroup:operation:100 | 100 | 1 | 50 | 50/50 | 50 | 0.0% | FAIL ❌ | +| createGroupByIdentifiers:network:100 | 100 | 5 | 35 | 2/78 | 175 | 100.0% | PASS ✅ | +| updateGroupName:operation:100 | 100 | 1 | 80 | 80/80 | 80 | 100.0% | PASS ✅ | +| syncGroup:network:100 | 100 | 5 | 35 | 4/77 | 175 | 100.0% | PASS ✅ | +| updateGroupName:network:100 | 100 | 5 | 35 | 2/77 | 175 | 100.0% | PASS ✅ | +| removeMembers:operation:100 | 100 | 1 | 94 | 94/94 | 250 | 100.0% | PASS ✅ | +| sendGroupMessage:operation:100 | 100 | 1 | 50 | 50/50 | 1000 | 100.0% | PASS ✅ | +| removeMembers:network:100 | 100 | 5 | 35 | 2/78 | 175 | 100.0% | PASS ✅ | +| receiveGroupMessage:operation:100 | 100 | 1 | 71 | 71/71 | 2000 | 100.0% | PASS ✅ | +| sendGroupMessage:network:100 | 100 | 5 | 34 | 2/74 | 175 | 100.0% | PASS ✅ | +| receiveGroupMessage:network:100 | 100 | 5 | 33 | 2/74 | 175 | 100.0% | PASS ✅ | +| createGroup:operation:150 | 150 | 1 | 1730 | 1730/1730 | 2300 | 100.0% | PASS ✅ | +| createGroup:network:150 | 150 | 5 | 33 | 2/73 | 175 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:operation:150 | 150 | 1 | 1688 | 1688/1688 | 2600 | 100.0% | PASS ✅ | +| syncGroup:operation:150 | 150 | 1 | 58 | 58/58 | 50 | 0.0% | FAIL ❌ | +| createGroupByIdentifiers:network:150 | 150 | 5 | 46 | 14/92 | 175 | 100.0% | PASS ✅ | +| syncGroup:network:150 | 150 | 5 | 36 | 6/77 | 175 | 100.0% | PASS ✅ | +| updateGroupName:operation:150 | 150 | 1 | 88 | 88/88 | 100 | 100.0% | PASS ✅ | +| updateGroupName:network:150 | 150 | 5 | 34 | 2/76 | 175 | 100.0% | PASS ✅ | +| removeMembers:operation:150 | 150 | 1 | 108 | 108/108 | 250 | 100.0% | PASS ✅ | +| sendGroupMessage:operation:150 | 150 | 1 | 48 | 48/48 | 1600 | 100.0% | PASS ✅ | +| removeMembers:network:150 | 150 | 5 | 33 | 2/72 | 175 | 100.0% | PASS ✅ | +| receiveGroupMessage:operation:150 | 150 | 1 | 83 | 83/83 | 2000 | 100.0% | PASS ✅ | +| sendGroupMessage:network:150 | 150 | 5 | 42 | 13/85 | 175 | 100.0% | PASS ✅ | +| receiveGroupMessage:network:150 | 150 | 5 | 44 | 2/101 | 175 | 100.0% | PASS ✅ | +| createGroup:operation:200 | 200 | 1 | 2582 | 2582/2582 | 3500 | 100.0% | PASS ✅ | +| createGroup:network:200 | 200 | 5 | 33 | 3/73 | 175 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:operation:200 | 200 | 1 | 2627 | 2627/2627 | 3500 | 100.0% | PASS ✅ | +| syncGroup:operation:200 | 200 | 1 | 75 | 75/75 | 70 | 0.0% | FAIL ❌ | +| createGroupByIdentifiers:network:200 | 200 | 5 | 36 | 4/79 | 175 | 100.0% | PASS ✅ | +| syncGroup:network:200 | 200 | 5 | 37 | 8/77 | 175 | 100.0% | PASS ✅ | +| updateGroupName:operation:200 | 200 | 1 | 99 | 99/99 | 100 | 100.0% | PASS ✅ | +| updateGroupName:network:200 | 200 | 5 | 34 | 2/76 | 175 | 100.0% | PASS ✅ | +| removeMembers:operation:200 | 200 | 1 | 137 | 137/137 | 250 | 100.0% | PASS ✅ | +| sendGroupMessage:operation:200 | 200 | 1 | 48 | 48/48 | 2500 | 100.0% | PASS ✅ | +| receiveGroupMessage:operation:200 | 200 | 1 | 85 | 85/85 | 2000 | 100.0% | PASS ✅ | +| sendGroupMessage:network:200 | 200 | 5 | 44 | 8/95 | 175 | 100.0% | PASS ✅ | +| removeMembers:network:200 | 200 | 5 | 91 | 15/154 | 175 | 100.0% | PASS ✅ | +| receiveGroupMessage:network:200 | 200 | 5 | 38 | 2/85 | 175 | 100.0% | PASS ✅ | +| createGroup:operation:250 | 250 | 1 | 3625 | 3625/3625 | 5500 | 100.0% | PASS ✅ | +| createGroup:network:250 | 250 | 5 | 33 | 2/72 | 175 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:operation:250 | 250 | 1 | 3566 | 3566/3566 | 5500 | 100.0% | PASS ✅ | +| syncGroup:operation:250 | 250 | 1 | 77 | 77/77 | 80 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network:250 | 250 | 5 | 36 | 3/78 | 175 | 100.0% | PASS ✅ | +| syncGroup:network:250 | 250 | 5 | 34 | 3/74 | 175 | 100.0% | PASS ✅ | +| updateGroupName:operation:250 | 250 | 1 | 131 | 131/131 | 120 | 0.0% | FAIL ❌ | +| updateGroupName:network:250 | 250 | 5 | 35 | 3/77 | 175 | 100.0% | PASS ✅ | +| removeMembers:operation:250 | 250 | 1 | 154 | 154/154 | 250 | 100.0% | PASS ✅ | +| sendGroupMessage:operation:250 | 250 | 1 | 53 | 53/53 | 3500 | 100.0% | PASS ✅ | +| removeMembers:network:250 | 250 | 5 | 67 | 15/115 | 175 | 100.0% | PASS ✅ | +| sendGroupMessage:network:250 | 250 | 5 | 39 | 2/88 | 175 | 100.0% | PASS ✅ | +| receiveGroupMessage:operation:250 | 250 | 1 | 113 | 113/113 | 2000 | 100.0% | PASS ✅ | +| receiveGroupMessage:network:250 | 250 | 5 | 34 | 2/76 | 175 | 100.0% | PASS ✅ | +| createGroup:operation:300 | 300 | 1 | 4863 | 4863/4863 | 5500 | 100.0% | PASS ✅ | +| createGroup:network:300 | 300 | 5 | 90 | 16/145 | 175 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:operation:300 | 300 | 1 | 4524 | 4524/4524 | 6500 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network:300 | 300 | 5 | 35 | 3/78 | 175 | 100.0% | PASS ✅ | +| syncGroup:operation:300 | 300 | 1 | 90 | 90/90 | 80 | 0.0% | FAIL ❌ | +| syncGroup:network:300 | 300 | 5 | 37 | 2/81 | 175 | 100.0% | PASS ✅ | +| updateGroupName:operation:300 | 300 | 1 | 138 | 138/138 | 150 | 100.0% | PASS ✅ | +| updateGroupName:network:300 | 300 | 5 | 35 | 2/77 | 175 | 100.0% | PASS ✅ | +| removeMembers:operation:300 | 300 | 1 | 173 | 173/173 | 250 | 100.0% | PASS ✅ | +| sendGroupMessage:operation:300 | 300 | 1 | 53 | 53/53 | 4500 | 100.0% | PASS ✅ | +| removeMembers:network:300 | 300 | 5 | 33 | 2/73 | 175 | 100.0% | PASS ✅ | +| sendGroupMessage:network:300 | 300 | 5 | 39 | 3/87 | 175 | 100.0% | PASS ✅ | +| receiveGroupMessage:operation:300 | 300 | 1 | 105 | 105/105 | 2000 | 100.0% | PASS ✅ | +| receiveGroupMessage:network:300 | 300 | 5 | 34 | 2/74 | 175 | 100.0% | PASS ✅ | +| createGroup:operation:350 | 350 | 1 | 5080 | 5080/5080 | 8000 | 100.0% | PASS ✅ | +| createGroup:network:350 | 350 | 5 | 35 | 4/76 | 175 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:operation:350 | 350 | 1 | 5347 | 5347/5347 | 6500 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network:350 | 350 | 5 | 33 | 2/73 | 175 | 100.0% | PASS ✅ | +| syncGroup:operation:350 | 350 | 1 | 104 | 104/104 | 100 | 0.0% | FAIL ❌ | +| syncGroup:network:350 | 350 | 5 | 38 | 2/84 | 175 | 100.0% | PASS ✅ | +| updateGroupName:operation:350 | 350 | 1 | 147 | 147/147 | 150 | 100.0% | PASS ✅ | +| updateGroupName:network:350 | 350 | 5 | 35 | 3/77 | 175 | 100.0% | PASS ✅ | +| removeMembers:operation:350 | 350 | 1 | 202 | 202/202 | 250 | 100.0% | PASS ✅ | +| sendGroupMessage:operation:350 | 350 | 1 | 57 | 57/57 | 5000 | 100.0% | PASS ✅ | +| removeMembers:network:350 | 350 | 5 | 48 | 3/110 | 175 | 100.0% | PASS ✅ | +| sendGroupMessage:network:350 | 350 | 5 | 42 | 13/85 | 175 | 100.0% | PASS ✅ | +| receiveGroupMessage:operation:350 | 350 | 1 | 175 | 175/175 | 2000 | 100.0% | PASS ✅ | +| receiveGroupMessage:network:350 | 350 | 5 | 34 | 2/75 | 175 | 100.0% | PASS ✅ | +| createGroup:operation:400 | 400 | 1 | 6364 | 6364/6364 | 9000 | 100.0% | PASS ✅ | +| createGroup:network:400 | 400 | 5 | 38 | 2/82 | 175 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:operation:400 | 400 | 1 | 6192 | 6192/6192 | 7500 | 100.0% | PASS ✅ | +| createGroupByIdentifiers:network:400 | 400 | 5 | 36 | 3/80 | 175 | 100.0% | PASS ✅ | +| syncGroup:operation:400 | 400 | 1 | 103 | 103/103 | 120 | 100.0% | PASS ✅ | +| syncGroup:network:400 | 400 | 5 | 43 | 4/95 | 175 | 100.0% | PASS ✅ | +| updateGroupName:operation:400 | 400 | 1 | 174 | 174/174 | 200 | 100.0% | PASS ✅ | +| updateGroupName:network:400 | 400 | 5 | 36 | 2/80 | 175 | 100.0% | PASS ✅ | +| removeMembers:operation:400 | 400 | 1 | 235 | 235/235 | 250 | 100.0% | PASS ✅ | +| sendGroupMessage:operation:400 | 400 | 1 | 61 | 61/61 | 6000 | 100.0% | PASS ✅ | +| removeMembers:network:400 | 400 | 5 | 55 | 15/102 | 175 | 100.0% | PASS ✅ | +| sendGroupMessage:network:400 | 400 | 5 | 35 | 2/77 | 175 | 100.0% | PASS ✅ | +| receiveGroupMessage:operation:400 | 400 | 1 | 127 | 127/127 | 2000 | 100.0% | PASS ✅ | diff --git a/helpers/datadog.ts b/helpers/datadog.ts index 4f1048df..6f4bb6e4 100644 --- a/helpers/datadog.ts +++ b/helpers/datadog.ts @@ -68,27 +68,27 @@ const THRESHOLDS = { "400": 6000, }, syncgroup: { - "50": 50, - "100": 50, - "150": 50, - "200": 70, - "250": 80, - "300": 80, + "50": 100, + "100": 100, + "150": 100, + "200": 100, + "250": 100, + "300": 100, "350": 100, - "400": 120, + "400": 100, }, updategroupname: { - "50": 75, + "50": 100, "100": 80, "150": 100, "200": 100, - "250": 120, - "300": 150, - "350": 150, - "400": 200, + "250": 100, + "300": 100, + "350": 100, + "400": 100, }, removemembers: { - "50": 80, + "50": 100, "100": 100, "150": 120, "200": 150, @@ -442,9 +442,9 @@ export function logMetricsSummary(testName: string): void { // Build the table content let fileContent = "METRICS SUMMARY\n===============\n\n"; fileContent += - "Operation | Members | Samples | Avg (ms) | Min/Max (ms) | Threshold (ms) | Pass Rate | Status\n"; + "Operation | Members | Samples | Avg (ms) | Min/Max (ms) | Threshold (ms) | Variance (ms) | Pass Rate | Status\n"; fileContent += - "----------|---------|---------|----------|--------------|----------------|-----------|-------\n"; + "----------|---------|---------|----------|--------------|----------------|---------------|-----------|-------\n"; for (const [operation, data] of Object.entries(collectedMetrics)) { // Skip workflow metrics in the summary table @@ -467,7 +467,13 @@ export function logMetricsSummary(testName: string): void { data.values.length) * 100; const status = average <= data.threshold ? "PASS ✅" : "FAIL ❌"; - fileContent += `${operation} | ${memberCount} | ${data.values.length} | ${Math.round(average)} | ${Math.round(min)}/${Math.round(max)} | ${data.threshold} | ${passRate.toFixed(1)}% | ${status}\n`; + + // Calculate variance between average and threshold + const variance = Math.round(average - data.threshold); + const varianceFormatted = + variance <= 0 ? variance.toString() : `+${variance}`; + + fileContent += `${operation} | ${memberCount} | ${data.values.length} | ${Math.round(average)} | ${Math.round(min)}/${Math.round(max)} | ${data.threshold} | ${varianceFormatted} | ${passRate.toFixed(1)}% | ${status}\n`; } // Write to file From ebc01ff7ef56b54ac7ed24d9432c01d45800e93c Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 16:51:50 -0300 Subject: [PATCH 16/57] artifact locations --- .github/workflows/TS_Performance.yml | 8 +- ...roduction.md.run-13931915774-production.md | 55 ------- ...roduction.md.run-13931915774-production.md | 139 ------------------ ...roduction.md.run-13932078204-production.md | 139 ------------------ helpers/datadog.ts | 2 +- 5 files changed, 5 insertions(+), 338 deletions(-) delete mode 100644 datadog/reports/ts_performance-south-america-production.md.run-13931915774-production.md delete mode 100644 datadog/reports/ts_performance-us-east-production.md.run-13931915774-production.md delete mode 100644 datadog/reports/ts_performance-us-east-production.md.run-13932078204-production.md diff --git a/.github/workflows/TS_Performance.yml b/.github/workflows/TS_Performance.yml index cd5e2129..999c6a12 100644 --- a/.github/workflows/TS_Performance.yml +++ b/.github/workflows/TS_Performance.yml @@ -38,10 +38,10 @@ jobs: cp -r logs/* artifacts/logs/ find artifacts/logs -type f -name "*.log" -exec bash -c 'mv "$0" "${0%.log}.run-${{ github.run_id }}-dev.txt"' {} \; find artifacts/logs -type f ! -name "*.run-*" -exec bash -c 'mv "$0" "$0.run-${{ github.run_id }}-dev.txt"' {} \; - - name: Rename datadog reports with run ID prefix + - name: Rename logs/reports with run ID prefix run: | mkdir -p artifacts/reports - cp -r datadog/reports/* artifacts/reports/ + cp -r logs/reports/* artifacts/reports/ find artifacts/reports -type f -name "*.md" -exec bash -c 'mv "$0" "${0%.md}.run-${{ github.run_id }}-dev.md"' {} \; find artifacts/reports -type f ! -name "*.run-*" -exec bash -c 'mv "$0" "$0.run-${{ github.run_id }}-dev.md"' {} \; - name: Upload all artifacts @@ -78,10 +78,10 @@ jobs: cp -r logs/* artifacts/logs/ find artifacts/logs -type f -name "*.log" -exec bash -c 'mv "$0" "${0%.log}.run-${{ github.run_id }}-production.txt"' {} \; find artifacts/logs -type f ! -name "*.run-*" -exec bash -c 'mv "$0" "$0.run-${{ github.run_id }}-production.txt"' {} \; - - name: Rename datadog reports with run ID prefix + - name: Rename logs/reports with run ID prefix run: | mkdir -p artifacts/reports - cp -r datadog/reports/* artifacts/reports/ + cp -r logs/reports/* artifacts/reports/ find artifacts/reports -type f -name "*.md" -exec bash -c 'mv "$0" "${0%.md}.run-${{ github.run_id }}-production.md"' {} \; find artifacts/reports -type f ! -name "*.run-*" -exec bash -c 'mv "$0" "$0.run-${{ github.run_id }}-production.md"' {} \; - name: Upload all artifacts diff --git a/datadog/reports/ts_performance-south-america-production.md.run-13931915774-production.md b/datadog/reports/ts_performance-south-america-production.md.run-13931915774-production.md deleted file mode 100644 index 2189345f..00000000 --- a/datadog/reports/ts_performance-south-america-production.md.run-13931915774-production.md +++ /dev/null @@ -1,55 +0,0 @@ -# METRICS SUMMARY - -| Operation | Members | Samples | Avg (ms) | Min/Max (ms) | Threshold (ms) | Pass Rate | Status | -| ------------------------------------ | ------- | ------- | -------- | ------------ | -------------- | --------- | ------- | -| clientCreate:operation | - | 1 | 4173 | 4173/4173 | 700 | 0.0% | FAIL ❌ | -| inboxState:operation | - | 1 | 198 | 198/198 | 100 | 0.0% | FAIL ❌ | -| clientCreate:network | - | 5 | 424 | 51/1008 | 100 | 40.0% | FAIL ❌ | -| inboxState:network | - | 5 | 326 | 2/812 | 100 | 40.0% | FAIL ❌ | -| createDM:operation | - | 1 | 2131 | 2131/2131 | 300 | 0.0% | FAIL ❌ | -| createDM:network | - | 5 | 330 | 2/823 | 100 | 40.0% | FAIL ❌ | -| sendGM:operation | - | 1 | 1210 | 1210/1210 | 200 | 0.0% | FAIL ❌ | -| receiveGM:operation | - | 1 | 615 | 615/615 | 200 | 0.0% | FAIL ❌ | -| sendGM:network | - | 5 | 326 | 3/813 | 100 | 40.0% | FAIL ❌ | -| receiveGM:network | - | 5 | 319 | 3/795 | 100 | 40.0% | FAIL ❌ | -| createGroup:operation | - | 1 | 1873 | 1873/1873 | 4000 | 100.0% | PASS ✅ | -| createGroup:network | - | 5 | 378 | 2/944 | 100 | 40.0% | FAIL ❌ | -| createGroupByIdentifiers:operation | - | 1 | 1911 | 1911/1911 | 4000 | 100.0% | PASS ✅ | -| syncGroup:operation | - | 1 | 729 | 729/729 | 4000 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network | - | 5 | 319 | 11/785 | 100 | 40.0% | FAIL ❌ | -| syncGroup:network | - | 5 | 299 | 3/745 | 100 | 40.0% | FAIL ❌ | -| updateGroupName:operation | - | 1 | 1020 | 1020/1020 | 4000 | 100.0% | PASS ✅ | -| updateGroupName:network | - | 5 | 386 | 4/962 | 100 | 40.0% | FAIL ❌ | -| removeMembers:operation | - | 1 | 1035 | 1035/1035 | 200 | 0.0% | FAIL ❌ | -| sendGroupMessage:operation | - | 1 | 621 | 621/621 | 4000 | 100.0% | PASS ✅ | -| removeMembers:network | - | 5 | 323 | 6/802 | 100 | 40.0% | FAIL ❌ | -| sendGroupMessage:network | - | 5 | 391 | 2/975 | 100 | 40.0% | FAIL ❌ | -| receiveGroupMessage:operation | - | 1 | 1213 | 1213/1213 | 4000 | 100.0% | PASS ✅ | -| receiveGroupMessage:network | - | 5 | 382 | 2/953 | 100 | 40.0% | FAIL ❌ | -| createGroup:operation:4 | 4 | 1 | 1879 | 1879/1879 | 4000 | 100.0% | PASS ✅ | -| createGroup:network:4 | 4 | 5 | 384 | 4/955 | 100 | 40.0% | FAIL ❌ | -| createGroupByIdentifiers:operation:4 | 4 | 1 | 2052 | 2052/2052 | 4000 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network:4 | 4 | 5 | 316 | 2/789 | 100 | 40.0% | FAIL ❌ | -| syncGroup:operation:4 | 4 | 1 | 811 | 811/811 | 4000 | 100.0% | PASS ✅ | -| syncGroup:network:4 | 4 | 5 | 306 | 4/760 | 100 | 40.0% | FAIL ❌ | -| updateGroupName:operation:4 | 4 | 1 | 1041 | 1041/1041 | 4000 | 100.0% | PASS ✅ | -| updateGroupName:network:4 | 4 | 5 | 312 | 2/779 | 100 | 40.0% | FAIL ❌ | -| removeMembers:operation:4 | 4 | 1 | 822 | 822/822 | 200 | 0.0% | FAIL ❌ | -| sendGroupMessage:operation:4 | 4 | 1 | 795 | 795/795 | 4000 | 100.0% | PASS ✅ | -| removeMembers:network:4 | 4 | 5 | 378 | 2/943 | 100 | 40.0% | FAIL ❌ | -| sendGroupMessage:network:4 | 4 | 5 | 310 | 3/773 | 100 | 40.0% | FAIL ❌ | -| receiveGroupMessage:operation:4 | 4 | 1 | 1211 | 1211/1211 | 4000 | 100.0% | PASS ✅ | -| receiveGroupMessage:network:4 | 4 | 5 | 307 | 2/766 | 100 | 40.0% | FAIL ❌ | -| createGroup:operation:8 | 8 | 1 | 1846 | 1846/1846 | 4000 | 100.0% | PASS ✅ | -| createGroup:network:8 | 8 | 5 | 307 | 6/761 | 100 | 40.0% | FAIL ❌ | -| createGroupByIdentifiers:operation:8 | 8 | 1 | 1903 | 1903/1903 | 4000 | 100.0% | PASS ✅ | -| syncGroup:operation:8 | 8 | 1 | 736 | 736/736 | 4000 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network:8 | 8 | 5 | 318 | 2/794 | 100 | 40.0% | FAIL ❌ | -| syncGroup:network:8 | 8 | 5 | 301 | 2/750 | 100 | 40.0% | FAIL ❌ | -| updateGroupName:operation:8 | 8 | 1 | 1056 | 1056/1056 | 4000 | 100.0% | PASS ✅ | -| updateGroupName:network:8 | 8 | 5 | 381 | 2/951 | 100 | 40.0% | FAIL ❌ | -| removeMembers:operation:8 | 8 | 1 | 1047 | 1047/1047 | 200 | 0.0% | FAIL ❌ | -| removeMembers:network:8 | 8 | 5 | 305 | 2/761 | 100 | 40.0% | FAIL ❌ | -| sendGroupMessage:operation:8 | 8 | 1 | 824 | 824/824 | 4000 | 100.0% | PASS ✅ | -| sendGroupMessage:network:8 | 8 | 5 | 303 | 2/757 | 100 | 40.0% | FAIL ❌ | -| receiveGroupMessage:operation:8 | 8 | 1 | 1038 | 1038/1038 | 4000 | 100.0% | PASS ✅ | diff --git a/datadog/reports/ts_performance-us-east-production.md.run-13931915774-production.md b/datadog/reports/ts_performance-us-east-production.md.run-13931915774-production.md deleted file mode 100644 index 0ca2f5c1..00000000 --- a/datadog/reports/ts_performance-us-east-production.md.run-13931915774-production.md +++ /dev/null @@ -1,139 +0,0 @@ -# METRICS SUMMARY - -| Operation | Members | Samples | Avg (ms) | Min/Max (ms) | Threshold (ms) | Pass Rate | Status | -| -------------------------------------- | ------- | ------- | -------- | ------------ | -------------- | --------- | ------- | -| clientCreate:operation | - | 1 | 502 | 502/502 | 350 | 0.0% | FAIL ❌ | -| inboxState:operation | - | 1 | 27 | 27/27 | 50 | 100.0% | PASS ✅ | -| inboxState:network | - | 5 | 102 | 22/174 | 50 | 20.0% | FAIL ❌ | -| clientCreate:network | - | 5 | 131 | 22/210 | 50 | 20.0% | FAIL ❌ | -| createDM:operation | - | 1 | 195 | 195/195 | 150 | 0.0% | FAIL ❌ | -| sendGM:operation | - | 1 | 87 | 87/87 | 100 | 100.0% | PASS ✅ | -| receiveGM:operation | - | 1 | 62 | 62/62 | 100 | 100.0% | PASS ✅ | -| createDM:network | - | 5 | 98 | 22/163 | 50 | 20.0% | FAIL ❌ | -| sendGM:network | - | 5 | 43 | 2/95 | 50 | 60.0% | PASS ✅ | -| receiveGM:network | - | 5 | 95 | 22/159 | 50 | 20.0% | FAIL ❌ | -| createGroup:operation | - | 1 | 225 | 225/225 | 2000 | 100.0% | PASS ✅ | -| createGroup:network | - | 5 | 95 | 23/159 | 50 | 20.0% | FAIL ❌ | -| createGroupByIdentifiers:operation | - | 1 | 232 | 232/232 | 2000 | 100.0% | PASS ✅ | -| syncGroup:operation | - | 1 | 54 | 54/54 | 2000 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network | - | 5 | 66 | 22/124 | 50 | 40.0% | FAIL ❌ | -| updateGroupName:operation | - | 1 | 102 | 102/102 | 2000 | 100.0% | PASS ✅ | -| syncGroup:network | - | 5 | 98 | 23/164 | 50 | 20.0% | FAIL ❌ | -| removeMembers:operation | - | 1 | 93 | 93/93 | 100 | 100.0% | PASS ✅ | -| updateGroupName:network | - | 5 | 52 | 14/106 | 50 | 60.0% | FAIL ❌ | -| sendGroupMessage:operation | - | 1 | 64 | 64/64 | 2000 | 100.0% | PASS ✅ | -| removeMembers:network | - | 5 | 65 | 22/124 | 50 | 40.0% | FAIL ❌ | -| receiveGroupMessage:operation | - | 1 | 91 | 91/91 | 2000 | 100.0% | PASS ✅ | -| sendGroupMessage:network | - | 5 | 51 | 13/103 | 50 | 60.0% | FAIL ❌ | -| receiveGroupMessage:network | - | 5 | 69 | 22/132 | 50 | 40.0% | FAIL ❌ | -| createGroup:operation:50 | 50 | 1 | 837 | 837/837 | 2000 | 100.0% | PASS ✅ | -| createGroup:network:50 | 50 | 5 | 44 | 2/97 | 50 | 60.0% | PASS ✅ | -| createGroupByIdentifiers:operation:50 | 50 | 1 | 558 | 558/558 | 2000 | 100.0% | PASS ✅ | -| syncGroup:operation:50 | 50 | 1 | 63 | 63/63 | 2000 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network:50 | 50 | 5 | 52 | 14/106 | 50 | 60.0% | FAIL ❌ | -| updateGroupName:operation:50 | 50 | 1 | 102 | 102/102 | 2000 | 100.0% | PASS ✅ | -| syncGroup:network:50 | 50 | 5 | 51 | 13/104 | 50 | 60.0% | FAIL ❌ | -| updateGroupName:network:50 | 50 | 5 | 45 | 2/99 | 50 | 60.0% | PASS ✅ | -| removeMembers:operation:50 | 50 | 1 | 122 | 122/122 | 100 | 0.0% | FAIL ❌ | -| sendGroupMessage:operation:50 | 50 | 1 | 63 | 63/63 | 2000 | 100.0% | PASS ✅ | -| removeMembers:network:50 | 50 | 5 | 51 | 13/104 | 50 | 60.0% | FAIL ❌ | -| receiveGroupMessage:operation:50 | 50 | 1 | 98 | 98/98 | 2000 | 100.0% | PASS ✅ | -| sendGroupMessage:network:50 | 50 | 5 | 43 | 2/94 | 50 | 60.0% | PASS ✅ | -| receiveGroupMessage:network:50 | 50 | 5 | 56 | 13/116 | 50 | 60.0% | FAIL ❌ | -| createGroup:operation:100 | 100 | 1 | 1577 | 1577/1577 | 2000 | 100.0% | PASS ✅ | -| createGroup:network:100 | 100 | 5 | 91 | 22/153 | 50 | 20.0% | FAIL ❌ | -| createGroupByIdentifiers:operation:100 | 100 | 1 | 1399 | 1399/1399 | 2000 | 100.0% | PASS ✅ | -| syncGroup:operation:100 | 100 | 1 | 65 | 65/65 | 2000 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network:100 | 100 | 5 | 54 | 14/109 | 50 | 60.0% | FAIL ❌ | -| syncGroup:network:100 | 100 | 5 | 45 | 4/97 | 50 | 60.0% | PASS ✅ | -| updateGroupName:operation:100 | 100 | 1 | 109 | 109/109 | 2000 | 100.0% | PASS ✅ | -| updateGroupName:network:100 | 100 | 5 | 45 | 2/98 | 50 | 60.0% | PASS ✅ | -| removeMembers:operation:100 | 100 | 1 | 120 | 120/120 | 100 | 0.0% | FAIL ❌ | -| sendGroupMessage:operation:100 | 100 | 1 | 62 | 62/62 | 2000 | 100.0% | PASS ✅ | -| receiveGroupMessage:operation:100 | 100 | 1 | 102 | 102/102 | 2000 | 100.0% | PASS ✅ | -| sendGroupMessage:network:100 | 100 | 5 | 45 | 2/101 | 50 | 60.0% | PASS ✅ | -| removeMembers:network:100 | 100 | 5 | 95 | 23/164 | 50 | 20.0% | FAIL ❌ | -| receiveGroupMessage:network:100 | 100 | 5 | 52 | 13/105 | 50 | 60.0% | FAIL ❌ | -| createGroup:operation:150 | 150 | 1 | 2266 | 2266/2266 | 2000 | 0.0% | FAIL ❌ | -| createGroup:network:150 | 150 | 5 | 55 | 16/110 | 50 | 60.0% | FAIL ❌ | -| createGroupByIdentifiers:operation:150 | 150 | 1 | 2562 | 2562/2562 | 2000 | 0.0% | FAIL ❌ | -| syncGroup:operation:150 | 150 | 1 | 88 | 88/88 | 2000 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network:150 | 150 | 5 | 43 | 2/94 | 50 | 60.0% | PASS ✅ | -| syncGroup:network:150 | 150 | 5 | 44 | 4/94 | 50 | 60.0% | PASS ✅ | -| updateGroupName:operation:150 | 150 | 1 | 122 | 122/122 | 2000 | 100.0% | PASS ✅ | -| updateGroupName:network:150 | 150 | 5 | 46 | 3/101 | 50 | 60.0% | PASS ✅ | -| removeMembers:operation:150 | 150 | 1 | 139 | 139/139 | 100 | 0.0% | FAIL ❌ | -| sendGroupMessage:operation:150 | 150 | 1 | 64 | 64/64 | 2000 | 100.0% | PASS ✅ | -| removeMembers:network:150 | 150 | 5 | 94 | 22/162 | 50 | 20.0% | FAIL ❌ | -| sendGroupMessage:network:150 | 150 | 5 | 43 | 2/95 | 50 | 60.0% | PASS ✅ | -| receiveGroupMessage:operation:150 | 150 | 1 | 113 | 113/113 | 2000 | 100.0% | PASS ✅ | -| receiveGroupMessage:network:150 | 150 | 5 | 45 | 2/98 | 50 | 60.0% | PASS ✅ | -| createGroup:operation:200 | 200 | 1 | 3306 | 3306/3306 | 2000 | 0.0% | FAIL ❌ | -| createGroup:network:200 | 200 | 5 | 44 | 3/96 | 50 | 60.0% | PASS ✅ | -| createGroupByIdentifiers:operation:200 | 200 | 1 | 3394 | 3394/3394 | 2000 | 0.0% | FAIL ❌ | -| syncGroup:operation:200 | 200 | 1 | 97 | 97/97 | 2000 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network:200 | 200 | 5 | 50 | 10/103 | 50 | 60.0% | PASS ✅ | -| updateGroupName:operation:200 | 200 | 1 | 137 | 137/137 | 2000 | 100.0% | PASS ✅ | -| syncGroup:network:200 | 200 | 5 | 93 | 23/159 | 50 | 20.0% | FAIL ❌ | -| removeMembers:operation:200 | 200 | 1 | 161 | 161/161 | 100 | 0.0% | FAIL ❌ | -| updateGroupName:network:200 | 200 | 5 | 98 | 22/164 | 50 | 20.0% | FAIL ❌ | -| sendGroupMessage:operation:200 | 200 | 1 | 67 | 67/67 | 2000 | 100.0% | PASS ✅ | -| removeMembers:network:200 | 200 | 5 | 43 | 2/94 | 50 | 60.0% | PASS ✅ | -| sendGroupMessage:network:200 | 200 | 5 | 44 | 2/97 | 50 | 60.0% | PASS ✅ | -| receiveGroupMessage:operation:200 | 200 | 1 | 129 | 129/129 | 2000 | 100.0% | PASS ✅ | -| receiveGroupMessage:network:200 | 200 | 5 | 43 | 3/94 | 50 | 60.0% | PASS ✅ | -| createGroup:operation:250 | 250 | 1 | 5425 | 5425/5425 | 2000 | 0.0% | FAIL ❌ | -| createGroup:network:250 | 250 | 5 | 43 | 3/94 | 50 | 60.0% | PASS ✅ | -| createGroupByIdentifiers:operation:250 | 250 | 1 | 5085 | 5085/5085 | 2000 | 0.0% | FAIL ❌ | -| syncGroup:operation:250 | 250 | 1 | 102 | 102/102 | 2000 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network:250 | 250 | 5 | 43 | 2/95 | 50 | 60.0% | PASS ✅ | -| syncGroup:network:250 | 250 | 5 | 42 | 2/93 | 50 | 60.0% | PASS ✅ | -| updateGroupName:operation:250 | 250 | 1 | 161 | 161/161 | 2000 | 100.0% | PASS ✅ | -| updateGroupName:network:250 | 250 | 5 | 46 | 3/100 | 50 | 60.0% | PASS ✅ | -| removeMembers:operation:250 | 250 | 1 | 200 | 200/200 | 100 | 0.0% | FAIL ❌ | -| sendGroupMessage:operation:250 | 250 | 1 | 72 | 72/72 | 2000 | 100.0% | PASS ✅ | -| removeMembers:network:250 | 250 | 5 | 44 | 2/98 | 50 | 60.0% | PASS ✅ | -| sendGroupMessage:network:250 | 250 | 5 | 46 | 2/102 | 50 | 60.0% | PASS ✅ | -| receiveGroupMessage:operation:250 | 250 | 1 | 132 | 132/132 | 2000 | 100.0% | PASS ✅ | -| receiveGroupMessage:network:250 | 250 | 5 | 43 | 2/94 | 50 | 60.0% | PASS ✅ | -| createGroup:operation:300 | 300 | 1 | 5392 | 5392/5392 | 2000 | 0.0% | FAIL ❌ | -| createGroup:network:300 | 300 | 5 | 45 | 3/97 | 50 | 60.0% | PASS ✅ | -| createGroupByIdentifiers:operation:300 | 300 | 1 | 6466 | 6466/6466 | 2000 | 0.0% | FAIL ❌ | -| createGroupByIdentifiers:network:300 | 300 | 5 | 44 | 3/95 | 50 | 60.0% | PASS ✅ | -| syncGroup:operation:300 | 300 | 1 | 114 | 114/114 | 2000 | 100.0% | PASS ✅ | -| syncGroup:network:300 | 300 | 5 | 55 | 2/124 | 50 | 60.0% | FAIL ❌ | -| updateGroupName:operation:300 | 300 | 1 | 177 | 177/177 | 2000 | 100.0% | PASS ✅ | -| updateGroupName:network:300 | 300 | 5 | 43 | 2/95 | 50 | 60.0% | PASS ✅ | -| removeMembers:operation:300 | 300 | 1 | 200 | 200/200 | 100 | 0.0% | FAIL ❌ | -| sendGroupMessage:operation:300 | 300 | 1 | 72 | 72/72 | 2000 | 100.0% | PASS ✅ | -| removeMembers:network:300 | 300 | 5 | 44 | 3/96 | 50 | 60.0% | PASS ✅ | -| sendGroupMessage:network:300 | 300 | 5 | 43 | 2/94 | 50 | 60.0% | PASS ✅ | -| receiveGroupMessage:operation:300 | 300 | 1 | 136 | 136/136 | 2000 | 100.0% | PASS ✅ | -| receiveGroupMessage:network:300 | 300 | 5 | 56 | 13/116 | 50 | 60.0% | FAIL ❌ | -| createGroup:operation:350 | 350 | 1 | 7734 | 7734/7734 | 2000 | 0.0% | FAIL ❌ | -| createGroup:network:350 | 350 | 5 | 44 | 2/96 | 50 | 60.0% | PASS ✅ | -| createGroupByIdentifiers:operation:350 | 350 | 1 | 6067 | 6067/6067 | 2000 | 0.0% | FAIL ❌ | -| createGroupByIdentifiers:network:350 | 350 | 5 | 43 | 3/95 | 50 | 60.0% | PASS ✅ | -| syncGroup:operation:350 | 350 | 1 | 121 | 121/121 | 2000 | 100.0% | PASS ✅ | -| syncGroup:network:350 | 350 | 5 | 49 | 10/102 | 50 | 60.0% | PASS ✅ | -| updateGroupName:operation:350 | 350 | 1 | 175 | 175/175 | 2000 | 100.0% | PASS ✅ | -| updateGroupName:network:350 | 350 | 5 | 46 | 2/102 | 50 | 60.0% | PASS ✅ | -| removeMembers:operation:350 | 350 | 1 | 234 | 234/234 | 100 | 0.0% | FAIL ❌ | -| sendGroupMessage:operation:350 | 350 | 1 | 76 | 76/76 | 2000 | 100.0% | PASS ✅ | -| removeMembers:network:350 | 350 | 5 | 43 | 2/95 | 50 | 60.0% | PASS ✅ | -| sendGroupMessage:network:350 | 350 | 5 | 45 | 2/99 | 50 | 60.0% | PASS ✅ | -| receiveGroupMessage:operation:350 | 350 | 1 | 150 | 150/150 | 2000 | 100.0% | PASS ✅ | -| receiveGroupMessage:network:350 | 350 | 5 | 44 | 3/95 | 50 | 60.0% | PASS ✅ | -| createGroup:operation:400 | 400 | 1 | 6742 | 6742/6742 | 2000 | 0.0% | FAIL ❌ | -| createGroup:network:400 | 400 | 5 | 163 | 23/245 | 50 | 20.0% | FAIL ❌ | -| createGroupByIdentifiers:operation:400 | 400 | 1 | 6513 | 6513/6513 | 2000 | 0.0% | FAIL ❌ | -| createGroupByIdentifiers:network:400 | 400 | 5 | 44 | 3/95 | 50 | 60.0% | PASS ✅ | -| syncGroup:operation:400 | 400 | 1 | 131 | 131/131 | 2000 | 100.0% | PASS ✅ | -| syncGroup:network:400 | 400 | 5 | 56 | 17/112 | 50 | 60.0% | FAIL ❌ | -| updateGroupName:operation:400 | 400 | 1 | 191 | 191/191 | 2000 | 100.0% | PASS ✅ | -| updateGroupName:network:400 | 400 | 5 | 59 | 22/114 | 50 | 60.0% | FAIL ❌ | -| removeMembers:operation:400 | 400 | 1 | 269 | 269/269 | 100 | 0.0% | FAIL ❌ | -| sendGroupMessage:operation:400 | 400 | 1 | 82 | 82/82 | 2000 | 100.0% | PASS ✅ | -| removeMembers:network:400 | 400 | 5 | 98 | 22/164 | 50 | 20.0% | FAIL ❌ | -| sendGroupMessage:network:400 | 400 | 5 | 43 | 3/95 | 50 | 60.0% | PASS ✅ | -| receiveGroupMessage:operation:400 | 400 | 1 | 158 | 158/158 | 2000 | 100.0% | PASS ✅ | diff --git a/datadog/reports/ts_performance-us-east-production.md.run-13932078204-production.md b/datadog/reports/ts_performance-us-east-production.md.run-13932078204-production.md deleted file mode 100644 index 4c3d2938..00000000 --- a/datadog/reports/ts_performance-us-east-production.md.run-13932078204-production.md +++ /dev/null @@ -1,139 +0,0 @@ -# METRICS SUMMARY - -| Operation | Members | Samples | Avg (ms) | Min/Max (ms) | Threshold (ms) | Pass Rate | Status | -| -------------------------------------- | ------- | ------- | -------- | ------------ | -------------- | --------- | ------- | -| clientCreate:operation | - | 1 | 355 | 355/355 | 550 | 100.0% | PASS ✅ | -| inboxState:operation | - | 1 | 19 | 19/19 | 50 | 100.0% | PASS ✅ | -| createDM:operation | - | 1 | 131 | 131/131 | 200 | 100.0% | PASS ✅ | -| inboxState:network | - | 5 | 82 | 16/135 | 175 | 100.0% | PASS ✅ | -| clientCreate:network | - | 5 | 106 | 16/166 | 175 | 100.0% | PASS ✅ | -| sendGM:operation | - | 1 | 59 | 59/59 | 100 | 100.0% | PASS ✅ | -| createDM:network | - | 5 | 33 | 2/73 | 175 | 100.0% | PASS ✅ | -| receiveGM:operation | - | 1 | 47 | 47/47 | 100 | 100.0% | PASS ✅ | -| sendGM:network | - | 5 | 55 | 15/102 | 175 | 100.0% | PASS ✅ | -| receiveGM:network | - | 5 | 56 | 15/102 | 175 | 100.0% | PASS ✅ | -| createGroup:operation | - | 1 | 149 | 149/149 | 900 | 100.0% | PASS ✅ | -| createGroup:network | - | 5 | 48 | 15/96 | 175 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:operation | - | 1 | 162 | 162/162 | 600 | 100.0% | PASS ✅ | -| syncGroup:operation | - | 1 | 36 | 36/36 | 50 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network | - | 5 | 58 | 15/104 | 175 | 100.0% | PASS ✅ | -| updateGroupName:operation | - | 1 | 67 | 67/67 | 75 | 100.0% | PASS ✅ | -| syncGroup:network | - | 5 | 89 | 16/144 | 175 | 100.0% | PASS ✅ | -| removeMembers:operation | - | 1 | 79 | 79/79 | 250 | 100.0% | PASS ✅ | -| updateGroupName:network | - | 5 | 33 | 2/74 | 175 | 100.0% | PASS ✅ | -| sendGroupMessage:operation | - | 1 | 79 | 79/79 | 650 | 100.0% | PASS ✅ | -| removeMembers:network | - | 5 | 43 | 9/90 | 175 | 100.0% | PASS ✅ | -| receiveGroupMessage:operation | - | 1 | 70 | 70/70 | 2000 | 100.0% | PASS ✅ | -| sendGroupMessage:network | - | 5 | 48 | 13/99 | 175 | 100.0% | PASS ✅ | -| receiveGroupMessage:network | - | 5 | 59 | 16/110 | 175 | 100.0% | PASS ✅ | -| createGroup:operation:50 | 50 | 1 | 2148 | 2148/2148 | 900 | 0.0% | FAIL ❌ | -| createGroup:network:50 | 50 | 5 | 38 | 9/79 | 175 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:operation:50 | 50 | 1 | 499 | 499/499 | 600 | 100.0% | PASS ✅ | -| syncGroup:operation:50 | 50 | 1 | 41 | 41/41 | 50 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network:50 | 50 | 5 | 35 | 3/76 | 175 | 100.0% | PASS ✅ | -| updateGroupName:operation:50 | 50 | 1 | 82 | 82/82 | 75 | 0.0% | FAIL ❌ | -| syncGroup:network:50 | 50 | 5 | 59 | 16/106 | 175 | 100.0% | PASS ✅ | -| removeMembers:operation:50 | 50 | 1 | 77 | 77/77 | 250 | 100.0% | PASS ✅ | -| updateGroupName:network:50 | 50 | 5 | 41 | 13/82 | 175 | 100.0% | PASS ✅ | -| sendGroupMessage:operation:50 | 50 | 1 | 43 | 43/43 | 650 | 100.0% | PASS ✅ | -| removeMembers:network:50 | 50 | 5 | 35 | 2/77 | 175 | 100.0% | PASS ✅ | -| receiveGroupMessage:operation:50 | 50 | 1 | 69 | 69/69 | 2000 | 100.0% | PASS ✅ | -| sendGroupMessage:network:50 | 50 | 5 | 61 | 15/110 | 175 | 100.0% | PASS ✅ | -| receiveGroupMessage:network:50 | 50 | 5 | 46 | 13/94 | 175 | 100.0% | PASS ✅ | -| createGroup:operation:100 | 100 | 1 | 1317 | 1317/1317 | 1600 | 100.0% | PASS ✅ | -| createGroup:network:100 | 100 | 5 | 34 | 3/74 | 175 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:operation:100 | 100 | 1 | 977 | 977/977 | 1500 | 100.0% | PASS ✅ | -| syncGroup:operation:100 | 100 | 1 | 50 | 50/50 | 50 | 0.0% | FAIL ❌ | -| createGroupByIdentifiers:network:100 | 100 | 5 | 35 | 2/78 | 175 | 100.0% | PASS ✅ | -| updateGroupName:operation:100 | 100 | 1 | 80 | 80/80 | 80 | 100.0% | PASS ✅ | -| syncGroup:network:100 | 100 | 5 | 35 | 4/77 | 175 | 100.0% | PASS ✅ | -| updateGroupName:network:100 | 100 | 5 | 35 | 2/77 | 175 | 100.0% | PASS ✅ | -| removeMembers:operation:100 | 100 | 1 | 94 | 94/94 | 250 | 100.0% | PASS ✅ | -| sendGroupMessage:operation:100 | 100 | 1 | 50 | 50/50 | 1000 | 100.0% | PASS ✅ | -| removeMembers:network:100 | 100 | 5 | 35 | 2/78 | 175 | 100.0% | PASS ✅ | -| receiveGroupMessage:operation:100 | 100 | 1 | 71 | 71/71 | 2000 | 100.0% | PASS ✅ | -| sendGroupMessage:network:100 | 100 | 5 | 34 | 2/74 | 175 | 100.0% | PASS ✅ | -| receiveGroupMessage:network:100 | 100 | 5 | 33 | 2/74 | 175 | 100.0% | PASS ✅ | -| createGroup:operation:150 | 150 | 1 | 1730 | 1730/1730 | 2300 | 100.0% | PASS ✅ | -| createGroup:network:150 | 150 | 5 | 33 | 2/73 | 175 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:operation:150 | 150 | 1 | 1688 | 1688/1688 | 2600 | 100.0% | PASS ✅ | -| syncGroup:operation:150 | 150 | 1 | 58 | 58/58 | 50 | 0.0% | FAIL ❌ | -| createGroupByIdentifiers:network:150 | 150 | 5 | 46 | 14/92 | 175 | 100.0% | PASS ✅ | -| syncGroup:network:150 | 150 | 5 | 36 | 6/77 | 175 | 100.0% | PASS ✅ | -| updateGroupName:operation:150 | 150 | 1 | 88 | 88/88 | 100 | 100.0% | PASS ✅ | -| updateGroupName:network:150 | 150 | 5 | 34 | 2/76 | 175 | 100.0% | PASS ✅ | -| removeMembers:operation:150 | 150 | 1 | 108 | 108/108 | 250 | 100.0% | PASS ✅ | -| sendGroupMessage:operation:150 | 150 | 1 | 48 | 48/48 | 1600 | 100.0% | PASS ✅ | -| removeMembers:network:150 | 150 | 5 | 33 | 2/72 | 175 | 100.0% | PASS ✅ | -| receiveGroupMessage:operation:150 | 150 | 1 | 83 | 83/83 | 2000 | 100.0% | PASS ✅ | -| sendGroupMessage:network:150 | 150 | 5 | 42 | 13/85 | 175 | 100.0% | PASS ✅ | -| receiveGroupMessage:network:150 | 150 | 5 | 44 | 2/101 | 175 | 100.0% | PASS ✅ | -| createGroup:operation:200 | 200 | 1 | 2582 | 2582/2582 | 3500 | 100.0% | PASS ✅ | -| createGroup:network:200 | 200 | 5 | 33 | 3/73 | 175 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:operation:200 | 200 | 1 | 2627 | 2627/2627 | 3500 | 100.0% | PASS ✅ | -| syncGroup:operation:200 | 200 | 1 | 75 | 75/75 | 70 | 0.0% | FAIL ❌ | -| createGroupByIdentifiers:network:200 | 200 | 5 | 36 | 4/79 | 175 | 100.0% | PASS ✅ | -| syncGroup:network:200 | 200 | 5 | 37 | 8/77 | 175 | 100.0% | PASS ✅ | -| updateGroupName:operation:200 | 200 | 1 | 99 | 99/99 | 100 | 100.0% | PASS ✅ | -| updateGroupName:network:200 | 200 | 5 | 34 | 2/76 | 175 | 100.0% | PASS ✅ | -| removeMembers:operation:200 | 200 | 1 | 137 | 137/137 | 250 | 100.0% | PASS ✅ | -| sendGroupMessage:operation:200 | 200 | 1 | 48 | 48/48 | 2500 | 100.0% | PASS ✅ | -| receiveGroupMessage:operation:200 | 200 | 1 | 85 | 85/85 | 2000 | 100.0% | PASS ✅ | -| sendGroupMessage:network:200 | 200 | 5 | 44 | 8/95 | 175 | 100.0% | PASS ✅ | -| removeMembers:network:200 | 200 | 5 | 91 | 15/154 | 175 | 100.0% | PASS ✅ | -| receiveGroupMessage:network:200 | 200 | 5 | 38 | 2/85 | 175 | 100.0% | PASS ✅ | -| createGroup:operation:250 | 250 | 1 | 3625 | 3625/3625 | 5500 | 100.0% | PASS ✅ | -| createGroup:network:250 | 250 | 5 | 33 | 2/72 | 175 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:operation:250 | 250 | 1 | 3566 | 3566/3566 | 5500 | 100.0% | PASS ✅ | -| syncGroup:operation:250 | 250 | 1 | 77 | 77/77 | 80 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network:250 | 250 | 5 | 36 | 3/78 | 175 | 100.0% | PASS ✅ | -| syncGroup:network:250 | 250 | 5 | 34 | 3/74 | 175 | 100.0% | PASS ✅ | -| updateGroupName:operation:250 | 250 | 1 | 131 | 131/131 | 120 | 0.0% | FAIL ❌ | -| updateGroupName:network:250 | 250 | 5 | 35 | 3/77 | 175 | 100.0% | PASS ✅ | -| removeMembers:operation:250 | 250 | 1 | 154 | 154/154 | 250 | 100.0% | PASS ✅ | -| sendGroupMessage:operation:250 | 250 | 1 | 53 | 53/53 | 3500 | 100.0% | PASS ✅ | -| removeMembers:network:250 | 250 | 5 | 67 | 15/115 | 175 | 100.0% | PASS ✅ | -| sendGroupMessage:network:250 | 250 | 5 | 39 | 2/88 | 175 | 100.0% | PASS ✅ | -| receiveGroupMessage:operation:250 | 250 | 1 | 113 | 113/113 | 2000 | 100.0% | PASS ✅ | -| receiveGroupMessage:network:250 | 250 | 5 | 34 | 2/76 | 175 | 100.0% | PASS ✅ | -| createGroup:operation:300 | 300 | 1 | 4863 | 4863/4863 | 5500 | 100.0% | PASS ✅ | -| createGroup:network:300 | 300 | 5 | 90 | 16/145 | 175 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:operation:300 | 300 | 1 | 4524 | 4524/4524 | 6500 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network:300 | 300 | 5 | 35 | 3/78 | 175 | 100.0% | PASS ✅ | -| syncGroup:operation:300 | 300 | 1 | 90 | 90/90 | 80 | 0.0% | FAIL ❌ | -| syncGroup:network:300 | 300 | 5 | 37 | 2/81 | 175 | 100.0% | PASS ✅ | -| updateGroupName:operation:300 | 300 | 1 | 138 | 138/138 | 150 | 100.0% | PASS ✅ | -| updateGroupName:network:300 | 300 | 5 | 35 | 2/77 | 175 | 100.0% | PASS ✅ | -| removeMembers:operation:300 | 300 | 1 | 173 | 173/173 | 250 | 100.0% | PASS ✅ | -| sendGroupMessage:operation:300 | 300 | 1 | 53 | 53/53 | 4500 | 100.0% | PASS ✅ | -| removeMembers:network:300 | 300 | 5 | 33 | 2/73 | 175 | 100.0% | PASS ✅ | -| sendGroupMessage:network:300 | 300 | 5 | 39 | 3/87 | 175 | 100.0% | PASS ✅ | -| receiveGroupMessage:operation:300 | 300 | 1 | 105 | 105/105 | 2000 | 100.0% | PASS ✅ | -| receiveGroupMessage:network:300 | 300 | 5 | 34 | 2/74 | 175 | 100.0% | PASS ✅ | -| createGroup:operation:350 | 350 | 1 | 5080 | 5080/5080 | 8000 | 100.0% | PASS ✅ | -| createGroup:network:350 | 350 | 5 | 35 | 4/76 | 175 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:operation:350 | 350 | 1 | 5347 | 5347/5347 | 6500 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network:350 | 350 | 5 | 33 | 2/73 | 175 | 100.0% | PASS ✅ | -| syncGroup:operation:350 | 350 | 1 | 104 | 104/104 | 100 | 0.0% | FAIL ❌ | -| syncGroup:network:350 | 350 | 5 | 38 | 2/84 | 175 | 100.0% | PASS ✅ | -| updateGroupName:operation:350 | 350 | 1 | 147 | 147/147 | 150 | 100.0% | PASS ✅ | -| updateGroupName:network:350 | 350 | 5 | 35 | 3/77 | 175 | 100.0% | PASS ✅ | -| removeMembers:operation:350 | 350 | 1 | 202 | 202/202 | 250 | 100.0% | PASS ✅ | -| sendGroupMessage:operation:350 | 350 | 1 | 57 | 57/57 | 5000 | 100.0% | PASS ✅ | -| removeMembers:network:350 | 350 | 5 | 48 | 3/110 | 175 | 100.0% | PASS ✅ | -| sendGroupMessage:network:350 | 350 | 5 | 42 | 13/85 | 175 | 100.0% | PASS ✅ | -| receiveGroupMessage:operation:350 | 350 | 1 | 175 | 175/175 | 2000 | 100.0% | PASS ✅ | -| receiveGroupMessage:network:350 | 350 | 5 | 34 | 2/75 | 175 | 100.0% | PASS ✅ | -| createGroup:operation:400 | 400 | 1 | 6364 | 6364/6364 | 9000 | 100.0% | PASS ✅ | -| createGroup:network:400 | 400 | 5 | 38 | 2/82 | 175 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:operation:400 | 400 | 1 | 6192 | 6192/6192 | 7500 | 100.0% | PASS ✅ | -| createGroupByIdentifiers:network:400 | 400 | 5 | 36 | 3/80 | 175 | 100.0% | PASS ✅ | -| syncGroup:operation:400 | 400 | 1 | 103 | 103/103 | 120 | 100.0% | PASS ✅ | -| syncGroup:network:400 | 400 | 5 | 43 | 4/95 | 175 | 100.0% | PASS ✅ | -| updateGroupName:operation:400 | 400 | 1 | 174 | 174/174 | 200 | 100.0% | PASS ✅ | -| updateGroupName:network:400 | 400 | 5 | 36 | 2/80 | 175 | 100.0% | PASS ✅ | -| removeMembers:operation:400 | 400 | 1 | 235 | 235/235 | 250 | 100.0% | PASS ✅ | -| sendGroupMessage:operation:400 | 400 | 1 | 61 | 61/61 | 6000 | 100.0% | PASS ✅ | -| removeMembers:network:400 | 400 | 5 | 55 | 15/102 | 175 | 100.0% | PASS ✅ | -| sendGroupMessage:network:400 | 400 | 5 | 35 | 2/77 | 175 | 100.0% | PASS ✅ | -| receiveGroupMessage:operation:400 | 400 | 1 | 127 | 127/127 | 2000 | 100.0% | PASS ✅ | diff --git a/helpers/datadog.ts b/helpers/datadog.ts index 6f4bb6e4..c1d57635 100644 --- a/helpers/datadog.ts +++ b/helpers/datadog.ts @@ -427,7 +427,7 @@ export function logMetricsSummary(testName: string): void { ); // Create a directory for reports if it doesn't exist - const reportsDir = path.join(process.cwd(), "datadog/reports"); + const reportsDir = path.join(process.cwd(), "logs/reports"); if (!fs.existsSync(reportsDir)) { fs.mkdirSync(reportsDir, { recursive: true }); } From 5f0ef677df7542e92e68055c02ad2b17a37644c0 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 16:54:52 -0300 Subject: [PATCH 17/57] better tressholds --- helpers/datadog.ts | 129 ++++++++++++++++----------------------------- 1 file changed, 44 insertions(+), 85 deletions(-) diff --git a/helpers/datadog.ts b/helpers/datadog.ts index c1d57635..8259a3fc 100644 --- a/helpers/datadog.ts +++ b/helpers/datadog.ts @@ -16,17 +16,18 @@ let collectedMetrics: Record< // Refactored thresholds into a single configuration object const THRESHOLDS = { core: { + creategroup: 600, + creategroupbyidentifiers: 550, + sendgroupmessage: 50, + syncgroup: 100, + updategroupname: 100, + removemembers: 75, + receivegroupmessage: 100, clientcreate: 550, createdm: 200, sendgm: 100, receivegm: 100, - creategroup: 2000, - creategroupbyidentifiers: 2000, - receivegroupmessage: 150, - updategroupname: 100, - syncgroup: 100, addmembers: 250, - removemembers: 250, inboxstate: 50, }, network: { @@ -37,65 +38,23 @@ const THRESHOLDS = { server_call: 175, }, group: { - creategroup: { - "50": 900, - "100": 1600, - "150": 2300, - "200": 3500, - "250": 5500, - "300": 5500, - "350": 8000, - "400": 9000, - }, - creategroupbyidentifiers: { - "50": 600, - "100": 1500, - "150": 2600, - "200": 3500, - "250": 5500, - "300": 6500, - "350": 6500, - "400": 7500, - }, - sendgroupmessage: { - "50": 650, - "100": 1000, - "150": 1600, - "200": 2500, - "250": 3500, - "300": 4500, - "350": 5000, - "400": 6000, - }, - syncgroup: { - "50": 100, - "100": 100, - "150": 100, - "200": 100, - "250": 100, - "300": 100, - "350": 100, - "400": 100, - }, - updategroupname: { - "50": 100, - "100": 80, - "150": 100, - "200": 100, - "250": 100, - "300": 100, - "350": 100, - "400": 100, + baseValues: { + creategroup: 600, + creategroupbyidentifiers: 550, + sendgroupmessage: 50, + syncgroup: 100, + updategroupname: 100, + removemembers: 75, + receivegroupmessage: 100, }, - removemembers: { - "50": 100, - "100": 100, - "150": 120, - "200": 150, - "250": 175, - "300": 200, - "350": 225, - "400": 250, + memberMultipliers: { + creategroup: 22, + creategroupbyidentifiers: 18, + sendgroupmessage: 0.12, + syncgroup: 0, + updategroupname: 0.2, + removemembers: 0.5, + receivegroupmessage: 0.05, }, }, regionMultipliers: { @@ -153,28 +112,28 @@ export function getThresholdForOperation( } if (operationType === "group") { - const size = members || "50"; - - // Get the operation-specific thresholds object - const groupThresholds = - THRESHOLDS.group[operationLower as keyof typeof THRESHOLDS.group]; - - // Safely check if this size exists, defaulting to 2000 if not - let baseThreshold = 2000; - if (groupThresholds) { - // Use type assertion to tell TypeScript this is a valid lookup - const sizeKey = size as unknown as keyof typeof groupThresholds; - if (sizeKey in groupThresholds) { - baseThreshold = groupThresholds[sizeKey]; - } - } + // Parse the member count, default to 50 if not provided or invalid + const memberCount = parseInt(members || "50", 10) || 50; - return Math.round( - baseThreshold * - (THRESHOLDS.regionMultipliers[ - region as keyof typeof THRESHOLDS.regionMultipliers - ] || 1.0), - ); + // Get the base value for this operation + const baseValue = + THRESHOLDS.group.baseValues[ + operationLower as keyof typeof THRESHOLDS.group.baseValues + ] || 500; + + // Get the multiplier for this operation + const memberMultiplier = + THRESHOLDS.group.memberMultipliers[ + operationLower as keyof typeof THRESHOLDS.group.memberMultipliers + ] || 0; + + // Calculate the threshold based on member count + const calculatedThreshold = baseValue + memberCount * memberMultiplier; + + // Apply region multiplier + const finalThreshold = Math.round(calculatedThreshold * regionMultiplier); + + return finalThreshold; } // For core operations, ensure we're using lowercase for lookup From f9aebf18c366af0c5c33943988148cc2f61a244c Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 16:55:35 -0300 Subject: [PATCH 18/57] better tressholds --- .github/workflows/TS_Performance.yml | 80 ++++++++++++++-------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/.github/workflows/TS_Performance.yml b/.github/workflows/TS_Performance.yml index 999c6a12..94069a8f 100644 --- a/.github/workflows/TS_Performance.yml +++ b/.github/workflows/TS_Performance.yml @@ -9,46 +9,46 @@ on: workflow_dispatch: jobs: - dev: - runs-on: ubuntu-latest - env: - DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} - LOGGING_LEVEL: ${{ vars.LOGGING_LEVEL }} - XMTP_ENV: "dev" - GEOLOCATION: ${{ vars.GEOLOCATION }} - BATCH_SIZE: ${{ vars.BATCH_SIZE }} - MAX_GROUP_SIZE: ${{ vars.MAX_GROUP_SIZE }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version-file: ".node-version" - cache: "yarn" - env: - SKIP_YARN_COREPACK_CHECK: "1" - - run: corepack enable - - run: yarn - - name: Make test script executable - run: chmod +x ./scripts/run-test.sh - - name: Run tests with retry - run: ./scripts/run-test.sh TS_Performance - - name: Rename logs with run ID prefix - run: | - mkdir -p artifacts/logs - cp -r logs/* artifacts/logs/ - find artifacts/logs -type f -name "*.log" -exec bash -c 'mv "$0" "${0%.log}.run-${{ github.run_id }}-dev.txt"' {} \; - find artifacts/logs -type f ! -name "*.run-*" -exec bash -c 'mv "$0" "$0.run-${{ github.run_id }}-dev.txt"' {} \; - - name: Rename logs/reports with run ID prefix - run: | - mkdir -p artifacts/reports - cp -r logs/reports/* artifacts/reports/ - find artifacts/reports -type f -name "*.md" -exec bash -c 'mv "$0" "${0%.md}.run-${{ github.run_id }}-dev.md"' {} \; - find artifacts/reports -type f ! -name "*.run-*" -exec bash -c 'mv "$0" "$0.run-${{ github.run_id }}-dev.md"' {} \; - - name: Upload all artifacts - uses: actions/upload-artifact@v4 - with: - name: ts_performance_${{ env.XMTP_ENV }} - path: artifacts/ + # dev: + # runs-on: ubuntu-latest + # env: + # DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} + # LOGGING_LEVEL: ${{ vars.LOGGING_LEVEL }} + # XMTP_ENV: "dev" + # GEOLOCATION: ${{ vars.GEOLOCATION }} + # BATCH_SIZE: ${{ vars.BATCH_SIZE }} + # MAX_GROUP_SIZE: ${{ vars.MAX_GROUP_SIZE }} + # steps: + # - uses: actions/checkout@v4 + # - uses: actions/setup-node@v4 + # with: + # node-version-file: ".node-version" + # cache: "yarn" + # env: + # SKIP_YARN_COREPACK_CHECK: "1" + # - run: corepack enable + # - run: yarn + # - name: Make test script executable + # run: chmod +x ./scripts/run-test.sh + # - name: Run tests with retry + # run: ./scripts/run-test.sh TS_Performance + # - name: Rename logs with run ID prefix + # run: | + # mkdir -p artifacts/logs + # cp -r logs/* artifacts/logs/ + # find artifacts/logs -type f -name "*.log" -exec bash -c 'mv "$0" "${0%.log}.run-${{ github.run_id }}-dev.txt"' {} \; + # find artifacts/logs -type f ! -name "*.run-*" -exec bash -c 'mv "$0" "$0.run-${{ github.run_id }}-dev.txt"' {} \; + # - name: Rename logs/reports with run ID prefix + # run: | + # mkdir -p artifacts/reports + # cp -r logs/reports/* artifacts/reports/ + # find artifacts/reports -type f -name "*.md" -exec bash -c 'mv "$0" "${0%.md}.run-${{ github.run_id }}-dev.md"' {} \; + # find artifacts/reports -type f ! -name "*.run-*" -exec bash -c 'mv "$0" "$0.run-${{ github.run_id }}-dev.md"' {} \; + # - name: Upload all artifacts + # uses: actions/upload-artifact@v4 + # with: + # name: ts_performance_${{ env.XMTP_ENV }} + # path: artifacts/ production: runs-on: ubuntu-latest env: From 9810b0fe93ad386d5caec6e0e876edf05b59d43e Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 17:01:14 -0300 Subject: [PATCH 19/57] group miltipliers --- helpers/datadog.ts | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/helpers/datadog.ts b/helpers/datadog.ts index 8259a3fc..720f3ef2 100644 --- a/helpers/datadog.ts +++ b/helpers/datadog.ts @@ -38,15 +38,6 @@ const THRESHOLDS = { server_call: 175, }, group: { - baseValues: { - creategroup: 600, - creategroupbyidentifiers: 550, - sendgroupmessage: 50, - syncgroup: 100, - updategroupname: 100, - removemembers: 75, - receivegroupmessage: 100, - }, memberMultipliers: { creategroup: 22, creategroupbyidentifiers: 18, @@ -117,9 +108,7 @@ export function getThresholdForOperation( // Get the base value for this operation const baseValue = - THRESHOLDS.group.baseValues[ - operationLower as keyof typeof THRESHOLDS.group.baseValues - ] || 500; + THRESHOLDS.core[operationLower as keyof typeof THRESHOLDS.core] || 500; // Get the multiplier for this operation const memberMultiplier = @@ -127,8 +116,15 @@ export function getThresholdForOperation( operationLower as keyof typeof THRESHOLDS.group.memberMultipliers ] || 0; - // Calculate the threshold based on member count - const calculatedThreshold = baseValue + memberCount * memberMultiplier; + // Calculate the threshold based on compounding member count + // Each member above the base threshold compounds the effect + let calculatedThreshold = baseValue; + + // Apply the multiplier in a compounding fashion + // For each member, increase the threshold by the multiplier based on the current value + for (let i = 0; i < memberCount; i++) { + calculatedThreshold += memberMultiplier; + } // Apply region multiplier const finalThreshold = Math.round(calculatedThreshold * regionMultiplier); From 30976307fccf15527c29417fafe5e0a14fadbdc4 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 17:03:02 -0300 Subject: [PATCH 20/57] group miltipliers --- .github/workflows/TS_Performance.yml | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/TS_Performance.yml b/.github/workflows/TS_Performance.yml index 94069a8f..be331ee2 100644 --- a/.github/workflows/TS_Performance.yml +++ b/.github/workflows/TS_Performance.yml @@ -72,20 +72,8 @@ jobs: run: chmod +x ./scripts/run-test.sh - name: Run tests with retry run: ./scripts/run-test.sh TS_Performance - - name: Rename logs with run ID prefix - run: | - mkdir -p artifacts/logs - cp -r logs/* artifacts/logs/ - find artifacts/logs -type f -name "*.log" -exec bash -c 'mv "$0" "${0%.log}.run-${{ github.run_id }}-production.txt"' {} \; - find artifacts/logs -type f ! -name "*.run-*" -exec bash -c 'mv "$0" "$0.run-${{ github.run_id }}-production.txt"' {} \; - - name: Rename logs/reports with run ID prefix - run: | - mkdir -p artifacts/reports - cp -r logs/reports/* artifacts/reports/ - find artifacts/reports -type f -name "*.md" -exec bash -c 'mv "$0" "${0%.md}.run-${{ github.run_id }}-production.md"' {} \; - find artifacts/reports -type f ! -name "*.run-*" -exec bash -c 'mv "$0" "$0.run-${{ github.run_id }}-production.md"' {} \; - - name: Upload all artifacts + - name: Upload reports artifacts uses: actions/upload-artifact@v4 with: - name: ts_performance_${{ env.XMTP_ENV }} - path: artifacts/ + name: ts_performance_${{ env.XMTP_ENV }}_reports + path: logs/ From 58abd7fddc28eda188fb0d2e720c56e5c2db0f03 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 17:06:24 -0300 Subject: [PATCH 21/57] group miltipliers --- helpers/datadog.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/helpers/datadog.ts b/helpers/datadog.ts index 720f3ef2..15c0a6b9 100644 --- a/helpers/datadog.ts +++ b/helpers/datadog.ts @@ -18,11 +18,11 @@ const THRESHOLDS = { core: { creategroup: 600, creategroupbyidentifiers: 550, - sendgroupmessage: 50, - syncgroup: 100, - updategroupname: 100, - removemembers: 75, - receivegroupmessage: 100, + sendgroupmessage: 98, + syncgroup: 135, + updategroupname: 200, + removemembers: 250, + receivegroupmessage: 150, clientcreate: 550, createdm: 200, sendgm: 100, @@ -42,10 +42,10 @@ const THRESHOLDS = { creategroup: 22, creategroupbyidentifiers: 18, sendgroupmessage: 0.12, - syncgroup: 0, - updategroupname: 0.2, - removemembers: 0.5, - receivegroupmessage: 0.05, + syncgroup: 0.1, + updategroupname: 0.4, + removemembers: 0.7, + receivegroupmessage: 0.1, }, }, regionMultipliers: { From 99a18995dff2918eac35ab4ed05014c547fd868a Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 17:11:02 -0300 Subject: [PATCH 22/57] group miltipliers --- helpers/datadog.ts | 49 +++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/helpers/datadog.ts b/helpers/datadog.ts index 15c0a6b9..c05630d9 100644 --- a/helpers/datadog.ts +++ b/helpers/datadog.ts @@ -16,13 +16,13 @@ let collectedMetrics: Record< // Refactored thresholds into a single configuration object const THRESHOLDS = { core: { - creategroup: 600, - creategroupbyidentifiers: 550, - sendgroupmessage: 98, - syncgroup: 135, - updategroupname: 200, + creategroup: 1700, + creategroupbyidentifiers: 1450, + sendgroupmessage: 56, + syncgroup: 100, + updategroupname: 110, removemembers: 250, - receivegroupmessage: 150, + receivegroupmessage: 102, clientcreate: 550, createdm: 200, sendgm: 100, @@ -37,17 +37,16 @@ const THRESHOLDS = { processing: 175, server_call: 175, }, - group: { - memberMultipliers: { - creategroup: 22, - creategroupbyidentifiers: 18, - sendgroupmessage: 0.12, - syncgroup: 0.1, - updategroupname: 0.4, - removemembers: 0.7, - receivegroupmessage: 0.1, - }, + memberMultipliers: { + creategroup: 1100, + creategroupbyidentifiers: 900, + sendgroupmessage: 6, + syncgroup: 0, + updategroupname: 10, + removemembers: 0, + receivegroupmessage: 2, }, + regionMultipliers: { "us-east": 1.0, "us-west": 1.0, @@ -106,24 +105,24 @@ export function getThresholdForOperation( // Parse the member count, default to 50 if not provided or invalid const memberCount = parseInt(members || "50", 10) || 50; - // Get the base value for this operation + // Get the base value for this operation (which is for 50 members) const baseValue = THRESHOLDS.core[operationLower as keyof typeof THRESHOLDS.core] || 500; // Get the multiplier for this operation const memberMultiplier = - THRESHOLDS.group.memberMultipliers[ - operationLower as keyof typeof THRESHOLDS.group.memberMultipliers + THRESHOLDS.memberMultipliers[ + operationLower as keyof typeof THRESHOLDS.memberMultipliers ] || 0; - // Calculate the threshold based on compounding member count - // Each member above the base threshold compounds the effect + // Calculate threshold based on group size increases in batches of 50 + // First 50 members use the base value + // Each additional batch of 50 members adds the multiplier let calculatedThreshold = baseValue; - // Apply the multiplier in a compounding fashion - // For each member, increase the threshold by the multiplier based on the current value - for (let i = 0; i < memberCount; i++) { - calculatedThreshold += memberMultiplier; + if (memberCount > 50) { + const additionalBatches = Math.floor((memberCount - 50) / 50); + calculatedThreshold += memberMultiplier * additionalBatches; } // Apply region multiplier From 3a077d1a01dc67645abd81e26e86c5ae118c959c Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 17:17:27 -0300 Subject: [PATCH 23/57] group miltipliers --- .github/workflows/TS_Performance.yml | 24 ++++++------------------ helpers/datadog.ts | 12 ++++++------ 2 files changed, 12 insertions(+), 24 deletions(-) diff --git a/.github/workflows/TS_Performance.yml b/.github/workflows/TS_Performance.yml index be331ee2..fb59f27e 100644 --- a/.github/workflows/TS_Performance.yml +++ b/.github/workflows/TS_Performance.yml @@ -32,23 +32,11 @@ jobs: # run: chmod +x ./scripts/run-test.sh # - name: Run tests with retry # run: ./scripts/run-test.sh TS_Performance - # - name: Rename logs with run ID prefix - # run: | - # mkdir -p artifacts/logs - # cp -r logs/* artifacts/logs/ - # find artifacts/logs -type f -name "*.log" -exec bash -c 'mv "$0" "${0%.log}.run-${{ github.run_id }}-dev.txt"' {} \; - # find artifacts/logs -type f ! -name "*.run-*" -exec bash -c 'mv "$0" "$0.run-${{ github.run_id }}-dev.txt"' {} \; - # - name: Rename logs/reports with run ID prefix - # run: | - # mkdir -p artifacts/reports - # cp -r logs/reports/* artifacts/reports/ - # find artifacts/reports -type f -name "*.md" -exec bash -c 'mv "$0" "${0%.md}.run-${{ github.run_id }}-dev.md"' {} \; - # find artifacts/reports -type f ! -name "*.run-*" -exec bash -c 'mv "$0" "$0.run-${{ github.run_id }}-dev.md"' {} \; - # - name: Upload all artifacts - # uses: actions/upload-artifact@v4 - # with: - # name: ts_performance_${{ env.XMTP_ENV }} - # path: artifacts/ + # - name: Upload reports artifacts + # uses: actions/upload-artifact@v4 + # with: + # name: ts_performance_${{ env.XMTP_ENV }} + # path: logs/ production: runs-on: ubuntu-latest env: @@ -75,5 +63,5 @@ jobs: - name: Upload reports artifacts uses: actions/upload-artifact@v4 with: - name: ts_performance_${{ env.XMTP_ENV }}_reports + name: ts_performance_${{ env.XMTP_ENV }} path: logs/ diff --git a/helpers/datadog.ts b/helpers/datadog.ts index c05630d9..d0dcf49b 100644 --- a/helpers/datadog.ts +++ b/helpers/datadog.ts @@ -38,13 +38,13 @@ const THRESHOLDS = { server_call: 175, }, memberMultipliers: { - creategroup: 1100, - creategroupbyidentifiers: 900, + creategroup: 2, + creategroupbyidentifiers: 2, sendgroupmessage: 6, - syncgroup: 0, - updategroupname: 10, - removemembers: 0, - receivegroupmessage: 2, + syncgroup: 0.1, + updategroupname: 0.2, + removemembers: 0.1, + receivegroupmessage: 0.2, }, regionMultipliers: { From 5afd1a58b74ea8b7e07691b0173bd462658669f0 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 17:30:36 -0300 Subject: [PATCH 24/57] mutlipliers --- helpers/datadog.ts | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/helpers/datadog.ts b/helpers/datadog.ts index d0dcf49b..062d1dd4 100644 --- a/helpers/datadog.ts +++ b/helpers/datadog.ts @@ -16,35 +16,35 @@ let collectedMetrics: Record< // Refactored thresholds into a single configuration object const THRESHOLDS = { core: { - creategroup: 1700, - creategroupbyidentifiers: 1450, - sendgroupmessage: 56, - syncgroup: 100, - updategroupname: 110, - removemembers: 250, - receivegroupmessage: 102, clientcreate: 550, + inboxstate: 50, createdm: 200, sendgm: 100, receivegm: 100, + receivegroupmessage: 102, + creategroup: 300, + creategroupbyidentifiers: 300, + sendgroupmessage: 56, + syncgroup: 100, + updategroupname: 110, + removemembers: 250, addmembers: 250, - inboxstate: 50, }, network: { - dns_lookup: 175, - tcp_connection: 175, - tls_handshake: 175, - processing: 175, - server_call: 175, + dns_lookup: 50, + tcp_connection: 50, + tls_handshake: 150, + processing: 100, + server_call: 250, }, memberMultipliers: { - creategroup: 2, - creategroupbyidentifiers: 2, - sendgroupmessage: 6, - syncgroup: 0.1, - updategroupname: 0.2, - removemembers: 0.1, - receivegroupmessage: 0.2, + creategroup: 3, + creategroupbyidentifiers: 3, + sendgroupmessage: 0.3, + syncgroup: 0.3, + updategroupname: 0.3, + removemembers: 0.3, + receivegroupmessage: 0.3, }, regionMultipliers: { @@ -123,6 +123,9 @@ export function getThresholdForOperation( if (memberCount > 50) { const additionalBatches = Math.floor((memberCount - 50) / 50); calculatedThreshold += memberMultiplier * additionalBatches; + console.log( + `calculatedThreshold: ${calculatedThreshold} for ${memberCount} members`, + ); } // Apply region multiplier From 49a9697f047fd46cc0c84b619290d873b850f8f2 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 17:32:10 -0300 Subject: [PATCH 25/57] better artifacts --- .github/workflows/TS_Delivery.yml | 8 ++++---- .github/workflows/TS_Geolocation.yml | 4 ++-- .github/workflows/TS_Gm.yml | 20 +++----------------- 3 files changed, 9 insertions(+), 23 deletions(-) diff --git a/.github/workflows/TS_Delivery.yml b/.github/workflows/TS_Delivery.yml index 2d4a95b5..504be80f 100644 --- a/.github/workflows/TS_Delivery.yml +++ b/.github/workflows/TS_Delivery.yml @@ -34,10 +34,10 @@ jobs: run: chmod +x ./scripts/run-test.sh - name: Run tests with retry run: ./scripts/run-test.sh TS_Delivery - - name: Upload log files + - name: Upload reports artifacts uses: actions/upload-artifact@v4 with: - name: delivery_logs_dev + name: ts_delivery_${{ env.XMTP_ENV }} path: logs/ production: runs-on: ubuntu-latest @@ -66,8 +66,8 @@ jobs: run: chmod +x ./scripts/run-test.sh - name: Run tests with retry run: ./scripts/run-test.sh TS_Delivery - - name: Upload log files + - name: Upload reports artifacts uses: actions/upload-artifact@v4 with: - name: delivery_logs_production + name: ts_delivery_${{ env.XMTP_ENV }} path: logs/ diff --git a/.github/workflows/TS_Geolocation.yml b/.github/workflows/TS_Geolocation.yml index 311d61e8..0ca725a9 100644 --- a/.github/workflows/TS_Geolocation.yml +++ b/.github/workflows/TS_Geolocation.yml @@ -71,7 +71,7 @@ jobs: - name: Upload log files uses: actions/upload-artifact@v4 with: - name: geolocation_logs_dev + name: ts_geolocation_${{ env.XMTP_ENV }} path: logs/ - name: Check for failures @@ -154,7 +154,7 @@ jobs: - name: Upload log files uses: actions/upload-artifact@v4 with: - name: geolocation_logs_production + name: ts_geolocation_${{ env.XMTP_ENV }} path: logs/ - name: Check for failures diff --git a/.github/workflows/TS_Gm.yml b/.github/workflows/TS_Gm.yml index 9badd501..ebef9e6d 100644 --- a/.github/workflows/TS_Gm.yml +++ b/.github/workflows/TS_Gm.yml @@ -38,18 +38,11 @@ jobs: run: chmod +x ./scripts/run-test.sh - name: Run tests with retry run: ./scripts/run-test.sh TS_Gm - - name: Upload log files + - name: Upload reports artifacts uses: actions/upload-artifact@v4 - if: always() with: - name: gm_logs_dev + name: ts_gm_${{ env.XMTP_ENV }} path: logs/ - - name: Upload snapshots - uses: actions/upload-artifact@v4 - if: always() - with: - name: playwright_snapshots - path: logs/snapshots/*.png production: runs-on: ubuntu-latest env: @@ -83,13 +76,6 @@ jobs: run: ./scripts/run-test.sh TS_Gm - name: Upload log files uses: actions/upload-artifact@v4 - if: always() with: - name: gm_logs_production + name: ts_gm_${{ env.XMTP_ENV }} path: logs/ - - name: Upload snapshots - uses: actions/upload-artifact@v4 - if: always() - with: - name: playwright_snapshots - path: logs/snapshots/*.png From 113de0b427d8c907000c7fbfc12c668dfd9ad58d Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 17:33:16 -0300 Subject: [PATCH 26/57] better artifacts --- helpers/datadog.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/helpers/datadog.ts b/helpers/datadog.ts index 062d1dd4..d7f33509 100644 --- a/helpers/datadog.ts +++ b/helpers/datadog.ts @@ -40,11 +40,11 @@ const THRESHOLDS = { memberMultipliers: { creategroup: 3, creategroupbyidentifiers: 3, - sendgroupmessage: 0.3, - syncgroup: 0.3, - updategroupname: 0.3, - removemembers: 0.3, - receivegroupmessage: 0.3, + sendgroupmessage: 1.3, + syncgroup: 1.3, + updategroupname: 1.3, + removemembers: 1.3, + receivegroupmessage: 1.3, }, regionMultipliers: { From 485d584929acf07b436848e994fc6f955c70a9a9 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 17:35:03 -0300 Subject: [PATCH 27/57] better artifacts --- helpers/datadog.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/helpers/datadog.ts b/helpers/datadog.ts index d7f33509..eb3032b4 100644 --- a/helpers/datadog.ts +++ b/helpers/datadog.ts @@ -115,14 +115,13 @@ export function getThresholdForOperation( operationLower as keyof typeof THRESHOLDS.memberMultipliers ] || 0; - // Calculate threshold based on group size increases in batches of 50 - // First 50 members use the base value - // Each additional batch of 50 members adds the multiplier let calculatedThreshold = baseValue; if (memberCount > 50) { const additionalBatches = Math.floor((memberCount - 50) / 50); - calculatedThreshold += memberMultiplier * additionalBatches; + // Apply multiplier as a percentage increase per batch instead of a fixed value + calculatedThreshold += + baseValue * ((memberMultiplier * additionalBatches) / 100); console.log( `calculatedThreshold: ${calculatedThreshold} for ${memberCount} members`, ); From 9d0c9336957339debe7e6c08831b2a3a1674c778 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 17:37:21 -0300 Subject: [PATCH 28/57] better artifacts --- helpers/datadog.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/helpers/datadog.ts b/helpers/datadog.ts index eb3032b4..5bced41f 100644 --- a/helpers/datadog.ts +++ b/helpers/datadog.ts @@ -119,11 +119,10 @@ export function getThresholdForOperation( if (memberCount > 50) { const additionalBatches = Math.floor((memberCount - 50) / 50); - // Apply multiplier as a percentage increase per batch instead of a fixed value - calculatedThreshold += - baseValue * ((memberMultiplier * additionalBatches) / 100); + calculatedThreshold = + baseValue * (1 + (memberMultiplier * additionalBatches) / 10); console.log( - `calculatedThreshold: ${calculatedThreshold} for ${memberCount} members`, + `calculatedThreshold: ${calculatedThreshold} for ${memberCount} members with multiplier ${memberMultiplier}`, ); } From 5598fc1623ce7a8c121b81aa139bf2ccef6af159 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 18:43:20 -0300 Subject: [PATCH 29/57] fix --- datadog/README.md | 47 ++ datadog/calculations.ts | 117 +++++ datadog/{ => dashboards}/delivery.json | 0 datadog/{ => dashboards}/performance.json | 0 datadog/{ => dashboards}/workflows.json | 0 datadog/helper.ts | 329 +++++++++++++ datadog/network.ts | 59 +++ datadog/summary.ts | 175 +++++++ datadog/thresholds.json | 48 ++ helpers/README.md | 19 - helpers/client.ts | 11 +- helpers/datadog.ts | 549 ---------------------- scripts/railway-test.ts | 2 +- tests/TS_Delivery.test.ts | 2 +- tests/TS_Gm.test.ts | 2 +- tests/TS_Performance.test.ts | 6 +- tests/dms.test.ts | 2 +- tests/groups.test.ts | 2 +- tests/offline.test.ts | 2 +- tsconfig.json | 1 + vitest.config.ts | 1 + 21 files changed, 794 insertions(+), 580 deletions(-) create mode 100644 datadog/README.md create mode 100644 datadog/calculations.ts rename datadog/{ => dashboards}/delivery.json (100%) rename datadog/{ => dashboards}/performance.json (100%) rename datadog/{ => dashboards}/workflows.json (100%) create mode 100644 datadog/helper.ts create mode 100644 datadog/network.ts create mode 100644 datadog/summary.ts create mode 100644 datadog/thresholds.json delete mode 100644 helpers/datadog.ts diff --git a/datadog/README.md b/datadog/README.md new file mode 100644 index 00000000..50e3af21 --- /dev/null +++ b/datadog/README.md @@ -0,0 +1,47 @@ +# XMTP Datadog integration + +This module provides tools for monitoring XMTP performance metrics via Datadog integration. + +## Overview + +The XMTP Datadog integration allows you to track crucial performance metrics, message delivery rates, and network statistics for your XMTP implementation. This data helps maintain optimal service quality and quickly identify potential issues. + +## Key components + +- `summary.ts` - Aggregates and formats metrics data +- `helper.ts` - Utility functions for Datadog integration +- `network.ts` - Network performance monitoring tools +- `thresholds.json` - Configuration for alert thresholds +- `dashboards/` - Datadog dashboard configuration files + +## Usage examples + +```typescript +// Initialize Datadog metrics +initDataDog(testName, envValue, geolocation, apiKey); + +// Send delivery rate metrics +sendDeliveryMetric(deliveryRate, testName, libxmtpVersion); + +// Send performance metrics +sendPerformanceMetric(durationMs, testName, libxmtpVersion); + +// Measure network performance +const networkStats = await getNetworkStats(); +``` + +## Common metrics + +- Message delivery rates +- End-to-end message latency +- Network performance statistics +- API response times +- Error rates and types + +## Dashboard integration + +The `dashboards/` directory contains configuration for visualizing your XMTP metrics in Datadog. These dashboards provide real-time monitoring of your application's performance. + +- [Performance Dashboard](./dashboards/performance.json) +- [Delivery Rate Dashboard](./dashboards/delivery-rate.json) +- [Network Performance Dashboard](./dashboards/network-performance.json) diff --git a/datadog/calculations.ts b/datadog/calculations.ts new file mode 100644 index 00000000..36f2e397 --- /dev/null +++ b/datadog/calculations.ts @@ -0,0 +1,117 @@ +import THRESHOLDS from "./thresholds.json"; + +// Simplified threshold function +export function getThresholdForOperation( + operation: string, + operationType: "core" | "group" | "network", + members: string = "", + region: string = "us-east", + batchSize: number = 10, + total: number = 10, +): number { + // Convert operation to lowercase for consistent lookups + const operationLower = operation.toLowerCase(); + + // Normalize region name and ensure it exists in the thresholds + const regionNormalized = region.toLowerCase().trim(); + + // Get the region multiplier with proper logging + const regionMultiplier = + THRESHOLDS.regionMultipliers[ + regionNormalized as keyof typeof THRESHOLDS.regionMultipliers + ] || 1.0; + + // Log if region multiplier not found (for debugging) + if (!(regionNormalized in THRESHOLDS.regionMultipliers)) { + console.warn( + `Region multiplier not found for: ${regionNormalized}, using default 1.0`, + ); + } + + if (operationType === "network") { + const networkThreshold = + THRESHOLDS.network[operationLower as keyof typeof THRESHOLDS.network]; + // Default to 200 if the specific network operation isn't found + const baseThreshold = + typeof networkThreshold === "number" ? networkThreshold : 200; + + // Apply region multiplier to network thresholds + const finalThreshold = Math.round(baseThreshold * regionMultiplier); + + return finalThreshold; + } + + if (operationType === "group") { + // Parse the member count from the members string if available + // Use a more robust parsing approach + let memberCount = total; // Default to total + + if (members && members !== "-") { + const parsedMember = parseInt(members); + if (!isNaN(parsedMember)) { + memberCount = parsedMember; + } + } + + console.log( + `Calculating threshold for ${operation} (${operationType}) with ${memberCount} members`, + ); + + // Get the base value for this operation + const baseValue = + THRESHOLDS.core[operationLower as keyof typeof THRESHOLDS.core]; + + // Get the multiplier for this operation + const memberMultiplier = + THRESHOLDS.memberMultipliers[ + operationLower as keyof typeof THRESHOLDS.memberMultipliers + ] || 0; + + let calculatedThreshold = baseValue; + + // Calculate based on the actual member count + const batches = Math.ceil(memberCount / batchSize); + calculatedThreshold = baseValue * (1 + memberMultiplier * (batches - 1)); + + console.log( + `Operation: ${operation}, Using memberCount: ${memberCount} (from members: ${members}), batches: ${batches}, threshold: ${calculatedThreshold}`, + ); + + // Apply region multiplier + const finalThreshold = Math.round(calculatedThreshold * regionMultiplier); + + return finalThreshold; + } + + // For core operations, ensure we're using lowercase for lookup + const baseThreshold = + operationLower in THRESHOLDS.core + ? THRESHOLDS.core[operationLower as keyof typeof THRESHOLDS.core] + : 300; + + // Check if this is a group operation that needs member multiplier + const isGroupOperation = operationLower in THRESHOLDS.memberMultipliers; + + if (isGroupOperation && members) { + // Get the multiplier for this operation + const memberMultiplier = + THRESHOLDS.memberMultipliers[ + operationLower as keyof typeof THRESHOLDS.memberMultipliers + ] || 0; + + let calculatedThreshold = baseThreshold; + + // Always calculate based on batch size and total + const batches = Math.ceil(total / batchSize); + calculatedThreshold = + baseThreshold * (1 + memberMultiplier * (batches - 1)); + + // Apply region multiplier + const finalThreshold = Math.round(calculatedThreshold * regionMultiplier); + return finalThreshold; + } + + // For regular core operations + const finalThreshold = Math.round(baseThreshold * regionMultiplier); + return finalThreshold; +} diff --git a/datadog/delivery.json b/datadog/dashboards/delivery.json similarity index 100% rename from datadog/delivery.json rename to datadog/dashboards/delivery.json diff --git a/datadog/performance.json b/datadog/dashboards/performance.json similarity index 100% rename from datadog/performance.json rename to datadog/dashboards/performance.json diff --git a/datadog/workflows.json b/datadog/dashboards/workflows.json similarity index 100% rename from datadog/workflows.json rename to datadog/dashboards/workflows.json diff --git a/datadog/helper.ts b/datadog/helper.ts new file mode 100644 index 00000000..3c06bde8 --- /dev/null +++ b/datadog/helper.ts @@ -0,0 +1,329 @@ +import type { WorkerManager } from "@workers/manager"; +import metrics from "datadog-metrics"; +import { getThresholdForOperation } from "./calculations"; +import { getNetworkStats } from "./network"; +import { logMetricsSummary } from "./summary"; +import THRESHOLDS from "./thresholds.json"; + +// Global state variables +let isInitialized = false; +let currentGeo: string = ""; +let collectedMetrics: Record< + string, + { values: number[]; threshold: number; members?: string } +> = {}; + +function getCountryCodeFromGeo(geolocation: string): string { + return ( + THRESHOLDS.GEO_TO_COUNTRY_CODE[ + geolocation as keyof typeof THRESHOLDS.GEO_TO_COUNTRY_CODE + ] || "US" + ); +} + +export const sendPerformanceResult = ( + expect: any, + workers: WorkerManager, + start: number, + batchSize?: number, + total?: number, +) => { + const testName = expect.getState().currentTestName; + if (testName) { + console.timeEnd(testName as string); + expect(workers.getWorkers()).toBeDefined(); + expect(workers.getWorkers().length).toBeGreaterThan(0); + void sendPerformanceMetric( + performance.now() - start, + testName as string, + workers.getVersion(), + false, + batchSize ?? undefined, + total ?? undefined, + ); + } +}; +// Add success status tag to duration metrics + +export function initDataDog( + testName: string, + envValue: string, + geolocation: string, + apiKey: string, +): boolean { + if (isInitialized) { + return true; + } + if (!testName.includes("ts_")) { + return true; + } + if (!apiKey) { + console.warn("⚠️ DATADOG_API_KEY not found. Metrics will not be sent."); + return false; + } + + try { + const countryCode = getCountryCodeFromGeo(geolocation); + currentGeo = geolocation; + const initConfig = { + apiKey: apiKey, + defaultTags: [ + `env:${envValue}`, + `test:${testName}`, + `region:${geolocation}`, + `country_iso_code:${countryCode}`, + ], + }; + metrics.init(initConfig); + isInitialized = true; + return true; + } catch (error) { + console.error("❌ Failed to initialize DataDog metrics:", error); + return false; + } +} + +// Modify the sendMetric function to collect metrics for summary +export function sendMetric( + metricName: string, + metricValue: number, + tags: Record, +): void { + if (!isInitialized) return; + + try { + console.log( + `SENDING METRIC: ${metricName}, Value: ${metricValue}, Members: ${tags.members || "none"}`, + ); + + const fullMetricName = `xmtp.sdk.${metricName}`; + const allTags = Object.entries({ ...tags }).map( + ([key, value]) => `${key}:${String(value)}`, + ); + // Add environment tag if not already present + if (!allTags.some((tag) => tag.startsWith("env:"))) { + allTags.push(`env:${process.env.XMTP_ENV as string}`); + } + // Add version tag if not already present + if (!allTags.some((tag) => tag.startsWith("xv:"))) { + allTags.push(`vm:${process.env.RAILWAY_SERVICE_ID || "unknown"}`); + } + + // Create a distinctive operation key that properly includes member count + // Format: operation_name-member_count (e.g., "createGroup-10") + const memberCount = tags.members || ""; + const operationKey = tags.operation + ? `${tags.operation}${memberCount ? `-${memberCount}` : ""}` + : metricName; + + // For debugging + console.log( + `METRIC KEY: ${operationKey} (Operation: ${tags.operation}, Members: ${memberCount})`, + ); + + if (!collectedMetrics[operationKey]) { + collectedMetrics[operationKey] = { + values: [], + threshold: tags.threshold || 0, + members: memberCount, + }; + } + collectedMetrics[operationKey].values.push(metricValue); + + metrics.gauge(fullMetricName, Math.round(metricValue), allTags); + } catch (error) { + console.error( + `❌ Error sending metric '${metricName}':`, + error instanceof Error ? error.message : String(error), + ); + } +} + +export function sendTestResults(hasFailures: boolean, testName: string): void { + if (!isInitialized) { + console.warn("Datadog metrics not initialized"); + return; + } + try { + const metricValue = hasFailures ? 0 : 1; + const metricName = `workflow`; + sendMetric(metricName, metricValue, { + workflow: testName, + metric_type: "workflow", + }); + console.log( + `The tests indicated that the test ${testName} was ${hasFailures}`, + ); + } catch (error) { + console.error("Error reporting to Datadog:", error); + } +} + +// Simplified version of sendPerformanceMetric +export async function sendPerformanceMetric( + metricValue: number, + testName: string, + libxmtpVersion: string, + skipNetworkStats: boolean = false, + batchSize?: number, + total?: number, +): Promise { + if (!isInitialized) return; + + try { + const metricNameParts = testName.split(":")[0]; + const metricName = metricNameParts.replaceAll(" > ", "."); + const metricDescription = testName.split(":")[1] || ""; + const operationParts = metricName.split("."); + const testNameExtracted = operationParts[0]; + + // Extract operation name and member count more reliably + let operationName = ""; + let members = ""; + + if (operationParts[1]) { + // Check for both formats: "operation-10" and "operation10" + const dashMatch = operationParts[1].match(/^([a-zA-Z]+)-(\d+)$/); + const noSeparatorMatch = operationParts[1].match(/^([a-zA-Z]+)(\d+)$/); + + if (dashMatch) { + operationName = dashMatch[1]; + members = dashMatch[2]; + } else if (noSeparatorMatch) { + operationName = noSeparatorMatch[1]; + members = noSeparatorMatch[2]; + } else { + operationName = operationParts[1]; + } + } + + // Debug output + console.log( + `Extracted from test name: Operation=${operationName}, Members=${members}, Test=${testNameExtracted}`, + ); + + const isGroupOperation = operationName.toLowerCase().includes("-"); + const operationType = isGroupOperation ? "group" : "core"; + + // Use the extracted member count for threshold calculation + const memberCount = members ? parseInt(members) : total; + const threshold = getThresholdForOperation( + operationName, + operationType, + members, + currentGeo, + batchSize ?? undefined, + memberCount, // Pass the actual member count here + ); + + const isSuccess = metricValue <= threshold; + + console.log(`Operation metrics:`, { + operation: operationName, + members: members, + testName: testName, + }); + + console.log(`Running test with ${memberCount} members`); + + sendMetric("duration", metricValue, { + libxmtp: libxmtpVersion, + operation: operationName, + test: testNameExtracted, + metric_type: "operation", + metric_subtype: operationType, + description: metricDescription, + members: members, + success: isSuccess, + threshold: threshold, + region: currentGeo, + }); + + // Network stats handling + if (!skipNetworkStats) { + //ignore group operations + const networkStats = await getNetworkStats(); + const countryCode = getCountryCodeFromGeo(currentGeo); + + for (const [statName, statValue] of Object.entries(networkStats)) { + const networkMetricValue = Math.round(statValue * 1000); + const networkPhase = statName.toLowerCase().replace(/\s+/g, "_"); + const networkThreshold = getThresholdForOperation( + networkPhase, + "network", + members, + currentGeo, + batchSize, + total, + ); + + sendMetric("duration", networkMetricValue, { + libxmtp: libxmtpVersion, + operation: operationName, + test: testNameExtracted, + metric_type: "network", + network_phase: networkPhase, + country_iso_code: countryCode, + members: members, + success: networkMetricValue <= networkThreshold, + threshold: networkThreshold, + region: currentGeo, + }); + } + } + } catch (error) { + console.error( + `❌ Error sending performance metric for '${testName}':`, + error, + ); + } +} + +// Modify flushMetrics to include the summary log +export function flushMetrics( + testName: string, + batchSize?: number, +): Promise { + return new Promise((resolve) => { + if (!isInitialized) { + resolve(); + return; + } + + logMetricsSummary( + testName, + currentGeo, + isInitialized, + collectedMetrics, + batchSize ?? undefined, + ); + + void metrics.flush().then(() => { + resolve(); + }); + }); +} + +// Unified delivery metrics function +export function sendDeliveryMetric( + metricValue: number, + version: string, + testName: string, + metricSubType: "stream" | "poll" | "recovery", + metricType: "delivery" | "order", +): void { + // Determine success based on the metric subtype + const threshold = THRESHOLDS.reliability; + + const isSuccess = metricValue >= threshold; + + // Send primary metric + sendMetric(metricType, Math.round(metricValue), { + libxmtp: version, + test: testName, + metric_type: metricType, + metric_subtype: metricSubType, + success: isSuccess, + threshold: threshold, + }); +} diff --git a/datadog/network.ts b/datadog/network.ts new file mode 100644 index 00000000..946131ca --- /dev/null +++ b/datadog/network.ts @@ -0,0 +1,59 @@ +import { exec } from "child_process"; +import { promisify } from "util"; + +const execAsync = promisify(exec); + +interface NetworkStats { + "DNS Lookup": number; + "TCP Connection": number; + "TLS Handshake": number; + Processing: number; + "Server Call": number; +} + +export async function getNetworkStats( + endpoint = "https://grpc.dev.xmtp.network:443", +): Promise { + const curlCommand = `curl -s -w "\\n{\\"DNS Lookup\\": %{time_namelookup}, \\"TCP Connection\\": %{time_connect}, \\"TLS Handshake\\": %{time_appconnect}, \\"Server Call\\": %{time_starttransfer}}" -o /dev/null --max-time 10 ${endpoint}`; + + let stdout: string; + + try { + const result = await execAsync(curlCommand); + stdout = result.stdout; + } catch (error: any) { + // Even if curl returns an error, we might still have useful stdout + if (error.stdout) { + stdout = error.stdout; + console.warn( + `⚠️ Curl command returned error code ${error.code}, but stdout is available.`, + ); + } else { + console.error(`❌ Curl command failed without stdout:`, error); + throw error; // rethrow if no stdout is available + } + } + + // Parse the JSON response + const stats = JSON.parse(stdout.trim()) as NetworkStats; + + // Handle the case where Server Call time is 0 + if (stats["Server Call"] === 0) { + console.warn( + `Network request to ${endpoint} returned Server Call time of 0.`, + ); + + // Use a reasonable estimate + stats["Server Call"] = stats["TLS Handshake"] + 0.1; + } + + // Calculate processing time + stats["Processing"] = stats["Server Call"] - stats["TLS Handshake"]; + + // Ensure processing time is not negative + if (stats["Processing"] < 0) { + stats["Processing"] = 0; + } + + return stats; +} diff --git a/datadog/summary.ts b/datadog/summary.ts new file mode 100644 index 00000000..c8f0c4a9 --- /dev/null +++ b/datadog/summary.ts @@ -0,0 +1,175 @@ +import fs from "fs"; +import path from "path"; +import { getThresholdForOperation } from "./calculations"; + +/** + * Logs a summary of all collected metrics against their thresholds + * Call this at the end of test execution + */ +export function logMetricsSummary( + testName: string, + currentGeo: string, + isInitialized: boolean, + collectedMetrics: Record< + string, + { + values: number[]; + threshold: number; + members?: string; + } + >, + batchSize?: number, +): void { + if (!isInitialized || Object.keys(collectedMetrics).length === 0) { + console.log("No metrics collected to summarize"); + return; + } + + console.log("\n📊 Creating metrics summary report"); + + // ADD THIS: Log the raw collected metrics to see what's actually there + console.log( + "RAW COLLECTED METRICS:", + JSON.stringify(collectedMetrics, null, 2), + ); + + // Create a simple text summary for the console + const passedMetrics = Object.entries(collectedMetrics).filter( + ([operation, data]) => { + // Skip workflow metrics when counting passed metrics + if (operation === "workflow") return false; + + if (data.values.length === 0) return false; + const average = + data.values.reduce((sum, val) => sum + val, 0) / data.values.length; + return average <= data.threshold; + }, + ).length; + + // Count only non-workflow metrics for total + const totalMetrics = Object.entries(collectedMetrics).filter( + ([operation]) => operation !== "workflow", + ).length; + + console.log( + `✅ Passed: ${passedMetrics}/${totalMetrics} metrics (${Math.round((passedMetrics / totalMetrics) * 100)}%)`, + ); + + // Create a directory for reports if it doesn't exist + const reportsDir = path.join(process.cwd(), "logs/reports"); + if (!fs.existsSync(reportsDir)) { + fs.mkdirSync(reportsDir, { recursive: true }); + } + + // Create filename with timestamp + const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); + const filename = path.join( + reportsDir, + `${testName}-${currentGeo}-${process.env.XMTP_ENV}.md`, + ); + + // Build the table content + let fileContent = "METRICS SUMMARY\n===============\n\n"; + fileContent += + "Operation | Members | Avg (ms) | Min/Max (ms) | Threshold (ms) | Variance (ms) | Status\n"; + fileContent += + "----------|---------|----------|--------------|----------------|---------------|-------\n"; + + // Group metrics by base operation name to associate network times with operations + const operationGroups = new Map(); + + // First pass - group and organize data + for (const [operation, data] of Object.entries(collectedMetrics)) { + // Skip workflow metrics in the summary table + if (operation === "workflow") continue; + if (data.values.length === 0) continue; + + // Look for operation-members format (e.g., "createGroup-10") + const dashMatch = operation.match(/^([a-zA-Z]+)-(\d+)$/); + const operationName = dashMatch ? dashMatch[1] : operation.split(":")[0]; + const memberCount = dashMatch ? dashMatch[2] : data.members || "-"; + + // Create a unique key for each operation + member count combination + const groupKey = `${operationName}-${memberCount}`; + + console.log( + `Processing metric: ${operation} → ${groupKey} (Members: ${memberCount})`, + ); + + if (!operationGroups.has(groupKey)) { + operationGroups.set(groupKey, { + operationName: operationName, + members: memberCount, + operationData: null, + }); + } + + const group = operationGroups.get(groupKey); + + group.operationData = data; + } + + // Second pass - generate table rows + for (const [groupKey, group] of operationGroups.entries()) { + if (!group.operationData) continue; + + const data = group.operationData; + const operationName = group.operationName; + + // Use the member count preserved in the group + const memberCount = + group.members !== "-" ? parseInt(group.members as string) : 10; + + console.log( + `Calculating threshold for ${operationName} with ${memberCount} members`, + ); + + // Recalculate threshold with correct values + const calculatedThreshold = getThresholdForOperation( + operationName as string, + operationName.toLowerCase().includes("-") ? "group" : "core", + group.members as string, + currentGeo, + batchSize, + memberCount, // Pass the preserved member count + ); + + // Update the threshold in the data + data.threshold = calculatedThreshold; + console.debug(data.threshold); + const average = + data.values.reduce((sum: number, val: number) => sum + val, 0) / + data.values.length; + const min = Math.min(...(data.values as number[])); + const max = Math.max(...(data.values as number[])); + + const status = average <= data.threshold ? "PASS ✅" : "FAIL ❌"; + + // Calculate variance between average and threshold + const variance = Math.round(average - data.threshold); + const varianceFormatted = + variance <= 0 ? variance.toString() : `+${variance}`; + + console.debug("getThresholdForOperation inputs:", { + operationName, + operationType: operationName.includes("-") ? "group" : "core", + memberCountStr: group.members, + currentGeo, + batchSize, + actualMemberCount: memberCount, + }); + + fileContent += `${operationName} | ${group.members} | ${Math.round(average)} | ${Math.round(min)}/${Math.round(max)} | ${data.threshold} | ${varianceFormatted} | ${status}\n`; + } + + // Write to file + try { + fs.writeFileSync(filename, fileContent); + console.log(`📝 Metrics summary written to: ${filename}`); + } catch (error) { + console.error(`❌ Error writing metrics summary to file:`, error); + } + + // Reset metrics collection for next test run + collectedMetrics = {}; +} diff --git a/datadog/thresholds.json b/datadog/thresholds.json new file mode 100644 index 00000000..06a22723 --- /dev/null +++ b/datadog/thresholds.json @@ -0,0 +1,48 @@ +{ + "core": { + "clientcreate": 550, + "inboxstate": 300, + "createdm": 200, + "sendgm": 100, + "receivegm": 100, + "receivegroupmessage": 102, + "creategroup": 300, + "creategroupbyidentifiers": 300, + "sendgroupmessage": 56, + "syncgroup": 100, + "updategroupname": 110, + "removemembers": 250, + "addmembers": 250 + }, + "network": { + "dns_lookup": 50, + "tcp_connection": 50, + "tls_handshake": 150, + "processing": 100, + "server_call": 250 + }, + "memberMultipliers": { + "creategroup": 3, + "creategroupbyidentifiers": 3, + "sendgroupmessage": 1.3, + "syncgroup": 1.3, + "updategroupname": 1.3, + "removemembers": 1.3, + "receivegroupmessage": 1.3 + }, + "regionMultipliers": { + "us-east": 1.0, + "us-west": 1.0, + "europe": 1.0, + "asia": 1.5, + "south-america": 10 + }, + "GEO_TO_COUNTRY_CODE": { + "us-east": "US", + "us-west": "US", + "europe": "FR", + "asia": "JP", + "south-america": "BR" + }, + "reliability": 99.9 +} diff --git a/helpers/README.md b/helpers/README.md index c234a978..47c67d40 100644 --- a/helpers/README.md +++ b/helpers/README.md @@ -7,7 +7,6 @@ This directory contains utility modules that power the XMTP testing framework. T | Module | Purpose | | ------------------------------ | ------------------------------------------------- | | [client.ts](#clientts) | Creates signers and manages keys for test workers | -| [datadog.ts](#datadogts) | Sends performance metrics to Datadog | | [group.ts](#groupts) | Creates test groups with specified participants | | [logger.ts](#loggerts) | Logging utilities for test output | | [test.ts](#testts) | Test utilities for creating and managing tests | @@ -32,24 +31,6 @@ const dbPath = getDbPath(workerName, accountAddress, testName); const encryptionKey = generateEncryptionKeyHex(); ``` -### datadog.ts - -Sends performance metrics to Datadog for monitoring: - -```typescript -// Initialize Datadog metrics -initDataDog(testName, envValue, geolocation, apiKey); - -// Send delivery rate metrics -sendDeliveryMetric(deliveryRate, testName, libxmtpVersion); - -// Send performance metrics -sendPerformanceMetric(durationMs, testName, libxmtpVersion); - -// Measure network performance -const networkStats = await getNetworkStats(); -``` - ### group.ts Utilities for creating and managing test groups: diff --git a/helpers/client.ts b/helpers/client.ts index 0906b3b6..19965a43 100644 --- a/helpers/client.ts +++ b/helpers/client.ts @@ -8,7 +8,8 @@ import { fromString, toString } from "uint8arrays"; import { createWalletClient, http, toBytes } from "viem"; import { privateKeyToAccount } from "viem/accounts"; import { sepolia } from "viem/chains"; -import { flushMetrics, initDataDog } from "./datadog"; +import { b } from "vitest/dist/chunks/suite.qtkXWc6R.js"; +import { flushMetrics, initDataDog } from "../datadog/git add . && git commit -m "fix datadog summary" && git push;"; import { createLogger, flushLogger, overrideConsole } from "./logger"; interface User { @@ -147,10 +148,14 @@ export function getAddressOfMember(members: GroupMember[], inboxId: string) { } } -export async function closeEnv(testName: string, workers: WorkerManager) { +export async function closeEnv( + testName: string, + workers: WorkerManager, + batchSize?: number, +) { flushLogger(testName); - await flushMetrics(testName); + await flushMetrics(testName, batchSize); if (workers && typeof workers.getWorkers === "function") { for (const worker of workers.getWorkers()) { await worker.worker.terminate(); diff --git a/helpers/datadog.ts b/helpers/datadog.ts deleted file mode 100644 index 5bced41f..00000000 --- a/helpers/datadog.ts +++ /dev/null @@ -1,549 +0,0 @@ -import { exec } from "child_process"; -import * as fs from "fs"; -import * as path from "path"; -import { promisify } from "util"; -import type { WorkerManager } from "@workers/manager"; -import metrics from "datadog-metrics"; - -// Global state variables -let isInitialized = false; -let currentGeo: string = ""; -let collectedMetrics: Record< - string, - { values: number[]; threshold: number; members?: string } -> = {}; - -// Refactored thresholds into a single configuration object -const THRESHOLDS = { - core: { - clientcreate: 550, - inboxstate: 50, - createdm: 200, - sendgm: 100, - receivegm: 100, - receivegroupmessage: 102, - creategroup: 300, - creategroupbyidentifiers: 300, - sendgroupmessage: 56, - syncgroup: 100, - updategroupname: 110, - removemembers: 250, - addmembers: 250, - }, - network: { - dns_lookup: 50, - tcp_connection: 50, - tls_handshake: 150, - processing: 100, - server_call: 250, - }, - memberMultipliers: { - creategroup: 3, - creategroupbyidentifiers: 3, - sendgroupmessage: 1.3, - syncgroup: 1.3, - updategroupname: 1.3, - removemembers: 1.3, - receivegroupmessage: 1.3, - }, - - regionMultipliers: { - "us-east": 1.0, - "us-west": 1.0, - europe: 1.0, - asia: 1.5, - "south-america": 2, - }, - GEO_TO_COUNTRY_CODE: { - "us-east": "US", - "us-west": "US", - europe: "FR", - asia: "JP", - "south-america": "BR", - }, - reliability: 99.9, -}; - -function getCountryCodeFromGeo(geolocation: string): string { - return ( - THRESHOLDS.GEO_TO_COUNTRY_CODE[ - geolocation as keyof typeof THRESHOLDS.GEO_TO_COUNTRY_CODE - ] || "US" - ); -} - -// Simplified threshold function -export function getThresholdForOperation( - operation: string, - operationType: string = "core", - members: string = "", - region: string = "us-east", -): number { - // Convert operation to lowercase for consistent lookups - const operationLower = operation.toLowerCase(); - - // Get the region multiplier - const regionMultiplier = - THRESHOLDS.regionMultipliers[ - region as keyof typeof THRESHOLDS.regionMultipliers - ] || 1.0; - - if (operationType === "network") { - const networkThreshold = - THRESHOLDS.network[operationLower as keyof typeof THRESHOLDS.network]; - // Default to 200 if the specific network operation isn't found - const baseThreshold = - typeof networkThreshold === "number" ? networkThreshold : 200; - - // Apply region multiplier to network thresholds - const finalThreshold = Math.round(baseThreshold * regionMultiplier); - - return finalThreshold; - } - - if (operationType === "group") { - // Parse the member count, default to 50 if not provided or invalid - const memberCount = parseInt(members || "50", 10) || 50; - - // Get the base value for this operation (which is for 50 members) - const baseValue = - THRESHOLDS.core[operationLower as keyof typeof THRESHOLDS.core] || 500; - - // Get the multiplier for this operation - const memberMultiplier = - THRESHOLDS.memberMultipliers[ - operationLower as keyof typeof THRESHOLDS.memberMultipliers - ] || 0; - - let calculatedThreshold = baseValue; - - if (memberCount > 50) { - const additionalBatches = Math.floor((memberCount - 50) / 50); - calculatedThreshold = - baseValue * (1 + (memberMultiplier * additionalBatches) / 10); - console.log( - `calculatedThreshold: ${calculatedThreshold} for ${memberCount} members with multiplier ${memberMultiplier}`, - ); - } - - // Apply region multiplier - const finalThreshold = Math.round(calculatedThreshold * regionMultiplier); - - return finalThreshold; - } - - // For core operations, ensure we're using lowercase for lookup - const baseThreshold = - operationLower in THRESHOLDS.core - ? THRESHOLDS.core[operationLower as keyof typeof THRESHOLDS.core] - : 300; - - const finalThreshold = Math.round(baseThreshold * regionMultiplier); - - return finalThreshold; -} - -export const sendPerformanceResult = ( - expect: any, - workers: WorkerManager, - start: number, -) => { - const testName = expect.getState().currentTestName; - if (testName) { - console.timeEnd(testName as string); - expect(workers.getWorkers()).toBeDefined(); - expect(workers.getWorkers().length).toBeGreaterThan(0); - void sendPerformanceMetric( - performance.now() - start, - testName as string, - workers.getVersion(), - ); - } -}; -// Add success status tag to duration metrics - -export function initDataDog( - testName: string, - envValue: string, - geolocation: string, - apiKey: string, -): boolean { - if (isInitialized) { - return true; - } - if (!testName.includes("ts_")) { - return true; - } - if (!apiKey) { - console.warn("⚠️ DATADOG_API_KEY not found. Metrics will not be sent."); - return false; - } - - try { - const countryCode = getCountryCodeFromGeo(geolocation); - currentGeo = geolocation; - const initConfig = { - apiKey: apiKey, - defaultTags: [ - `env:${envValue}`, - `test:${testName}`, - `region:${geolocation}`, - `country_iso_code:${countryCode}`, - ], - }; - metrics.init(initConfig); - isInitialized = true; - return true; - } catch (error) { - console.error("❌ Failed to initialize DataDog metrics:", error); - return false; - } -} - -// Modify the sendMetric function to collect metrics for summary -export function sendMetric( - metricName: string, - metricValue: number, - tags: Record, -): void { - if (!isInitialized) return; - - try { - const fullMetricName = `xmtp.sdk.${metricName}`; - const allTags = Object.entries({ ...tags }).map( - ([key, value]) => `${key}:${String(value)}`, - ); - // Add environment tag if not already present - if (!allTags.some((tag) => tag.startsWith("env:"))) { - allTags.push(`env:${process.env.XMTP_ENV as string}`); - } - // Add version tag if not already present - if (!allTags.some((tag) => tag.startsWith("xv:"))) { - allTags.push(`vm:${process.env.RAILWAY_SERVICE_ID || "unknown"}`); - } - - // Track metrics for summary log - // Include members in the operation key if available - const operationKey = tags.operation - ? `${tags.operation}:${tags.metric_type || ""}${tags.members ? `:${tags.members}` : ""}` - : metricName; - - if (!collectedMetrics[operationKey]) { - collectedMetrics[operationKey] = { - values: [], - threshold: tags.threshold || 0, - members: tags.members || "-", // Store members info separately - }; - } - collectedMetrics[operationKey].values.push(metricValue); - - metrics.gauge(fullMetricName, Math.round(metricValue), allTags); - } catch (error) { - console.error( - `❌ Error sending metric '${metricName}':`, - error instanceof Error ? error.message : String(error), - ); - } -} - -export function sendTestResults(hasFailures: boolean, testName: string): void { - if (!isInitialized) { - console.warn("Datadog metrics not initialized"); - return; - } - try { - const metricValue = hasFailures ? 0 : 1; - const metricName = `workflow`; - sendMetric(metricName, metricValue, { - workflow: testName, - metric_type: "workflow", - }); - console.log( - `The tests indicated that the test ${testName} was ${hasFailures}`, - ); - } catch (error) { - console.error("Error reporting to Datadog:", error); - } -} - -// Simplified version of sendPerformanceMetric -export async function sendPerformanceMetric( - metricValue: number, - testName: string, - libxmtpVersion: string, - skipNetworkStats: boolean = false, -): Promise { - if (!isInitialized) return; - - try { - const metricNameParts = testName.split(":")[0]; - const metricName = metricNameParts.replaceAll(" > ", "."); - const metricDescription = testName.split(":")[1] || ""; - const operationParts = metricName.split("."); - const testNameExtracted = operationParts[0]; - const operationName = operationParts[1]?.split("-")[0] || ""; - const members = operationParts[1]?.split("-")[1] || ""; - - // Use a more reliable approach to determine if this is a group operation - const isGroupOperation = operationName.toLowerCase().includes("group"); - const operationType = isGroupOperation ? "group" : "core"; - - const threshold = getThresholdForOperation( - operationName, - operationType, - members, - currentGeo, - ); - const isSuccess = metricValue <= threshold; - - sendMetric("duration", metricValue, { - libxmtp: libxmtpVersion, - operation: operationName, - test: testNameExtracted, - metric_type: "operation", - metric_subtype: operationType, - description: metricDescription, - members: members, - success: isSuccess, - threshold: threshold, - region: currentGeo, - }); - - // Network stats handling - if (!skipNetworkStats) { - //ignore group operations - const networkStats = await getNetworkStats(); - const countryCode = getCountryCodeFromGeo(currentGeo); - - for (const [statName, statValue] of Object.entries(networkStats)) { - const networkMetricValue = Math.round(statValue * 1000); - const networkPhase = statName.toLowerCase().replace(/\s+/g, "_"); - const networkThreshold = getThresholdForOperation( - networkPhase, - "network", - members, - currentGeo, - ); - - sendMetric("duration", networkMetricValue, { - libxmtp: libxmtpVersion, - operation: operationName, - test: testNameExtracted, - metric_type: "network", - network_phase: networkPhase, - country_iso_code: countryCode, - members: members, - success: networkMetricValue <= networkThreshold, - threshold: networkThreshold, - region: currentGeo, - }); - } - } - } catch (error) { - console.error( - `❌ Error sending performance metric for '${testName}':`, - error, - ); - } -} - -/** - * Logs a summary of all collected metrics against their thresholds - * Call this at the end of test execution - */ -export function logMetricsSummary(testName: string): void { - if (!isInitialized || Object.keys(collectedMetrics).length === 0) { - console.log("No metrics collected to summarize"); - return; - } - - console.log("\n📊 Creating metrics summary report"); - - // Create a simple text summary for the console - const passedMetrics = Object.entries(collectedMetrics).filter( - ([operation, data]) => { - // Skip workflow metrics when counting passed metrics - if (operation === "workflow") return false; - - if (data.values.length === 0) return false; - const average = - data.values.reduce((sum, val) => sum + val, 0) / data.values.length; - return average <= data.threshold; - }, - ).length; - - // Count only non-workflow metrics for total - const totalMetrics = Object.entries(collectedMetrics).filter( - ([operation]) => operation !== "workflow", - ).length; - - console.log( - `✅ Passed: ${passedMetrics}/${totalMetrics} metrics (${Math.round((passedMetrics / totalMetrics) * 100)}%)`, - ); - - // Create a directory for reports if it doesn't exist - const reportsDir = path.join(process.cwd(), "logs/reports"); - if (!fs.existsSync(reportsDir)) { - fs.mkdirSync(reportsDir, { recursive: true }); - } - - // Create filename with timestamp - const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); - const filename = path.join( - reportsDir, - `${testName}-${currentGeo}-${process.env.XMTP_ENV}.md`, - ); - - // Build the table content - let fileContent = "METRICS SUMMARY\n===============\n\n"; - fileContent += - "Operation | Members | Samples | Avg (ms) | Min/Max (ms) | Threshold (ms) | Variance (ms) | Pass Rate | Status\n"; - fileContent += - "----------|---------|---------|----------|--------------|----------------|---------------|-----------|-------\n"; - - for (const [operation, data] of Object.entries(collectedMetrics)) { - // Skip workflow metrics in the summary table - if (operation === "workflow") continue; - - if (data.values.length === 0) continue; - - // Extract member count from operationKey - const parts = operation.split(":"); - // The member count should be the last segment if it exists - const memberCount = - parts.length > 2 ? parts[parts.length - 1] : data.members || "-"; - - const average = - data.values.reduce((sum, val) => sum + val, 0) / data.values.length; - const min = Math.min(...data.values); - const max = Math.max(...data.values); - const passRate = - (data.values.filter((v) => v <= data.threshold).length / - data.values.length) * - 100; - const status = average <= data.threshold ? "PASS ✅" : "FAIL ❌"; - - // Calculate variance between average and threshold - const variance = Math.round(average - data.threshold); - const varianceFormatted = - variance <= 0 ? variance.toString() : `+${variance}`; - - fileContent += `${operation} | ${memberCount} | ${data.values.length} | ${Math.round(average)} | ${Math.round(min)}/${Math.round(max)} | ${data.threshold} | ${varianceFormatted} | ${passRate.toFixed(1)}% | ${status}\n`; - } - - // Write to file - try { - fs.writeFileSync(filename, fileContent); - console.log(`📝 Metrics summary written to: ${filename}`); - } catch (error) { - console.error(`❌ Error writing metrics summary to file:`, error); - } - - // Reset metrics collection for next test run - collectedMetrics = {}; -} - -// Modify flushMetrics to include the summary log -export function flushMetrics(testName: string): Promise { - return new Promise((resolve) => { - if (!isInitialized) { - resolve(); - return; - } - - // Log metrics summary before flushing - logMetricsSummary(testName); - - void metrics.flush().then(() => { - resolve(); - }); - }); -} - -const execAsync = promisify(exec); - -/** - * Get network performance statistics for a specific endpoint - * @param endpoint The endpoint to monitor (defaults to XMTP gRPC endpoint) - * @returns Object containing timing information in seconds - */ -let firstLogShared = false; - -interface NetworkStats { - "DNS Lookup": number; - "TCP Connection": number; - "TLS Handshake": number; - Processing: number; - "Server Call": number; -} - -export async function getNetworkStats( - endpoint = "https://grpc.dev.xmtp.network:443", -): Promise { - const curlCommand = `curl -s -w "\\n{\\"DNS Lookup\\": %{time_namelookup}, \\"TCP Connection\\": %{time_connect}, \\"TLS Handshake\\": %{time_appconnect}, \\"Server Call\\": %{time_starttransfer}}" -o /dev/null --max-time 10 ${endpoint}`; - - let stdout: string; - - try { - const result = await execAsync(curlCommand); - stdout = result.stdout; - } catch (error: any) { - // Even if curl returns an error, we might still have useful stdout - if (error.stdout) { - stdout = error.stdout; - console.warn( - `⚠️ Curl command returned error code ${error.code}, but stdout is available.`, - ); - } else { - console.error(`❌ Curl command failed without stdout:`, error); - throw error; // rethrow if no stdout is available - } - } - - // Parse the JSON response - const stats = JSON.parse(stdout.trim()) as NetworkStats; - - // Handle the case where Server Call time is 0 - if (stats["Server Call"] === 0) { - console.warn( - `Network request to ${endpoint} returned Server Call time of 0.`, - ); - - // Use a reasonable estimate - stats["Server Call"] = stats["TLS Handshake"] + 0.1; - } - - // Calculate processing time - stats["Processing"] = stats["Server Call"] - stats["TLS Handshake"]; - - // Ensure processing time is not negative - if (stats["Processing"] < 0) { - stats["Processing"] = 0; - } - - return stats; -} - -// Unified delivery metrics function -export function sendDeliveryMetric( - metricValue: number, - version: string, - testName: string, - metricSubType: "stream" | "poll" | "recovery", - metricType: "delivery" | "order", -): void { - // Determine success based on the metric subtype - const threshold = THRESHOLDS.reliability; - - const isSuccess = metricValue >= threshold; - - // Send primary metric - sendMetric(metricType, Math.round(metricValue), { - libxmtp: version, - test: testName, - metric_type: metricType, - metric_subtype: metricSubType, - success: isSuccess, - threshold: threshold, - }); -} diff --git a/scripts/railway-test.ts b/scripts/railway-test.ts index fd1ce6b0..13c6539e 100755 --- a/scripts/railway-test.ts +++ b/scripts/railway-test.ts @@ -1,5 +1,5 @@ import { execSync } from "child_process"; -import { sendTestResults } from "@helpers/datadog"; +import { sendTestResults } from "@datadog/git add . && git commit -m "fix datadog summary" && git push;"; function runTests(): void { let hasFailures: boolean = false; diff --git a/tests/TS_Delivery.test.ts b/tests/TS_Delivery.test.ts index d3cd3efe..2829ffed 100644 --- a/tests/TS_Delivery.test.ts +++ b/tests/TS_Delivery.test.ts @@ -1,5 +1,5 @@ +import { sendDeliveryMetric, sendTestResults } from "@datadog/git add . && git commit -m "fix datadog summary" && git push;"; import { closeEnv, loadEnv } from "@helpers/client"; -import { sendDeliveryMetric, sendTestResults } from "@helpers/datadog"; import { logError } from "@helpers/logger"; import { defaultValues, diff --git a/tests/TS_Gm.test.ts b/tests/TS_Gm.test.ts index 596e11b3..ae44d17c 100644 --- a/tests/TS_Gm.test.ts +++ b/tests/TS_Gm.test.ts @@ -1,5 +1,5 @@ +import { sendTestResults } from "@datadog/git add . && git commit -m "fix datadog summary" && git push;"; import { closeEnv, loadEnv } from "@helpers/client"; -import { sendTestResults } from "@helpers/datadog"; import generatedInboxes from "@helpers/generated-inboxes.json"; import { logError } from "@helpers/logger"; import { createGroupAndReceiveGm } from "@helpers/playwright"; diff --git a/tests/TS_Performance.test.ts b/tests/TS_Performance.test.ts index c064534e..e235a09a 100644 --- a/tests/TS_Performance.test.ts +++ b/tests/TS_Performance.test.ts @@ -1,5 +1,5 @@ +import { sendPerformanceResult, sendTestResults } from "@datadog/git add . && git commit -m "fix datadog summary" && git push;"; import { closeEnv, loadEnv } from "@helpers/client"; -import { sendPerformanceResult, sendTestResults } from "@helpers/datadog"; import generatedInboxes from "@helpers/generated-inboxes.json"; import { logError } from "@helpers/logger"; import { @@ -68,7 +68,7 @@ describe(testName, () => { afterEach(function () { try { - sendPerformanceResult(expect, workers, start); + sendPerformanceResult(expect, workers, start, batchSize, total); } catch (e) { hasFailures = logError(e, expect); throw e; @@ -78,7 +78,7 @@ describe(testName, () => { afterAll(async () => { try { sendTestResults(hasFailures, testName); - await closeEnv(testName, workers); + await closeEnv(testName, workers, batchSize); } catch (e) { hasFailures = logError(e, expect); throw e; diff --git a/tests/dms.test.ts b/tests/dms.test.ts index ee039a55..89620802 100644 --- a/tests/dms.test.ts +++ b/tests/dms.test.ts @@ -1,5 +1,5 @@ +import { sendPerformanceResult, sendTestResults } from "@datadog/git add . && git commit -m "fix datadog summary" && git push;"; import { closeEnv, loadEnv } from "@helpers/client"; -import { sendPerformanceResult, sendTestResults } from "@helpers/datadog"; import { logError } from "@helpers/logger"; import { type Conversation, type WorkerManager } from "@helpers/types"; import { verifyStream } from "@helpers/verify"; diff --git a/tests/groups.test.ts b/tests/groups.test.ts index 6bf60b65..0226527f 100644 --- a/tests/groups.test.ts +++ b/tests/groups.test.ts @@ -1,5 +1,5 @@ +import { sendPerformanceResult, sendTestResults } from "@datadog/git add . && git commit -m "fix datadog summary" && git push;"; import { closeEnv, loadEnv } from "@helpers/client"; -import { sendPerformanceResult, sendTestResults } from "@helpers/datadog"; import generatedInboxes from "@helpers/generated-inboxes.json"; import { logError } from "@helpers/logger"; import { diff --git a/tests/offline.test.ts b/tests/offline.test.ts index 0d5fd063..e174acda 100644 --- a/tests/offline.test.ts +++ b/tests/offline.test.ts @@ -1,5 +1,5 @@ +import { sendDeliveryMetric } from "@datadog/git add . && git commit -m "fix datadog summary" && git push;"; import { closeEnv, loadEnv } from "@helpers/client"; -import { sendDeliveryMetric } from "@helpers/datadog"; import { logError } from "@helpers/logger"; import { type Group, type WorkerManager } from "@helpers/types"; import { calculateMessageStats } from "@helpers/verify"; diff --git a/tsconfig.json b/tsconfig.json index 462745de..9d3b1cde 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -21,6 +21,7 @@ "paths": { "@helpers/*": ["./helpers/*"], "@scripts/*": ["./scripts/*"], + "@datadog/*": ["./datadog/*"], "@workers/*": ["./workers/*"] } }, diff --git a/vitest.config.ts b/vitest.config.ts index aa7ce759..1107724f 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -8,6 +8,7 @@ export default defineConfig({ "@helpers": resolve(__dirname, "./helpers"), "@workers": resolve(__dirname, "./workers"), "@scripts": resolve(__dirname, "./scripts"), + "@datadog": resolve(__dirname, "./datadog"), }, }, test: { From 5e96ba8ef34e367b4830688f9c852dde8524d39e Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 20:37:36 -0300 Subject: [PATCH 30/57] fi bug --- helpers/client.ts | 2 +- scripts/railway-test.ts | 2 +- tests/TS_Delivery.test.ts | 2 +- tests/TS_Gm.test.ts | 2 +- tests/TS_Performance.test.ts | 2 +- tests/dms.test.ts | 2 +- tests/groups.test.ts | 2 +- tests/offline.test.ts | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/helpers/client.ts b/helpers/client.ts index 19965a43..f0fcf7a8 100644 --- a/helpers/client.ts +++ b/helpers/client.ts @@ -9,7 +9,7 @@ import { createWalletClient, http, toBytes } from "viem"; import { privateKeyToAccount } from "viem/accounts"; import { sepolia } from "viem/chains"; import { b } from "vitest/dist/chunks/suite.qtkXWc6R.js"; -import { flushMetrics, initDataDog } from "../datadog/git add . && git commit -m "fix datadog summary" && git push;"; +import { flushMetrics, initDataDog } from "../datadog/helper"; import { createLogger, flushLogger, overrideConsole } from "./logger"; interface User { diff --git a/scripts/railway-test.ts b/scripts/railway-test.ts index 13c6539e..074530e4 100755 --- a/scripts/railway-test.ts +++ b/scripts/railway-test.ts @@ -1,5 +1,5 @@ import { execSync } from "child_process"; -import { sendTestResults } from "@datadog/git add . && git commit -m "fix datadog summary" && git push;"; +import { sendTestResults } from "@datadog/helper"; function runTests(): void { let hasFailures: boolean = false; diff --git a/tests/TS_Delivery.test.ts b/tests/TS_Delivery.test.ts index 2829ffed..1b3c5063 100644 --- a/tests/TS_Delivery.test.ts +++ b/tests/TS_Delivery.test.ts @@ -1,4 +1,4 @@ -import { sendDeliveryMetric, sendTestResults } from "@datadog/git add . && git commit -m "fix datadog summary" && git push;"; +import { sendDeliveryMetric, sendTestResults } from "@datadog/helper"; import { closeEnv, loadEnv } from "@helpers/client"; import { logError } from "@helpers/logger"; import { diff --git a/tests/TS_Gm.test.ts b/tests/TS_Gm.test.ts index ae44d17c..e0d55c90 100644 --- a/tests/TS_Gm.test.ts +++ b/tests/TS_Gm.test.ts @@ -1,4 +1,4 @@ -import { sendTestResults } from "@datadog/git add . && git commit -m "fix datadog summary" && git push;"; +import { sendTestResults } from "@datadog/helper"; import { closeEnv, loadEnv } from "@helpers/client"; import generatedInboxes from "@helpers/generated-inboxes.json"; import { logError } from "@helpers/logger"; diff --git a/tests/TS_Performance.test.ts b/tests/TS_Performance.test.ts index e235a09a..350842a1 100644 --- a/tests/TS_Performance.test.ts +++ b/tests/TS_Performance.test.ts @@ -1,4 +1,4 @@ -import { sendPerformanceResult, sendTestResults } from "@datadog/git add . && git commit -m "fix datadog summary" && git push;"; +import { sendPerformanceResult, sendTestResults } from "@datadog/helper"; import { closeEnv, loadEnv } from "@helpers/client"; import generatedInboxes from "@helpers/generated-inboxes.json"; import { logError } from "@helpers/logger"; diff --git a/tests/dms.test.ts b/tests/dms.test.ts index 89620802..16bd0139 100644 --- a/tests/dms.test.ts +++ b/tests/dms.test.ts @@ -1,4 +1,4 @@ -import { sendPerformanceResult, sendTestResults } from "@datadog/git add . && git commit -m "fix datadog summary" && git push;"; +import { sendPerformanceResult, sendTestResults } from "@datadog/helper"; import { closeEnv, loadEnv } from "@helpers/client"; import { logError } from "@helpers/logger"; import { type Conversation, type WorkerManager } from "@helpers/types"; diff --git a/tests/groups.test.ts b/tests/groups.test.ts index 0226527f..4bae9446 100644 --- a/tests/groups.test.ts +++ b/tests/groups.test.ts @@ -1,4 +1,4 @@ -import { sendPerformanceResult, sendTestResults } from "@datadog/git add . && git commit -m "fix datadog summary" && git push;"; +import { sendPerformanceResult, sendTestResults } from "@datadog/helper"; import { closeEnv, loadEnv } from "@helpers/client"; import generatedInboxes from "@helpers/generated-inboxes.json"; import { logError } from "@helpers/logger"; diff --git a/tests/offline.test.ts b/tests/offline.test.ts index e174acda..c07b1180 100644 --- a/tests/offline.test.ts +++ b/tests/offline.test.ts @@ -1,4 +1,4 @@ -import { sendDeliveryMetric } from "@datadog/git add . && git commit -m "fix datadog summary" && git push;"; +import { sendDeliveryMetric } from "@datadog/helper"; import { closeEnv, loadEnv } from "@helpers/client"; import { logError } from "@helpers/logger"; import { type Group, type WorkerManager } from "@helpers/types"; From bccc7ff9bab8ffda295c314a1a986ecd9de2270e Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 20:44:52 -0300 Subject: [PATCH 31/57] fi bug --- datadog/thresholds.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/datadog/thresholds.json b/datadog/thresholds.json index 06a22723..d326a24a 100644 --- a/datadog/thresholds.json +++ b/datadog/thresholds.json @@ -1,18 +1,18 @@ { "core": { - "clientcreate": 550, - "inboxstate": 300, + "clientcreate": 350, + "inboxstate": 200, "createdm": 200, "sendgm": 100, "receivegm": 100, "receivegroupmessage": 102, - "creategroup": 300, - "creategroupbyidentifiers": 300, + "creategroup": 200, + "creategroupbyidentifiers": 150, "sendgroupmessage": 56, "syncgroup": 100, "updategroupname": 110, - "removemembers": 250, - "addmembers": 250 + "removemembers": 100, + "addmembers": 100 }, "network": { "dns_lookup": 50, @@ -22,12 +22,12 @@ "server_call": 250 }, "memberMultipliers": { - "creategroup": 3, - "creategroupbyidentifiers": 3, + "creategroup": 1.5, + "creategroupbyidentifiers": 1.5, "sendgroupmessage": 1.3, "syncgroup": 1.3, "updategroupname": 1.3, - "removemembers": 1.3, + "removemembers": 0.8, "receivegroupmessage": 1.3 }, "regionMultipliers": { From acf27c40bc265180652f78230e11dd5617c9b165 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 20:52:45 -0300 Subject: [PATCH 32/57] fix tresshold --- datadog/calculations.ts | 53 ++++++++++++++++++++++++++++------------- datadog/helper.ts | 4 ---- datadog/summary.ts | 2 -- datadog/thresholds.json | 16 ++++++------- 4 files changed, 44 insertions(+), 31 deletions(-) diff --git a/datadog/calculations.ts b/datadog/calculations.ts index 36f2e397..f0d32d22 100644 --- a/datadog/calculations.ts +++ b/datadog/calculations.ts @@ -6,12 +6,15 @@ export function getThresholdForOperation( operationType: "core" | "group" | "network", members: string = "", region: string = "us-east", - batchSize: number = 10, - total: number = 10, ): number { // Convert operation to lowercase for consistent lookups const operationLower = operation.toLowerCase(); + // Extract batch size from operation name if it contains a dash + const batchSize = operation.includes("-") + ? parseInt(operation.split("-")[1]) || 1 + : 1; + // Normalize region name and ensure it exists in the thresholds const regionNormalized = region.toLowerCase().trim(); @@ -31,9 +34,11 @@ export function getThresholdForOperation( if (operationType === "network") { const networkThreshold = THRESHOLDS.network[operationLower as keyof typeof THRESHOLDS.network]; - // Default to 200 if the specific network operation isn't found + // Default to network threshold based on the operation or fallback value const baseThreshold = - typeof networkThreshold === "number" ? networkThreshold : 200; + typeof networkThreshold === "number" + ? networkThreshold + : THRESHOLDS.network.server_call; // Apply region multiplier to network thresholds const finalThreshold = Math.round(baseThreshold * regionMultiplier); @@ -44,7 +49,7 @@ export function getThresholdForOperation( if (operationType === "group") { // Parse the member count from the members string if available // Use a more robust parsing approach - let memberCount = total; // Default to total + let memberCount = 0; if (members && members !== "-") { const parsedMember = parseInt(members); @@ -70,12 +75,14 @@ export function getThresholdForOperation( let calculatedThreshold = baseValue; // Calculate based on the actual member count - const batches = Math.ceil(memberCount / batchSize); - calculatedThreshold = baseValue * (1 + memberMultiplier * (batches - 1)); + if (memberCount > 0) { + const batches = Math.ceil(memberCount / batchSize); + calculatedThreshold = baseValue * (1 + memberMultiplier * (batches - 1)); - console.log( - `Operation: ${operation}, Using memberCount: ${memberCount} (from members: ${members}), batches: ${batches}, threshold: ${calculatedThreshold}`, - ); + console.log( + `Operation: ${operation}, Using memberCount: ${memberCount} (from members: ${members}), batches: ${batches}, threshold: ${calculatedThreshold}`, + ); + } // Apply region multiplier const finalThreshold = Math.round(calculatedThreshold * regionMultiplier); @@ -87,7 +94,7 @@ export function getThresholdForOperation( const baseThreshold = operationLower in THRESHOLDS.core ? THRESHOLDS.core[operationLower as keyof typeof THRESHOLDS.core] - : 300; + : null; // Check if this is a group operation that needs member multiplier const isGroupOperation = operationLower in THRESHOLDS.memberMultipliers; @@ -100,18 +107,30 @@ export function getThresholdForOperation( ] || 0; let calculatedThreshold = baseThreshold; + let memberCount = 0; + + if (members && members !== "-") { + const parsedMember = parseInt(members); + if (!isNaN(parsedMember)) { + memberCount = parsedMember; + } + } - // Always calculate based on batch size and total - const batches = Math.ceil(total / batchSize); - calculatedThreshold = - baseThreshold * (1 + memberMultiplier * (batches - 1)); + // Calculate based on batch size and member count + if (memberCount > 0) { + const batches = Math.ceil(memberCount / batchSize); + calculatedThreshold = + (baseThreshold ?? 0) * (1 + memberMultiplier * (batches - 1)); + } // Apply region multiplier - const finalThreshold = Math.round(calculatedThreshold * regionMultiplier); + const finalThreshold = Math.round( + (calculatedThreshold ?? 0) * regionMultiplier, + ); return finalThreshold; } // For regular core operations - const finalThreshold = Math.round(baseThreshold * regionMultiplier); + const finalThreshold = Math.round((baseThreshold ?? 0) * regionMultiplier); return finalThreshold; } diff --git a/datadog/helper.ts b/datadog/helper.ts index 3c06bde8..50739429 100644 --- a/datadog/helper.ts +++ b/datadog/helper.ts @@ -212,8 +212,6 @@ export async function sendPerformanceMetric( operationType, members, currentGeo, - batchSize ?? undefined, - memberCount, // Pass the actual member count here ); const isSuccess = metricValue <= threshold; @@ -253,8 +251,6 @@ export async function sendPerformanceMetric( "network", members, currentGeo, - batchSize, - total, ); sendMetric("duration", networkMetricValue, { diff --git a/datadog/summary.ts b/datadog/summary.ts index c8f0c4a9..fffc7b1f 100644 --- a/datadog/summary.ts +++ b/datadog/summary.ts @@ -130,8 +130,6 @@ export function logMetricsSummary( operationName.toLowerCase().includes("-") ? "group" : "core", group.members as string, currentGeo, - batchSize, - memberCount, // Pass the preserved member count ); // Update the threshold in the data diff --git a/datadog/thresholds.json b/datadog/thresholds.json index d326a24a..067499b1 100644 --- a/datadog/thresholds.json +++ b/datadog/thresholds.json @@ -3,16 +3,16 @@ "clientcreate": 350, "inboxstate": 200, "createdm": 200, - "sendgm": 100, - "receivegm": 100, - "receivegroupmessage": 102, + "sendgm": 200, + "receivegm": 200, + "receivegroupmessage": 200, "creategroup": 200, - "creategroupbyidentifiers": 150, - "sendgroupmessage": 56, + "creategroupbyidentifiers": 200, + "sendgroupmessage": 200, "syncgroup": 100, - "updategroupname": 110, - "removemembers": 100, - "addmembers": 100 + "updategroupname": 200, + "removemembers": 200, + "addmembers": 200 }, "network": { "dns_lookup": 50, From 31922945040ecbb412ad7d451f76e422d397b677 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 21:18:48 -0300 Subject: [PATCH 33/57] test --- datadog/calculations.ts | 119 ++++++----------------------------- datadog/helper.ts | 46 +------------- datadog/summary.ts | 6 -- datadog/thresholds.json | 2 +- helpers/client.ts | 8 +-- tests/TS_Performance.test.ts | 4 +- yarn.lock | 6 +- 7 files changed, 30 insertions(+), 161 deletions(-) diff --git a/datadog/calculations.ts b/datadog/calculations.ts index f0d32d22..48279f53 100644 --- a/datadog/calculations.ts +++ b/datadog/calculations.ts @@ -1,136 +1,57 @@ import THRESHOLDS from "./thresholds.json"; -// Simplified threshold function export function getThresholdForOperation( operation: string, operationType: "core" | "group" | "network", members: string = "", region: string = "us-east", ): number { - // Convert operation to lowercase for consistent lookups + // Normalize inputs const operationLower = operation.toLowerCase(); - - // Extract batch size from operation name if it contains a dash + const regionNormalized = region.toLowerCase().trim(); const batchSize = operation.includes("-") ? parseInt(operation.split("-")[1]) || 1 : 1; - // Normalize region name and ensure it exists in the thresholds - const regionNormalized = region.toLowerCase().trim(); - - // Get the region multiplier with proper logging + // Get region multiplier const regionMultiplier = THRESHOLDS.regionMultipliers[ regionNormalized as keyof typeof THRESHOLDS.regionMultipliers ] || 1.0; - // Log if region multiplier not found (for debugging) - if (!(regionNormalized in THRESHOLDS.regionMultipliers)) { - console.warn( - `Region multiplier not found for: ${regionNormalized}, using default 1.0`, - ); - } + // Parse member count + const memberCount = members && members !== "-" ? parseInt(members) || 0 : 0; + + // Calculate base threshold based on operation type + let baseThreshold: number; if (operationType === "network") { const networkThreshold = THRESHOLDS.network[operationLower as keyof typeof THRESHOLDS.network]; - // Default to network threshold based on the operation or fallback value - const baseThreshold = + baseThreshold = typeof networkThreshold === "number" ? networkThreshold : THRESHOLDS.network.server_call; + } else { + // For both "core" and "group" types, get the base value + baseThreshold = + THRESHOLDS.core[operationLower as keyof typeof THRESHOLDS.core] || 0; - // Apply region multiplier to network thresholds - const finalThreshold = Math.round(baseThreshold * regionMultiplier); - - return finalThreshold; - } - - if (operationType === "group") { - // Parse the member count from the members string if available - // Use a more robust parsing approach - let memberCount = 0; - - if (members && members !== "-") { - const parsedMember = parseInt(members); - if (!isNaN(parsedMember)) { - memberCount = parsedMember; - } - } - - console.log( - `Calculating threshold for ${operation} (${operationType}) with ${memberCount} members`, - ); - - // Get the base value for this operation - const baseValue = - THRESHOLDS.core[operationLower as keyof typeof THRESHOLDS.core]; - - // Get the multiplier for this operation + // Apply member multiplier if applicable const memberMultiplier = THRESHOLDS.memberMultipliers[ operationLower as keyof typeof THRESHOLDS.memberMultipliers ] || 0; - let calculatedThreshold = baseValue; - - // Calculate based on the actual member count - if (memberCount > 0) { + if (memberMultiplier > 0 && memberCount > 0) { const batches = Math.ceil(memberCount / batchSize); - calculatedThreshold = baseValue * (1 + memberMultiplier * (batches - 1)); - - console.log( - `Operation: ${operation}, Using memberCount: ${memberCount} (from members: ${members}), batches: ${batches}, threshold: ${calculatedThreshold}`, + baseThreshold = baseThreshold * (1 + memberMultiplier * (batches - 1)); + console.warn( + `Operation: ${operation}, member count: ${memberCount}, batches: ${batches}, multiplier: ${memberMultiplier}, adjusted threshold: ${baseThreshold}`, ); } - - // Apply region multiplier - const finalThreshold = Math.round(calculatedThreshold * regionMultiplier); - - return finalThreshold; - } - - // For core operations, ensure we're using lowercase for lookup - const baseThreshold = - operationLower in THRESHOLDS.core - ? THRESHOLDS.core[operationLower as keyof typeof THRESHOLDS.core] - : null; - - // Check if this is a group operation that needs member multiplier - const isGroupOperation = operationLower in THRESHOLDS.memberMultipliers; - - if (isGroupOperation && members) { - // Get the multiplier for this operation - const memberMultiplier = - THRESHOLDS.memberMultipliers[ - operationLower as keyof typeof THRESHOLDS.memberMultipliers - ] || 0; - - let calculatedThreshold = baseThreshold; - let memberCount = 0; - - if (members && members !== "-") { - const parsedMember = parseInt(members); - if (!isNaN(parsedMember)) { - memberCount = parsedMember; - } - } - - // Calculate based on batch size and member count - if (memberCount > 0) { - const batches = Math.ceil(memberCount / batchSize); - calculatedThreshold = - (baseThreshold ?? 0) * (1 + memberMultiplier * (batches - 1)); - } - - // Apply region multiplier - const finalThreshold = Math.round( - (calculatedThreshold ?? 0) * regionMultiplier, - ); - return finalThreshold; } - // For regular core operations - const finalThreshold = Math.round((baseThreshold ?? 0) * regionMultiplier); - return finalThreshold; + // Apply region multiplier and round + return Math.round(baseThreshold * regionMultiplier); } diff --git a/datadog/helper.ts b/datadog/helper.ts index 50739429..b27de019 100644 --- a/datadog/helper.ts +++ b/datadog/helper.ts @@ -25,8 +25,6 @@ export const sendPerformanceResult = ( expect: any, workers: WorkerManager, start: number, - batchSize?: number, - total?: number, ) => { const testName = expect.getState().currentTestName; if (testName) { @@ -38,8 +36,6 @@ export const sendPerformanceResult = ( testName as string, workers.getVersion(), false, - batchSize ?? undefined, - total ?? undefined, ); } }; @@ -92,10 +88,6 @@ export function sendMetric( if (!isInitialized) return; try { - console.log( - `SENDING METRIC: ${metricName}, Value: ${metricValue}, Members: ${tags.members || "none"}`, - ); - const fullMetricName = `xmtp.sdk.${metricName}`; const allTags = Object.entries({ ...tags }).map( ([key, value]) => `${key}:${String(value)}`, @@ -116,11 +108,6 @@ export function sendMetric( ? `${tags.operation}${memberCount ? `-${memberCount}` : ""}` : metricName; - // For debugging - console.log( - `METRIC KEY: ${operationKey} (Operation: ${tags.operation}, Members: ${memberCount})`, - ); - if (!collectedMetrics[operationKey]) { collectedMetrics[operationKey] = { values: [], @@ -151,9 +138,6 @@ export function sendTestResults(hasFailures: boolean, testName: string): void { workflow: testName, metric_type: "workflow", }); - console.log( - `The tests indicated that the test ${testName} was ${hasFailures}`, - ); } catch (error) { console.error("Error reporting to Datadog:", error); } @@ -165,8 +149,6 @@ export async function sendPerformanceMetric( testName: string, libxmtpVersion: string, skipNetworkStats: boolean = false, - batchSize?: number, - total?: number, ): Promise { if (!isInitialized) return; @@ -197,16 +179,9 @@ export async function sendPerformanceMetric( } } - // Debug output - console.log( - `Extracted from test name: Operation=${operationName}, Members=${members}, Test=${testNameExtracted}`, - ); - const isGroupOperation = operationName.toLowerCase().includes("-"); const operationType = isGroupOperation ? "group" : "core"; - // Use the extracted member count for threshold calculation - const memberCount = members ? parseInt(members) : total; const threshold = getThresholdForOperation( operationName, operationType, @@ -216,14 +191,6 @@ export async function sendPerformanceMetric( const isSuccess = metricValue <= threshold; - console.log(`Operation metrics:`, { - operation: operationName, - members: members, - testName: testName, - }); - - console.log(`Running test with ${memberCount} members`); - sendMetric("duration", metricValue, { libxmtp: libxmtpVersion, operation: operationName, @@ -276,23 +243,14 @@ export async function sendPerformanceMetric( } // Modify flushMetrics to include the summary log -export function flushMetrics( - testName: string, - batchSize?: number, -): Promise { +export function flushMetrics(testName: string): Promise { return new Promise((resolve) => { if (!isInitialized) { resolve(); return; } - logMetricsSummary( - testName, - currentGeo, - isInitialized, - collectedMetrics, - batchSize ?? undefined, - ); + logMetricsSummary(testName, currentGeo, isInitialized, collectedMetrics); void metrics.flush().then(() => { resolve(); diff --git a/datadog/summary.ts b/datadog/summary.ts index fffc7b1f..a84d84b3 100644 --- a/datadog/summary.ts +++ b/datadog/summary.ts @@ -27,12 +27,6 @@ export function logMetricsSummary( console.log("\n📊 Creating metrics summary report"); - // ADD THIS: Log the raw collected metrics to see what's actually there - console.log( - "RAW COLLECTED METRICS:", - JSON.stringify(collectedMetrics, null, 2), - ); - // Create a simple text summary for the console const passedMetrics = Object.entries(collectedMetrics).filter( ([operation, data]) => { diff --git a/datadog/thresholds.json b/datadog/thresholds.json index 067499b1..01bbfdfb 100644 --- a/datadog/thresholds.json +++ b/datadog/thresholds.json @@ -9,7 +9,7 @@ "creategroup": 200, "creategroupbyidentifiers": 200, "sendgroupmessage": 200, - "syncgroup": 100, + "syncgroup": 200, "updategroupname": 200, "removemembers": 200, "addmembers": 200 diff --git a/helpers/client.ts b/helpers/client.ts index f0fcf7a8..6507886d 100644 --- a/helpers/client.ts +++ b/helpers/client.ts @@ -148,14 +148,10 @@ export function getAddressOfMember(members: GroupMember[], inboxId: string) { } } -export async function closeEnv( - testName: string, - workers: WorkerManager, - batchSize?: number, -) { +export async function closeEnv(testName: string, workers: WorkerManager) { flushLogger(testName); - await flushMetrics(testName, batchSize); + await flushMetrics(testName); if (workers && typeof workers.getWorkers === "function") { for (const worker of workers.getWorkers()) { await worker.worker.terminate(); diff --git a/tests/TS_Performance.test.ts b/tests/TS_Performance.test.ts index 350842a1..8fd05bd6 100644 --- a/tests/TS_Performance.test.ts +++ b/tests/TS_Performance.test.ts @@ -68,7 +68,7 @@ describe(testName, () => { afterEach(function () { try { - sendPerformanceResult(expect, workers, start, batchSize, total); + sendPerformanceResult(expect, workers, start); } catch (e) { hasFailures = logError(e, expect); throw e; @@ -78,7 +78,7 @@ describe(testName, () => { afterAll(async () => { try { sendTestResults(hasFailures, testName); - await closeEnv(testName, workers, batchSize); + await closeEnv(testName, workers); } catch (e) { hasFailures = logError(e, expect); throw e; diff --git a/yarn.lock b/yarn.lock index 841fbafd..3b74d7c2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1357,9 +1357,9 @@ __metadata: linkType: hard "@xmtp/node-bindings@npm:^1.0.0-rc2": - version: 1.0.0-rc3 - resolution: "@xmtp/node-bindings@npm:1.0.0-rc3" - checksum: 10/b9afdfd4ca409778c6845acf342ba53220829be2e1b1c818cdc57fd6611c39930a07038d78ff31f8e06c6756bcd3996dfae04b695064639541295d112b977ae8 + version: 1.0.0 + resolution: "@xmtp/node-bindings@npm:1.0.0" + checksum: 10/8bf91003ea3859f1c535f871013d13eb1096728f51e3c81471eb5ad9a27de65a19a9817eb8efbd56e257c07076af94771845643211764ff028103e202779f6ce languageName: node linkType: hard From d925183b50e83f0440f61af616e77ed0ddb0eefc Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 23:18:25 -0300 Subject: [PATCH 34/57] adjusted batch sixe --- datadog/calculations.ts | 10 +++++++--- datadog/summary.ts | 16 +++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/datadog/calculations.ts b/datadog/calculations.ts index 48279f53..55b71751 100644 --- a/datadog/calculations.ts +++ b/datadog/calculations.ts @@ -44,10 +44,14 @@ export function getThresholdForOperation( ] || 0; if (memberMultiplier > 0 && memberCount > 0) { - const batches = Math.ceil(memberCount / batchSize); - baseThreshold = baseThreshold * (1 + memberMultiplier * (batches - 1)); + const batchNumber = Math.ceil(memberCount / batchSize); + + // Apply the multiplier based on batch number + // Batch 1 gets multiplier x1, Batch 2 gets x2, etc. + baseThreshold = baseThreshold * batchNumber * memberMultiplier; + console.warn( - `Operation: ${operation}, member count: ${memberCount}, batches: ${batches}, multiplier: ${memberMultiplier}, adjusted threshold: ${baseThreshold}`, + `Operation: ${operation}, member count: ${memberCount}, batch: ${batchNumber}, multiplier: ${memberMultiplier}x, adjusted threshold: ${baseThreshold}`, ); } } diff --git a/datadog/summary.ts b/datadog/summary.ts index a84d84b3..a1178d18 100644 --- a/datadog/summary.ts +++ b/datadog/summary.ts @@ -18,7 +18,6 @@ export function logMetricsSummary( members?: string; } >, - batchSize?: number, ): void { if (!isInitialized || Object.keys(collectedMetrics).length === 0) { console.log("No metrics collected to summarize"); @@ -142,14 +141,13 @@ export function logMetricsSummary( const varianceFormatted = variance <= 0 ? variance.toString() : `+${variance}`; - console.debug("getThresholdForOperation inputs:", { - operationName, - operationType: operationName.includes("-") ? "group" : "core", - memberCountStr: group.members, - currentGeo, - batchSize, - actualMemberCount: memberCount, - }); + // console.debug("getThresholdForOperation inputs:", { + // operationName, + // operationType: operationName.includes("-") ? "group" : "core", + // memberCountStr: group.members, + // currentGeo, + // actualMemberCount: memberCount, + // }); fileContent += `${operationName} | ${group.members} | ${Math.round(average)} | ${Math.round(min)}/${Math.round(max)} | ${data.threshold} | ${varianceFormatted} | ${status}\n`; } From bfecd41bf342f0f1a0cd6cc58eb09d9d6eade067 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 23:42:34 -0300 Subject: [PATCH 35/57] adjust --- datadog/calculations.ts | 36 +++++++++++++++--------------------- datadog/summary.ts | 2 +- datadog/thresholds.json | 16 ++++++++-------- 3 files changed, 24 insertions(+), 30 deletions(-) diff --git a/datadog/calculations.ts b/datadog/calculations.ts index 55b71751..ea5818d5 100644 --- a/datadog/calculations.ts +++ b/datadog/calculations.ts @@ -1,17 +1,15 @@ import THRESHOLDS from "./thresholds.json"; +let batches = new Set(); export function getThresholdForOperation( operation: string, operationType: "core" | "group" | "network", - members: string = "", + members: number = 0, region: string = "us-east", ): number { // Normalize inputs const operationLower = operation.toLowerCase(); const regionNormalized = region.toLowerCase().trim(); - const batchSize = operation.includes("-") - ? parseInt(operation.split("-")[1]) || 1 - : 1; // Get region multiplier const regionMultiplier = @@ -19,11 +17,7 @@ export function getThresholdForOperation( regionNormalized as keyof typeof THRESHOLDS.regionMultipliers ] || 1.0; - // Parse member count - const memberCount = members && members !== "-" ? parseInt(members) || 0 : 0; - - // Calculate base threshold based on operation type - let baseThreshold: number; + let baseThreshold: number = 1; if (operationType === "network") { const networkThreshold = @@ -32,28 +26,28 @@ export function getThresholdForOperation( typeof networkThreshold === "number" ? networkThreshold : THRESHOLDS.network.server_call; - } else { + } else if (operationType === "group" || operationType === "core") { // For both "core" and "group" types, get the base value baseThreshold = THRESHOLDS.core[operationLower as keyof typeof THRESHOLDS.core] || 0; // Apply member multiplier if applicable - const memberMultiplier = + const operationMultiplier = THRESHOLDS.memberMultipliers[ operationLower as keyof typeof THRESHOLDS.memberMultipliers - ] || 0; + ] || 1; - if (memberMultiplier > 0 && memberCount > 0) { - const batchNumber = Math.ceil(memberCount / batchSize); + batches.add(members > 0 ? parseInt(members.toString()) : 1); + console.log("Current batch sizes:", Array.from(batches)); - // Apply the multiplier based on batch number - // Batch 1 gets multiplier x1, Batch 2 gets x2, etc. - baseThreshold = baseThreshold * batchNumber * memberMultiplier; + // Apply the multiplier based on batch number + // Batch 1 gets multiplier x1, Batch 2 gets x2, etc. + const multiplier = batches.size * operationMultiplier; + baseThreshold = baseThreshold * multiplier; - console.warn( - `Operation: ${operation}, member count: ${memberCount}, batch: ${batchNumber}, multiplier: ${memberMultiplier}x, adjusted threshold: ${baseThreshold}`, - ); - } + console.warn( + `Operation: ${operation}, batch: ${batches.size}, multiplier: ${multiplier}x, adjusted threshold: ${baseThreshold}`, + ); } // Apply region multiplier and round diff --git a/datadog/summary.ts b/datadog/summary.ts index a1178d18..c2372a03 100644 --- a/datadog/summary.ts +++ b/datadog/summary.ts @@ -121,7 +121,7 @@ export function logMetricsSummary( const calculatedThreshold = getThresholdForOperation( operationName as string, operationName.toLowerCase().includes("-") ? "group" : "core", - group.members as string, + group.members as number, currentGeo, ); diff --git a/datadog/thresholds.json b/datadog/thresholds.json index 01bbfdfb..ed5b2d06 100644 --- a/datadog/thresholds.json +++ b/datadog/thresholds.json @@ -22,20 +22,20 @@ "server_call": 250 }, "memberMultipliers": { - "creategroup": 1.5, - "creategroupbyidentifiers": 1.5, - "sendgroupmessage": 1.3, - "syncgroup": 1.3, - "updategroupname": 1.3, - "removemembers": 0.8, - "receivegroupmessage": 1.3 + "creategroup": 1.1, + "creategroupbyidentifiers": 1.01, + "sendgroupmessage": 1.03, + "syncgroup": 1.03, + "updategroupname": 1.03, + "removemembers": 1.03, + "receivegroupmessage": 1.1 }, "regionMultipliers": { "us-east": 1.0, "us-west": 1.0, "europe": 1.0, "asia": 1.5, - "south-america": 10 + "south-america": 2 }, "GEO_TO_COUNTRY_CODE": { "us-east": "US", From 0ecc004074580b375764c077e6e82e7089a58428 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 23:46:52 -0300 Subject: [PATCH 36/57] adjust --- datadog/helper.ts | 4 ++-- datadog/summary.ts | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/datadog/helper.ts b/datadog/helper.ts index b27de019..7364b2be 100644 --- a/datadog/helper.ts +++ b/datadog/helper.ts @@ -185,7 +185,7 @@ export async function sendPerformanceMetric( const threshold = getThresholdForOperation( operationName, operationType, - members, + parseInt(members), currentGeo, ); @@ -216,7 +216,7 @@ export async function sendPerformanceMetric( const networkThreshold = getThresholdForOperation( networkPhase, "network", - members, + parseInt(members), currentGeo, ); diff --git a/datadog/summary.ts b/datadog/summary.ts index c2372a03..b8065ffb 100644 --- a/datadog/summary.ts +++ b/datadog/summary.ts @@ -117,17 +117,19 @@ export function logMetricsSummary( `Calculating threshold for ${operationName} with ${memberCount} members`, ); + const operationType = operationName.toLowerCase().includes("-") + ? "group" + : "core"; // Recalculate threshold with correct values const calculatedThreshold = getThresholdForOperation( operationName as string, - operationName.toLowerCase().includes("-") ? "group" : "core", + operationType, group.members as number, currentGeo, ); // Update the threshold in the data data.threshold = calculatedThreshold; - console.debug(data.threshold); const average = data.values.reduce((sum: number, val: number) => sum + val, 0) / data.values.length; @@ -152,6 +154,7 @@ export function logMetricsSummary( fileContent += `${operationName} | ${group.members} | ${Math.round(average)} | ${Math.round(min)}/${Math.round(max)} | ${data.threshold} | ${varianceFormatted} | ${status}\n`; } + console.log(`Passed: ${passedMetrics}/${totalMetrics}`); // Write to file try { fs.writeFileSync(filename, fileContent); From 864d9b9e64cada4da5a501d825f43182de2394d3 Mon Sep 17 00:00:00 2001 From: fabri Date: Tue, 18 Mar 2025 23:47:37 -0300 Subject: [PATCH 37/57] adjust --- datadog/summary.ts | 2 +- helpers/playwright.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datadog/summary.ts b/datadog/summary.ts index b8065ffb..c8c24035 100644 --- a/datadog/summary.ts +++ b/datadog/summary.ts @@ -49,7 +49,7 @@ export function logMetricsSummary( ); // Create a directory for reports if it doesn't exist - const reportsDir = path.join(process.cwd(), "logs/reports"); + const reportsDir = path.join(process.cwd(), "logs"); if (!fs.existsSync(reportsDir)) { fs.mkdirSync(reportsDir, { recursive: true }); } diff --git a/helpers/playwright.ts b/helpers/playwright.ts index 555272c2..4acd85fe 100644 --- a/helpers/playwright.ts +++ b/helpers/playwright.ts @@ -8,7 +8,7 @@ import { type Page, } from "playwright-chromium"; -const snapshotDir = path.join(process.cwd(), "./logs/snapshots"); +const snapshotDir = path.join(process.cwd(), "./logs"); let browser: Browser | null = null; if (!fs.existsSync(snapshotDir)) { fs.mkdirSync(snapshotDir, { recursive: true }); From eb711a73e46b0058a952d9cd6be246145c89d41c Mon Sep 17 00:00:00 2001 From: fabri Date: Wed, 19 Mar 2025 00:04:52 -0300 Subject: [PATCH 38/57] adjust --- datadog/calculations.ts | 9 ++++----- datadog/summary.ts | 11 ----------- datadog/thresholds.json | 16 ++++++++-------- package.json | 2 +- tests/TS_Performance.test.ts | 1 + tests/groups.test.ts | 2 ++ 6 files changed, 16 insertions(+), 25 deletions(-) diff --git a/datadog/calculations.ts b/datadog/calculations.ts index ea5818d5..e5577a3a 100644 --- a/datadog/calculations.ts +++ b/datadog/calculations.ts @@ -37,16 +37,15 @@ export function getThresholdForOperation( operationLower as keyof typeof THRESHOLDS.memberMultipliers ] || 1; - batches.add(members > 0 ? parseInt(members.toString()) : 1); - console.log("Current batch sizes:", Array.from(batches)); - + batches.add(members); + const batchSize = batches.size - 1; // Apply the multiplier based on batch number // Batch 1 gets multiplier x1, Batch 2 gets x2, etc. - const multiplier = batches.size * operationMultiplier; + const multiplier = batchSize * operationMultiplier; baseThreshold = baseThreshold * multiplier; console.warn( - `Operation: ${operation}, batch: ${batches.size}, multiplier: ${multiplier}x, adjusted threshold: ${baseThreshold}`, + `Operation: ${operation}, batch: ${batchSize}, multiplier: ${multiplier}x, adjusted threshold: ${baseThreshold}`, ); } diff --git a/datadog/summary.ts b/datadog/summary.ts index c8c24035..adba01eb 100644 --- a/datadog/summary.ts +++ b/datadog/summary.ts @@ -85,10 +85,6 @@ export function logMetricsSummary( // Create a unique key for each operation + member count combination const groupKey = `${operationName}-${memberCount}`; - console.log( - `Processing metric: ${operation} → ${groupKey} (Members: ${memberCount})`, - ); - if (!operationGroups.has(groupKey)) { operationGroups.set(groupKey, { operationName: operationName, @@ -113,10 +109,6 @@ export function logMetricsSummary( const memberCount = group.members !== "-" ? parseInt(group.members as string) : 10; - console.log( - `Calculating threshold for ${operationName} with ${memberCount} members`, - ); - const operationType = operationName.toLowerCase().includes("-") ? "group" : "core"; @@ -154,11 +146,8 @@ export function logMetricsSummary( fileContent += `${operationName} | ${group.members} | ${Math.round(average)} | ${Math.round(min)}/${Math.round(max)} | ${data.threshold} | ${varianceFormatted} | ${status}\n`; } - console.log(`Passed: ${passedMetrics}/${totalMetrics}`); - // Write to file try { fs.writeFileSync(filename, fileContent); - console.log(`📝 Metrics summary written to: ${filename}`); } catch (error) { console.error(`❌ Error writing metrics summary to file:`, error); } diff --git a/datadog/thresholds.json b/datadog/thresholds.json index ed5b2d06..b21db633 100644 --- a/datadog/thresholds.json +++ b/datadog/thresholds.json @@ -22,20 +22,20 @@ "server_call": 250 }, "memberMultipliers": { - "creategroup": 1.1, - "creategroupbyidentifiers": 1.01, - "sendgroupmessage": 1.03, - "syncgroup": 1.03, - "updategroupname": 1.03, - "removemembers": 1.03, - "receivegroupmessage": 1.1 + "creategroup": 1.2, + "creategroupbyidentifiers": 1.2, + "sendgroupmessage": 1, + "syncgroup": 1, + "updategroupname": 1, + "removemembers": 1, + "receivegroupmessage": 1 }, "regionMultipliers": { "us-east": 1.0, "us-west": 1.0, "europe": 1.0, "asia": 1.5, - "south-america": 2 + "south-america": 1.5 }, "GEO_TO_COUNTRY_CODE": { "us-east": "US", diff --git a/package.json b/package.json index 42d830d2..0f874aa4 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "bot:simple": "tsx bots/simple/index.ts", "bot:test": "tsx bots/test/index.ts", "build": "tsc", - "clean": "rimraf .data/ ||:", + "clean": "rimraf .data/ ||: && rimraf logs/ ||:", "datadog": "tsx scripts/send-to-datadog.ts", "format": "prettier -w .", "format:check": "prettier -c .", diff --git a/tests/TS_Performance.test.ts b/tests/TS_Performance.test.ts index 8fd05bd6..d22ef3ba 100644 --- a/tests/TS_Performance.test.ts +++ b/tests/TS_Performance.test.ts @@ -155,6 +155,7 @@ describe(testName, () => { newGroup = await workers .get("henry")! .client.conversations.newGroup(sliced.map((inbox) => inbox.inboxId)); + console.log("New group created", newGroup.id); expect(newGroup.id).toBeDefined(); } catch (e) { hasFailures = logError(e, expect); diff --git a/tests/groups.test.ts b/tests/groups.test.ts index 4bae9446..830940a4 100644 --- a/tests/groups.test.ts +++ b/tests/groups.test.ts @@ -83,9 +83,11 @@ describe(testName, () => { it(`createGroup-${i}: should create a large group of ${i} participants ${i}`, async () => { try { const sliced = generatedInboxes.slice(0, i); + console.log("Creating group with", sliced.length, "participants"); newGroup = await workers .get("henry")! .client.conversations.newGroup(sliced.map((inbox) => inbox.inboxId)); + console.log("Group created", newGroup.id); expect(newGroup.id).toBeDefined(); } catch (e) { hasFailures = logError(e, expect); From 00ffcf99b53226320fff43d7d94c3126caf11053 Mon Sep 17 00:00:00 2001 From: fabri Date: Wed, 19 Mar 2025 00:12:11 -0300 Subject: [PATCH 39/57] adjust --- datadog/summary.ts | 4 ++-- datadog/thresholds.json | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/datadog/summary.ts b/datadog/summary.ts index adba01eb..97c96f25 100644 --- a/datadog/summary.ts +++ b/datadog/summary.ts @@ -114,9 +114,9 @@ export function logMetricsSummary( : "core"; // Recalculate threshold with correct values const calculatedThreshold = getThresholdForOperation( - operationName as string, + operationName, operationType, - group.members as number, + memberCount, currentGeo, ); diff --git a/datadog/thresholds.json b/datadog/thresholds.json index b21db633..4fcafece 100644 --- a/datadog/thresholds.json +++ b/datadog/thresholds.json @@ -31,9 +31,9 @@ "receivegroupmessage": 1 }, "regionMultipliers": { - "us-east": 1.0, - "us-west": 1.0, - "europe": 1.0, + "us-east": 1, + "us-west": 1, + "europe": 1, "asia": 1.5, "south-america": 1.5 }, From b6c97147734ee1ad9d20c7e231be0f217dfd1ab0 Mon Sep 17 00:00:00 2001 From: fabri Date: Wed, 19 Mar 2025 00:33:48 -0300 Subject: [PATCH 40/57] adjust --- Dockerfile.test | 17 ----- datadog/calculations.ts | 54 -------------- datadog/helper.ts | 2 +- datadog/slo.ts | 146 ++++++++++++++++++++++++++++++++++++++ datadog/summary.ts | 151 ++++++++++++++++------------------------ datadog/thresholds.json | 95 ++++++++++++++++++++++--- 6 files changed, 292 insertions(+), 173 deletions(-) delete mode 100644 Dockerfile.test delete mode 100644 datadog/calculations.ts create mode 100644 datadog/slo.ts diff --git a/Dockerfile.test b/Dockerfile.test deleted file mode 100644 index 33e5985b..00000000 --- a/Dockerfile.test +++ /dev/null @@ -1,17 +0,0 @@ -FROM node:16 - -WORKDIR /app - -# Copy package.json and yarn.lock -COPY package.json yarn.lock ./ -RUN yarn install - -# Copy the rest of the code -COPY . . - -# Set the environment variables -ENV XMTP_ENV=production -ENV GEOLOCATION=us-east - -# Run the test -CMD ["yarn", "test", "ts_performance"] diff --git a/datadog/calculations.ts b/datadog/calculations.ts deleted file mode 100644 index e5577a3a..00000000 --- a/datadog/calculations.ts +++ /dev/null @@ -1,54 +0,0 @@ -import THRESHOLDS from "./thresholds.json"; - -let batches = new Set(); -export function getThresholdForOperation( - operation: string, - operationType: "core" | "group" | "network", - members: number = 0, - region: string = "us-east", -): number { - // Normalize inputs - const operationLower = operation.toLowerCase(); - const regionNormalized = region.toLowerCase().trim(); - - // Get region multiplier - const regionMultiplier = - THRESHOLDS.regionMultipliers[ - regionNormalized as keyof typeof THRESHOLDS.regionMultipliers - ] || 1.0; - - let baseThreshold: number = 1; - - if (operationType === "network") { - const networkThreshold = - THRESHOLDS.network[operationLower as keyof typeof THRESHOLDS.network]; - baseThreshold = - typeof networkThreshold === "number" - ? networkThreshold - : THRESHOLDS.network.server_call; - } else if (operationType === "group" || operationType === "core") { - // For both "core" and "group" types, get the base value - baseThreshold = - THRESHOLDS.core[operationLower as keyof typeof THRESHOLDS.core] || 0; - - // Apply member multiplier if applicable - const operationMultiplier = - THRESHOLDS.memberMultipliers[ - operationLower as keyof typeof THRESHOLDS.memberMultipliers - ] || 1; - - batches.add(members); - const batchSize = batches.size - 1; - // Apply the multiplier based on batch number - // Batch 1 gets multiplier x1, Batch 2 gets x2, etc. - const multiplier = batchSize * operationMultiplier; - baseThreshold = baseThreshold * multiplier; - - console.warn( - `Operation: ${operation}, batch: ${batchSize}, multiplier: ${multiplier}x, adjusted threshold: ${baseThreshold}`, - ); - } - - // Apply region multiplier and round - return Math.round(baseThreshold * regionMultiplier); -} diff --git a/datadog/helper.ts b/datadog/helper.ts index 7364b2be..a963a770 100644 --- a/datadog/helper.ts +++ b/datadog/helper.ts @@ -1,7 +1,7 @@ import type { WorkerManager } from "@workers/manager"; import metrics from "datadog-metrics"; -import { getThresholdForOperation } from "./calculations"; import { getNetworkStats } from "./network"; +import { getThresholdForOperation } from "./slo"; import { logMetricsSummary } from "./summary"; import THRESHOLDS from "./thresholds.json"; diff --git a/datadog/slo.ts b/datadog/slo.ts new file mode 100644 index 00000000..89cf2659 --- /dev/null +++ b/datadog/slo.ts @@ -0,0 +1,146 @@ +import THRESHOLDS from "./thresholds.json"; + +// Define the proper types to avoid TypeScript errors +interface MemberThresholds { + creategroup: number; + creategroupbyidentifiers: number; + syncgroup: number; + updategroupname: number; + removemembers: number; + sendgroupmessage: number; + receivegroupmessage: number; + [key: string]: number; // Allow for other properties +} + +interface ThresholdsData { + core: { + [key: string]: number; + }; + network: { + [key: string]: number; + }; + memberBasedThresholds: { + [memberCount: string]: MemberThresholds; + }; + regionMultipliers: { + [region: string]: number; + }; + GEO_TO_COUNTRY_CODE: { + [region: string]: string; + }; + reliability: number; +} + +// Type assertion to enforce our expected structure +const typedThresholds = THRESHOLDS as unknown as ThresholdsData; + +export function getThresholdForOperation( + operation: string, + operationType: "core" | "group" | "network", + members: number = 0, + region: string = "us-east", +): number { + // Normalize inputs + const operationLower = operation.toLowerCase(); + const regionNormalized = region.toLowerCase().trim(); + + // Get region multiplier + const regionMultiplier = + typedThresholds.regionMultipliers[regionNormalized] || 1.0; + + let baseThreshold: number = 0; + + if (operationType === "network") { + const networkThreshold = typedThresholds.network[operationLower]; + baseThreshold = + typeof networkThreshold === "number" + ? networkThreshold + : typedThresholds.network.server_call; + } else if (operationType === "core") { + // For core operations, use thresholds directly from the core settings + baseThreshold = typedThresholds.core[operationLower] || 0; + } else if (operationType === "group") { + // For group operations with member count > 0, use memberBasedThresholds + if (members > 0) { + // Find the appropriate member bucket for the given member count + const memberBuckets = Object.keys(typedThresholds.memberBasedThresholds) + .map(Number) + .sort((a, b) => a - b); + + // Find the highest bucket that's less than or equal to the member count + let applicableBucket = "0"; + for (const bucket of memberBuckets) { + if (members >= bucket) { + applicableBucket = bucket.toString(); + } else { + break; + } + } + + // Get threshold from the member-based table + const memberThresholds = + typedThresholds.memberBasedThresholds[applicableBucket]; + if (memberThresholds && operationLower in memberThresholds) { + baseThreshold = memberThresholds[operationLower]; + + console.warn( + `Operation: ${operation}, members: ${members}, using bucket: ${applicableBucket}, threshold: ${baseThreshold}`, + ); + } else { + // Fallback to core thresholds if member-based threshold not found + baseThreshold = typedThresholds.core[operationLower] || 0; + + console.warn( + `Operation: ${operation}, members: ${members}, using core threshold: ${baseThreshold}`, + ); + } + } else { + // For group operations with members = 0, use core thresholds + baseThreshold = typedThresholds.core[operationLower] || 0; + } + } + + // Apply region multiplier and round + return Math.round(baseThreshold * regionMultiplier); +} + +/** + * Calculate average of numeric values + */ +export function calculateAverage(values: number[]): number { + return values.reduce((sum, val) => sum + val, 0) / values.length; +} + +/** + * Group metrics by operation name and member count + */ +export function groupMetricsByOperation( + metrics: [ + string, + { values: number[]; threshold: number; members?: string }, + ][], +): Map { + const groups = new Map(); + + for (const [operation, data] of metrics) { + // Parse operation name and member count + const dashMatch = operation.match(/^([a-zA-Z]+)-(\d+)$/); + const operationName = dashMatch ? dashMatch[1] : operation.split(":")[0]; + const memberCount = dashMatch ? dashMatch[2] : data.members || "-"; + + // Create unique key for this operation + member combination + const groupKey = `${operationName}-${memberCount}`; + + if (!groups.has(groupKey)) { + groups.set(groupKey, { + operationName, + members: memberCount, + operationData: null, + }); + } + + groups.get(groupKey).operationData = data; + } + + return groups; +} diff --git a/datadog/summary.ts b/datadog/summary.ts index 97c96f25..acd1f65a 100644 --- a/datadog/summary.ts +++ b/datadog/summary.ts @@ -1,6 +1,10 @@ import fs from "fs"; import path from "path"; -import { getThresholdForOperation } from "./calculations"; +import { + calculateAverage, + getThresholdForOperation, + groupMetricsByOperation, +} from "./slo"; /** * Logs a summary of all collected metrics against their thresholds @@ -26,132 +30,99 @@ export function logMetricsSummary( console.log("\n📊 Creating metrics summary report"); - // Create a simple text summary for the console - const passedMetrics = Object.entries(collectedMetrics).filter( - ([operation, data]) => { - // Skip workflow metrics when counting passed metrics - if (operation === "workflow") return false; + // Filter out workflow metrics and empty value arrays + const validMetrics = Object.entries(collectedMetrics).filter( + ([operation, data]) => operation !== "workflow" && data.values.length > 0, + ); - if (data.values.length === 0) return false; - const average = - data.values.reduce((sum, val) => sum + val, 0) / data.values.length; - return average <= data.threshold; - }, - ).length; + // Count passed metrics + const passedMetrics = validMetrics.filter(([_, data]) => { + const average = calculateAverage(data.values); + return average <= data.threshold; + }).length; - // Count only non-workflow metrics for total - const totalMetrics = Object.entries(collectedMetrics).filter( - ([operation]) => operation !== "workflow", - ).length; + const totalMetrics = validMetrics.length; + const passRate = + totalMetrics > 0 ? Math.round((passedMetrics / totalMetrics) * 100) : 0; console.log( - `✅ Passed: ${passedMetrics}/${totalMetrics} metrics (${Math.round((passedMetrics / totalMetrics) * 100)}%)`, + `✅ Passed: ${passedMetrics}/${totalMetrics} metrics (${passRate}%)`, ); - // Create a directory for reports if it doesn't exist + // Create directory for reports const reportsDir = path.join(process.cwd(), "logs"); if (!fs.existsSync(reportsDir)) { fs.mkdirSync(reportsDir, { recursive: true }); } - // Create filename with timestamp - const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); + // Create filename with environment info const filename = path.join( reportsDir, `${testName}-${currentGeo}-${process.env.XMTP_ENV}.md`, ); - // Build the table content - let fileContent = "METRICS SUMMARY\n===============\n\n"; - fileContent += - "Operation | Members | Avg (ms) | Min/Max (ms) | Threshold (ms) | Variance (ms) | Status\n"; - fileContent += - "----------|---------|----------|--------------|----------------|---------------|-------\n"; - - // Group metrics by base operation name to associate network times with operations - const operationGroups = new Map(); - - // First pass - group and organize data - for (const [operation, data] of Object.entries(collectedMetrics)) { - // Skip workflow metrics in the summary table - if (operation === "workflow") continue; - if (data.values.length === 0) continue; - - // Look for operation-members format (e.g., "createGroup-10") - const dashMatch = operation.match(/^([a-zA-Z]+)-(\d+)$/); - const operationName = dashMatch ? dashMatch[1] : operation.split(":")[0]; - const memberCount = dashMatch ? dashMatch[2] : data.members || "-"; - - // Create a unique key for each operation + member count combination - const groupKey = `${operationName}-${memberCount}`; - - if (!operationGroups.has(groupKey)) { - operationGroups.set(groupKey, { - operationName: operationName, - members: memberCount, - operationData: null, - }); - } + // Generate report content + const fileContent = generateReportContent(validMetrics, currentGeo); - const group = operationGroups.get(groupKey); - - group.operationData = data; + try { + fs.writeFileSync(filename, fileContent); + } catch (error) { + console.error(`❌ Error writing metrics summary to file:`, error); } - // Second pass - generate table rows - for (const [groupKey, group] of operationGroups.entries()) { - if (!group.operationData) continue; + // Reset metrics collection for next test run + collectedMetrics = {}; +} - const data = group.operationData; - const operationName = group.operationName; +/** + * Generate the markdown report content + */ +function generateReportContent( + validMetrics: [ + string, + { values: number[]; threshold: number; members?: string }, + ][], + currentGeo: string, +): string { + let content = "METRICS SUMMARY\n===============\n\n"; + content += + "Operation | Members | Avg (ms) | Min/Max (ms) | Threshold (ms) | Variance (ms) | Status\n"; + content += + "----------|---------|----------|--------------|----------------|---------------|-------\n"; + + // Group metrics by operation name and member count + const operationGroups = groupMetricsByOperation(validMetrics); - // Use the member count preserved in the group - const memberCount = - group.members !== "-" ? parseInt(group.members as string) : 10; + // Generate table rows + for (const group of operationGroups.values()) { + if (!group.operationData) continue; + const { operationName, members, operationData: data } = group; + const memberCount = members !== "-" ? parseInt(members as string) : 10; const operationType = operationName.toLowerCase().includes("-") ? "group" : "core"; - // Recalculate threshold with correct values - const calculatedThreshold = getThresholdForOperation( + + // Calculate threshold with correct values + data.threshold = getThresholdForOperation( operationName, operationType, memberCount, currentGeo, ); - // Update the threshold in the data - data.threshold = calculatedThreshold; - const average = - data.values.reduce((sum: number, val: number) => sum + val, 0) / - data.values.length; - const min = Math.min(...(data.values as number[])); - const max = Math.max(...(data.values as number[])); - + const average = calculateAverage(data.values); + const min = Math.min(...data.values); + const max = Math.max(...data.values); const status = average <= data.threshold ? "PASS ✅" : "FAIL ❌"; - // Calculate variance between average and threshold + // Format variance const variance = Math.round(average - data.threshold); const varianceFormatted = variance <= 0 ? variance.toString() : `+${variance}`; - // console.debug("getThresholdForOperation inputs:", { - // operationName, - // operationType: operationName.includes("-") ? "group" : "core", - // memberCountStr: group.members, - // currentGeo, - // actualMemberCount: memberCount, - // }); - - fileContent += `${operationName} | ${group.members} | ${Math.round(average)} | ${Math.round(min)}/${Math.round(max)} | ${data.threshold} | ${varianceFormatted} | ${status}\n`; + content += `${operationName} | ${members} | ${Math.round(average)} | ${Math.round(min)}/${Math.round(max)} | ${data.threshold} | ${varianceFormatted} | ${status}\n`; } - try { - fs.writeFileSync(filename, fileContent); - } catch (error) { - console.error(`❌ Error writing metrics summary to file:`, error); - } - - // Reset metrics collection for next test run - collectedMetrics = {}; + return content; } diff --git a/datadog/thresholds.json b/datadog/thresholds.json index 4fcafece..b8c077a1 100644 --- a/datadog/thresholds.json +++ b/datadog/thresholds.json @@ -5,14 +5,14 @@ "createdm": 200, "sendgm": 200, "receivegm": 200, - "receivegroupmessage": 200, "creategroup": 200, "creategroupbyidentifiers": 200, "sendgroupmessage": 200, "syncgroup": 200, "updategroupname": 200, "removemembers": 200, - "addmembers": 200 + "addmembers": 200, + "receivegroupmessage": 200 }, "network": { "dns_lookup": 50, @@ -21,21 +21,94 @@ "processing": 100, "server_call": 250 }, - "memberMultipliers": { - "creategroup": 1.2, - "creategroupbyidentifiers": 1.2, - "sendgroupmessage": 1, - "syncgroup": 1, - "updategroupname": 1, - "removemembers": 1, - "receivegroupmessage": 1 + "memberBasedThresholds": { + "50": { + "creategroup": 200, + "creategroupbyidentifiers": 200, + "sendgroupmessage": 200, + "syncgroup": 200, + "updategroupname": 200, + "removemembers": 200, + "addmembers": 200, + "receivegroupmessage": 200 + }, + "100": { + "creategroup": 200, + "creategroupbyidentifiers": 200, + "sendgroupmessage": 200, + "syncgroup": 200, + "updategroupname": 200, + "removemembers": 200, + "addmembers": 200, + "receivegroupmessage": 200 + }, + "150": { + "creategroup": 200, + "creategroupbyidentifiers": 200, + "sendgroupmessage": 200, + "syncgroup": 200, + "updategroupname": 200, + "removemembers": 200, + "addmembers": 200, + "receivegroupmessage": 200 + }, + "200": { + "creategroup": 200, + "creategroupbyidentifiers": 200, + "sendgroupmessage": 200, + "syncgroup": 200, + "updategroupname": 200, + "removemembers": 200, + "addmembers": 200, + "receivegroupmessage": 200 + }, + "250": { + "creategroup": 200, + "creategroupbyidentifiers": 200, + "sendgroupmessage": 200, + "syncgroup": 200, + "updategroupname": 200, + "removemembers": 200, + "addmembers": 200, + "receivegroupmessage": 200 + }, + "300": { + "creategroup": 200, + "creategroupbyidentifiers": 200, + "sendgroupmessage": 200, + "syncgroup": 200, + "updategroupname": 200, + "removemembers": 200, + "addmembers": 200, + "receivegroupmessage": 200 + }, + "350": { + "creategroup": 200, + "creategroupbyidentifiers": 200, + "sendgroupmessage": 200, + "syncgroup": 200, + "updategroupname": 200, + "removemembers": 200, + "addmembers": 200, + "receivegroupmessage": 200 + }, + "400": { + "creategroup": 200, + "creategroupbyidentifiers": 200, + "sendgroupmessage": 200, + "syncgroup": 200, + "updategroupname": 200, + "removemembers": 200, + "addmembers": 200, + "receivegroupmessage": 200 + } }, "regionMultipliers": { "us-east": 1, "us-west": 1, "europe": 1, "asia": 1.5, - "south-america": 1.5 + "south-america": 3 }, "GEO_TO_COUNTRY_CODE": { "us-east": "US", From 565fa48d9b3a5728b6d4946050226845341bd996 Mon Sep 17 00:00:00 2001 From: fabri Date: Wed, 19 Mar 2025 00:36:04 -0300 Subject: [PATCH 41/57] adjust --- datadog/summary.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/datadog/summary.ts b/datadog/summary.ts index acd1f65a..a39603b2 100644 --- a/datadog/summary.ts +++ b/datadog/summary.ts @@ -86,9 +86,9 @@ function generateReportContent( ): string { let content = "METRICS SUMMARY\n===============\n\n"; content += - "Operation | Members | Avg (ms) | Min/Max (ms) | Threshold (ms) | Variance (ms) | Status\n"; + "Operation | Members | Avg (ms) | Min/Max (ms) | Threshold (ms) | Variance (ms) | Network (ms) | Status\n"; content += - "----------|---------|----------|--------------|----------------|---------------|-------\n"; + "----------|---------|----------|--------------|----------------|---------------|-------------|-------\n"; // Group metrics by operation name and member count const operationGroups = groupMetricsByOperation(validMetrics); @@ -111,6 +111,14 @@ function generateReportContent( currentGeo, ); + // Get network threshold for this operation + const networkThreshold = getThresholdForOperation( + operationName, + "network", + memberCount, + currentGeo, + ); + const average = calculateAverage(data.values); const min = Math.min(...data.values); const max = Math.max(...data.values); @@ -121,7 +129,7 @@ function generateReportContent( const varianceFormatted = variance <= 0 ? variance.toString() : `+${variance}`; - content += `${operationName} | ${members} | ${Math.round(average)} | ${Math.round(min)}/${Math.round(max)} | ${data.threshold} | ${varianceFormatted} | ${status}\n`; + content += `${operationName} | ${members} | ${Math.round(average)} | ${Math.round(min)}/${Math.round(max)} | ${data.threshold} | ${varianceFormatted} | ${networkThreshold} | ${status}\n`; } return content; From c37e40a557eb0b9dfc32f5ef0b3599adeb98280f Mon Sep 17 00:00:00 2001 From: fabri Date: Wed, 19 Mar 2025 00:38:20 -0300 Subject: [PATCH 42/57] adjust --- datadog/thresholds.json | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/datadog/thresholds.json b/datadog/thresholds.json index b8c077a1..348bd2c9 100644 --- a/datadog/thresholds.json +++ b/datadog/thresholds.json @@ -33,8 +33,8 @@ "receivegroupmessage": 200 }, "100": { - "creategroup": 200, - "creategroupbyidentifiers": 200, + "creategroup": 300, + "creategroupbyidentifiers": 300, "sendgroupmessage": 200, "syncgroup": 200, "updategroupname": 200, @@ -43,8 +43,8 @@ "receivegroupmessage": 200 }, "150": { - "creategroup": 200, - "creategroupbyidentifiers": 200, + "creategroup": 400, + "creategroupbyidentifiers": 400, "sendgroupmessage": 200, "syncgroup": 200, "updategroupname": 200, @@ -53,8 +53,8 @@ "receivegroupmessage": 200 }, "200": { - "creategroup": 200, - "creategroupbyidentifiers": 200, + "creategroup": 500, + "creategroupbyidentifiers": 500, "sendgroupmessage": 200, "syncgroup": 200, "updategroupname": 200, @@ -63,8 +63,8 @@ "receivegroupmessage": 200 }, "250": { - "creategroup": 200, - "creategroupbyidentifiers": 200, + "creategroup": 700, + "creategroupbyidentifiers": 700, "sendgroupmessage": 200, "syncgroup": 200, "updategroupname": 200, @@ -73,8 +73,8 @@ "receivegroupmessage": 200 }, "300": { - "creategroup": 200, - "creategroupbyidentifiers": 200, + "creategroup": 1000, + "creategroupbyidentifiers": 1000, "sendgroupmessage": 200, "syncgroup": 200, "updategroupname": 200, @@ -83,8 +83,8 @@ "receivegroupmessage": 200 }, "350": { - "creategroup": 200, - "creategroupbyidentifiers": 200, + "creategroup": 1000, + "creategroupbyidentifiers": 1000, "sendgroupmessage": 200, "syncgroup": 200, "updategroupname": 200, @@ -93,8 +93,8 @@ "receivegroupmessage": 200 }, "400": { - "creategroup": 200, - "creategroupbyidentifiers": 200, + "creategroup": 1300, + "creategroupbyidentifiers": 1300, "sendgroupmessage": 200, "syncgroup": 200, "updategroupname": 200, From 6618b2079f33b67479f36529d1735999e471ac6f Mon Sep 17 00:00:00 2001 From: fabri Date: Wed, 19 Mar 2025 01:51:52 -0300 Subject: [PATCH 43/57] fix --- datadog/{dashboards => }/delivery.json | 0 datadog/helper.ts | 283 -------- datadog/network.ts | 59 -- datadog/{dashboards => }/performance.json | 0 datadog/slo.ts | 146 ---- datadog/summary.ts | 136 ---- datadog/thresholds.json | 121 ---- datadog/{dashboards => }/workflows.json | 0 helpers/client.ts | 2 +- helpers/datadog.ts | 798 ++++++++++++++++++++++ scripts/railway-test.ts | 2 +- tests/TS_Delivery.test.ts | 2 +- tests/TS_Gm.test.ts | 2 +- tests/TS_Performance.test.ts | 2 +- tests/dms.test.ts | 2 +- tests/groups.test.ts | 2 +- tests/offline.test.ts | 2 +- tsconfig.json | 1 - vitest.config.ts | 1 - 19 files changed, 806 insertions(+), 755 deletions(-) rename datadog/{dashboards => }/delivery.json (100%) delete mode 100644 datadog/helper.ts delete mode 100644 datadog/network.ts rename datadog/{dashboards => }/performance.json (100%) delete mode 100644 datadog/slo.ts delete mode 100644 datadog/summary.ts delete mode 100644 datadog/thresholds.json rename datadog/{dashboards => }/workflows.json (100%) create mode 100644 helpers/datadog.ts diff --git a/datadog/dashboards/delivery.json b/datadog/delivery.json similarity index 100% rename from datadog/dashboards/delivery.json rename to datadog/delivery.json diff --git a/datadog/helper.ts b/datadog/helper.ts deleted file mode 100644 index a963a770..00000000 --- a/datadog/helper.ts +++ /dev/null @@ -1,283 +0,0 @@ -import type { WorkerManager } from "@workers/manager"; -import metrics from "datadog-metrics"; -import { getNetworkStats } from "./network"; -import { getThresholdForOperation } from "./slo"; -import { logMetricsSummary } from "./summary"; -import THRESHOLDS from "./thresholds.json"; - -// Global state variables -let isInitialized = false; -let currentGeo: string = ""; -let collectedMetrics: Record< - string, - { values: number[]; threshold: number; members?: string } -> = {}; - -function getCountryCodeFromGeo(geolocation: string): string { - return ( - THRESHOLDS.GEO_TO_COUNTRY_CODE[ - geolocation as keyof typeof THRESHOLDS.GEO_TO_COUNTRY_CODE - ] || "US" - ); -} - -export const sendPerformanceResult = ( - expect: any, - workers: WorkerManager, - start: number, -) => { - const testName = expect.getState().currentTestName; - if (testName) { - console.timeEnd(testName as string); - expect(workers.getWorkers()).toBeDefined(); - expect(workers.getWorkers().length).toBeGreaterThan(0); - void sendPerformanceMetric( - performance.now() - start, - testName as string, - workers.getVersion(), - false, - ); - } -}; -// Add success status tag to duration metrics - -export function initDataDog( - testName: string, - envValue: string, - geolocation: string, - apiKey: string, -): boolean { - if (isInitialized) { - return true; - } - if (!testName.includes("ts_")) { - return true; - } - if (!apiKey) { - console.warn("⚠️ DATADOG_API_KEY not found. Metrics will not be sent."); - return false; - } - - try { - const countryCode = getCountryCodeFromGeo(geolocation); - currentGeo = geolocation; - const initConfig = { - apiKey: apiKey, - defaultTags: [ - `env:${envValue}`, - `test:${testName}`, - `region:${geolocation}`, - `country_iso_code:${countryCode}`, - ], - }; - metrics.init(initConfig); - isInitialized = true; - return true; - } catch (error) { - console.error("❌ Failed to initialize DataDog metrics:", error); - return false; - } -} - -// Modify the sendMetric function to collect metrics for summary -export function sendMetric( - metricName: string, - metricValue: number, - tags: Record, -): void { - if (!isInitialized) return; - - try { - const fullMetricName = `xmtp.sdk.${metricName}`; - const allTags = Object.entries({ ...tags }).map( - ([key, value]) => `${key}:${String(value)}`, - ); - // Add environment tag if not already present - if (!allTags.some((tag) => tag.startsWith("env:"))) { - allTags.push(`env:${process.env.XMTP_ENV as string}`); - } - // Add version tag if not already present - if (!allTags.some((tag) => tag.startsWith("xv:"))) { - allTags.push(`vm:${process.env.RAILWAY_SERVICE_ID || "unknown"}`); - } - - // Create a distinctive operation key that properly includes member count - // Format: operation_name-member_count (e.g., "createGroup-10") - const memberCount = tags.members || ""; - const operationKey = tags.operation - ? `${tags.operation}${memberCount ? `-${memberCount}` : ""}` - : metricName; - - if (!collectedMetrics[operationKey]) { - collectedMetrics[operationKey] = { - values: [], - threshold: tags.threshold || 0, - members: memberCount, - }; - } - collectedMetrics[operationKey].values.push(metricValue); - - metrics.gauge(fullMetricName, Math.round(metricValue), allTags); - } catch (error) { - console.error( - `❌ Error sending metric '${metricName}':`, - error instanceof Error ? error.message : String(error), - ); - } -} - -export function sendTestResults(hasFailures: boolean, testName: string): void { - if (!isInitialized) { - console.warn("Datadog metrics not initialized"); - return; - } - try { - const metricValue = hasFailures ? 0 : 1; - const metricName = `workflow`; - sendMetric(metricName, metricValue, { - workflow: testName, - metric_type: "workflow", - }); - } catch (error) { - console.error("Error reporting to Datadog:", error); - } -} - -// Simplified version of sendPerformanceMetric -export async function sendPerformanceMetric( - metricValue: number, - testName: string, - libxmtpVersion: string, - skipNetworkStats: boolean = false, -): Promise { - if (!isInitialized) return; - - try { - const metricNameParts = testName.split(":")[0]; - const metricName = metricNameParts.replaceAll(" > ", "."); - const metricDescription = testName.split(":")[1] || ""; - const operationParts = metricName.split("."); - const testNameExtracted = operationParts[0]; - - // Extract operation name and member count more reliably - let operationName = ""; - let members = ""; - - if (operationParts[1]) { - // Check for both formats: "operation-10" and "operation10" - const dashMatch = operationParts[1].match(/^([a-zA-Z]+)-(\d+)$/); - const noSeparatorMatch = operationParts[1].match(/^([a-zA-Z]+)(\d+)$/); - - if (dashMatch) { - operationName = dashMatch[1]; - members = dashMatch[2]; - } else if (noSeparatorMatch) { - operationName = noSeparatorMatch[1]; - members = noSeparatorMatch[2]; - } else { - operationName = operationParts[1]; - } - } - - const isGroupOperation = operationName.toLowerCase().includes("-"); - const operationType = isGroupOperation ? "group" : "core"; - - const threshold = getThresholdForOperation( - operationName, - operationType, - parseInt(members), - currentGeo, - ); - - const isSuccess = metricValue <= threshold; - - sendMetric("duration", metricValue, { - libxmtp: libxmtpVersion, - operation: operationName, - test: testNameExtracted, - metric_type: "operation", - metric_subtype: operationType, - description: metricDescription, - members: members, - success: isSuccess, - threshold: threshold, - region: currentGeo, - }); - - // Network stats handling - if (!skipNetworkStats) { - //ignore group operations - const networkStats = await getNetworkStats(); - const countryCode = getCountryCodeFromGeo(currentGeo); - - for (const [statName, statValue] of Object.entries(networkStats)) { - const networkMetricValue = Math.round(statValue * 1000); - const networkPhase = statName.toLowerCase().replace(/\s+/g, "_"); - const networkThreshold = getThresholdForOperation( - networkPhase, - "network", - parseInt(members), - currentGeo, - ); - - sendMetric("duration", networkMetricValue, { - libxmtp: libxmtpVersion, - operation: operationName, - test: testNameExtracted, - metric_type: "network", - network_phase: networkPhase, - country_iso_code: countryCode, - members: members, - success: networkMetricValue <= networkThreshold, - threshold: networkThreshold, - region: currentGeo, - }); - } - } - } catch (error) { - console.error( - `❌ Error sending performance metric for '${testName}':`, - error, - ); - } -} - -// Modify flushMetrics to include the summary log -export function flushMetrics(testName: string): Promise { - return new Promise((resolve) => { - if (!isInitialized) { - resolve(); - return; - } - - logMetricsSummary(testName, currentGeo, isInitialized, collectedMetrics); - - void metrics.flush().then(() => { - resolve(); - }); - }); -} - -// Unified delivery metrics function -export function sendDeliveryMetric( - metricValue: number, - version: string, - testName: string, - metricSubType: "stream" | "poll" | "recovery", - metricType: "delivery" | "order", -): void { - // Determine success based on the metric subtype - const threshold = THRESHOLDS.reliability; - - const isSuccess = metricValue >= threshold; - - // Send primary metric - sendMetric(metricType, Math.round(metricValue), { - libxmtp: version, - test: testName, - metric_type: metricType, - metric_subtype: metricSubType, - success: isSuccess, - threshold: threshold, - }); -} diff --git a/datadog/network.ts b/datadog/network.ts deleted file mode 100644 index 946131ca..00000000 --- a/datadog/network.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { exec } from "child_process"; -import { promisify } from "util"; - -const execAsync = promisify(exec); - -interface NetworkStats { - "DNS Lookup": number; - "TCP Connection": number; - "TLS Handshake": number; - Processing: number; - "Server Call": number; -} - -export async function getNetworkStats( - endpoint = "https://grpc.dev.xmtp.network:443", -): Promise { - const curlCommand = `curl -s -w "\\n{\\"DNS Lookup\\": %{time_namelookup}, \\"TCP Connection\\": %{time_connect}, \\"TLS Handshake\\": %{time_appconnect}, \\"Server Call\\": %{time_starttransfer}}" -o /dev/null --max-time 10 ${endpoint}`; - - let stdout: string; - - try { - const result = await execAsync(curlCommand); - stdout = result.stdout; - } catch (error: any) { - // Even if curl returns an error, we might still have useful stdout - if (error.stdout) { - stdout = error.stdout; - console.warn( - `⚠️ Curl command returned error code ${error.code}, but stdout is available.`, - ); - } else { - console.error(`❌ Curl command failed without stdout:`, error); - throw error; // rethrow if no stdout is available - } - } - - // Parse the JSON response - const stats = JSON.parse(stdout.trim()) as NetworkStats; - - // Handle the case where Server Call time is 0 - if (stats["Server Call"] === 0) { - console.warn( - `Network request to ${endpoint} returned Server Call time of 0.`, - ); - - // Use a reasonable estimate - stats["Server Call"] = stats["TLS Handshake"] + 0.1; - } - - // Calculate processing time - stats["Processing"] = stats["Server Call"] - stats["TLS Handshake"]; - - // Ensure processing time is not negative - if (stats["Processing"] < 0) { - stats["Processing"] = 0; - } - - return stats; -} diff --git a/datadog/dashboards/performance.json b/datadog/performance.json similarity index 100% rename from datadog/dashboards/performance.json rename to datadog/performance.json diff --git a/datadog/slo.ts b/datadog/slo.ts deleted file mode 100644 index 89cf2659..00000000 --- a/datadog/slo.ts +++ /dev/null @@ -1,146 +0,0 @@ -import THRESHOLDS from "./thresholds.json"; - -// Define the proper types to avoid TypeScript errors -interface MemberThresholds { - creategroup: number; - creategroupbyidentifiers: number; - syncgroup: number; - updategroupname: number; - removemembers: number; - sendgroupmessage: number; - receivegroupmessage: number; - [key: string]: number; // Allow for other properties -} - -interface ThresholdsData { - core: { - [key: string]: number; - }; - network: { - [key: string]: number; - }; - memberBasedThresholds: { - [memberCount: string]: MemberThresholds; - }; - regionMultipliers: { - [region: string]: number; - }; - GEO_TO_COUNTRY_CODE: { - [region: string]: string; - }; - reliability: number; -} - -// Type assertion to enforce our expected structure -const typedThresholds = THRESHOLDS as unknown as ThresholdsData; - -export function getThresholdForOperation( - operation: string, - operationType: "core" | "group" | "network", - members: number = 0, - region: string = "us-east", -): number { - // Normalize inputs - const operationLower = operation.toLowerCase(); - const regionNormalized = region.toLowerCase().trim(); - - // Get region multiplier - const regionMultiplier = - typedThresholds.regionMultipliers[regionNormalized] || 1.0; - - let baseThreshold: number = 0; - - if (operationType === "network") { - const networkThreshold = typedThresholds.network[operationLower]; - baseThreshold = - typeof networkThreshold === "number" - ? networkThreshold - : typedThresholds.network.server_call; - } else if (operationType === "core") { - // For core operations, use thresholds directly from the core settings - baseThreshold = typedThresholds.core[operationLower] || 0; - } else if (operationType === "group") { - // For group operations with member count > 0, use memberBasedThresholds - if (members > 0) { - // Find the appropriate member bucket for the given member count - const memberBuckets = Object.keys(typedThresholds.memberBasedThresholds) - .map(Number) - .sort((a, b) => a - b); - - // Find the highest bucket that's less than or equal to the member count - let applicableBucket = "0"; - for (const bucket of memberBuckets) { - if (members >= bucket) { - applicableBucket = bucket.toString(); - } else { - break; - } - } - - // Get threshold from the member-based table - const memberThresholds = - typedThresholds.memberBasedThresholds[applicableBucket]; - if (memberThresholds && operationLower in memberThresholds) { - baseThreshold = memberThresholds[operationLower]; - - console.warn( - `Operation: ${operation}, members: ${members}, using bucket: ${applicableBucket}, threshold: ${baseThreshold}`, - ); - } else { - // Fallback to core thresholds if member-based threshold not found - baseThreshold = typedThresholds.core[operationLower] || 0; - - console.warn( - `Operation: ${operation}, members: ${members}, using core threshold: ${baseThreshold}`, - ); - } - } else { - // For group operations with members = 0, use core thresholds - baseThreshold = typedThresholds.core[operationLower] || 0; - } - } - - // Apply region multiplier and round - return Math.round(baseThreshold * regionMultiplier); -} - -/** - * Calculate average of numeric values - */ -export function calculateAverage(values: number[]): number { - return values.reduce((sum, val) => sum + val, 0) / values.length; -} - -/** - * Group metrics by operation name and member count - */ -export function groupMetricsByOperation( - metrics: [ - string, - { values: number[]; threshold: number; members?: string }, - ][], -): Map { - const groups = new Map(); - - for (const [operation, data] of metrics) { - // Parse operation name and member count - const dashMatch = operation.match(/^([a-zA-Z]+)-(\d+)$/); - const operationName = dashMatch ? dashMatch[1] : operation.split(":")[0]; - const memberCount = dashMatch ? dashMatch[2] : data.members || "-"; - - // Create unique key for this operation + member combination - const groupKey = `${operationName}-${memberCount}`; - - if (!groups.has(groupKey)) { - groups.set(groupKey, { - operationName, - members: memberCount, - operationData: null, - }); - } - - groups.get(groupKey).operationData = data; - } - - return groups; -} diff --git a/datadog/summary.ts b/datadog/summary.ts deleted file mode 100644 index a39603b2..00000000 --- a/datadog/summary.ts +++ /dev/null @@ -1,136 +0,0 @@ -import fs from "fs"; -import path from "path"; -import { - calculateAverage, - getThresholdForOperation, - groupMetricsByOperation, -} from "./slo"; - -/** - * Logs a summary of all collected metrics against their thresholds - * Call this at the end of test execution - */ -export function logMetricsSummary( - testName: string, - currentGeo: string, - isInitialized: boolean, - collectedMetrics: Record< - string, - { - values: number[]; - threshold: number; - members?: string; - } - >, -): void { - if (!isInitialized || Object.keys(collectedMetrics).length === 0) { - console.log("No metrics collected to summarize"); - return; - } - - console.log("\n📊 Creating metrics summary report"); - - // Filter out workflow metrics and empty value arrays - const validMetrics = Object.entries(collectedMetrics).filter( - ([operation, data]) => operation !== "workflow" && data.values.length > 0, - ); - - // Count passed metrics - const passedMetrics = validMetrics.filter(([_, data]) => { - const average = calculateAverage(data.values); - return average <= data.threshold; - }).length; - - const totalMetrics = validMetrics.length; - const passRate = - totalMetrics > 0 ? Math.round((passedMetrics / totalMetrics) * 100) : 0; - - console.log( - `✅ Passed: ${passedMetrics}/${totalMetrics} metrics (${passRate}%)`, - ); - - // Create directory for reports - const reportsDir = path.join(process.cwd(), "logs"); - if (!fs.existsSync(reportsDir)) { - fs.mkdirSync(reportsDir, { recursive: true }); - } - - // Create filename with environment info - const filename = path.join( - reportsDir, - `${testName}-${currentGeo}-${process.env.XMTP_ENV}.md`, - ); - - // Generate report content - const fileContent = generateReportContent(validMetrics, currentGeo); - - try { - fs.writeFileSync(filename, fileContent); - } catch (error) { - console.error(`❌ Error writing metrics summary to file:`, error); - } - - // Reset metrics collection for next test run - collectedMetrics = {}; -} - -/** - * Generate the markdown report content - */ -function generateReportContent( - validMetrics: [ - string, - { values: number[]; threshold: number; members?: string }, - ][], - currentGeo: string, -): string { - let content = "METRICS SUMMARY\n===============\n\n"; - content += - "Operation | Members | Avg (ms) | Min/Max (ms) | Threshold (ms) | Variance (ms) | Network (ms) | Status\n"; - content += - "----------|---------|----------|--------------|----------------|---------------|-------------|-------\n"; - - // Group metrics by operation name and member count - const operationGroups = groupMetricsByOperation(validMetrics); - - // Generate table rows - for (const group of operationGroups.values()) { - if (!group.operationData) continue; - - const { operationName, members, operationData: data } = group; - const memberCount = members !== "-" ? parseInt(members as string) : 10; - const operationType = operationName.toLowerCase().includes("-") - ? "group" - : "core"; - - // Calculate threshold with correct values - data.threshold = getThresholdForOperation( - operationName, - operationType, - memberCount, - currentGeo, - ); - - // Get network threshold for this operation - const networkThreshold = getThresholdForOperation( - operationName, - "network", - memberCount, - currentGeo, - ); - - const average = calculateAverage(data.values); - const min = Math.min(...data.values); - const max = Math.max(...data.values); - const status = average <= data.threshold ? "PASS ✅" : "FAIL ❌"; - - // Format variance - const variance = Math.round(average - data.threshold); - const varianceFormatted = - variance <= 0 ? variance.toString() : `+${variance}`; - - content += `${operationName} | ${members} | ${Math.round(average)} | ${Math.round(min)}/${Math.round(max)} | ${data.threshold} | ${varianceFormatted} | ${networkThreshold} | ${status}\n`; - } - - return content; -} diff --git a/datadog/thresholds.json b/datadog/thresholds.json deleted file mode 100644 index 348bd2c9..00000000 --- a/datadog/thresholds.json +++ /dev/null @@ -1,121 +0,0 @@ -{ - "core": { - "clientcreate": 350, - "inboxstate": 200, - "createdm": 200, - "sendgm": 200, - "receivegm": 200, - "creategroup": 200, - "creategroupbyidentifiers": 200, - "sendgroupmessage": 200, - "syncgroup": 200, - "updategroupname": 200, - "removemembers": 200, - "addmembers": 200, - "receivegroupmessage": 200 - }, - "network": { - "dns_lookup": 50, - "tcp_connection": 50, - "tls_handshake": 150, - "processing": 100, - "server_call": 250 - }, - "memberBasedThresholds": { - "50": { - "creategroup": 200, - "creategroupbyidentifiers": 200, - "sendgroupmessage": 200, - "syncgroup": 200, - "updategroupname": 200, - "removemembers": 200, - "addmembers": 200, - "receivegroupmessage": 200 - }, - "100": { - "creategroup": 300, - "creategroupbyidentifiers": 300, - "sendgroupmessage": 200, - "syncgroup": 200, - "updategroupname": 200, - "removemembers": 200, - "addmembers": 200, - "receivegroupmessage": 200 - }, - "150": { - "creategroup": 400, - "creategroupbyidentifiers": 400, - "sendgroupmessage": 200, - "syncgroup": 200, - "updategroupname": 200, - "removemembers": 200, - "addmembers": 200, - "receivegroupmessage": 200 - }, - "200": { - "creategroup": 500, - "creategroupbyidentifiers": 500, - "sendgroupmessage": 200, - "syncgroup": 200, - "updategroupname": 200, - "removemembers": 200, - "addmembers": 200, - "receivegroupmessage": 200 - }, - "250": { - "creategroup": 700, - "creategroupbyidentifiers": 700, - "sendgroupmessage": 200, - "syncgroup": 200, - "updategroupname": 200, - "removemembers": 200, - "addmembers": 200, - "receivegroupmessage": 200 - }, - "300": { - "creategroup": 1000, - "creategroupbyidentifiers": 1000, - "sendgroupmessage": 200, - "syncgroup": 200, - "updategroupname": 200, - "removemembers": 200, - "addmembers": 200, - "receivegroupmessage": 200 - }, - "350": { - "creategroup": 1000, - "creategroupbyidentifiers": 1000, - "sendgroupmessage": 200, - "syncgroup": 200, - "updategroupname": 200, - "removemembers": 200, - "addmembers": 200, - "receivegroupmessage": 200 - }, - "400": { - "creategroup": 1300, - "creategroupbyidentifiers": 1300, - "sendgroupmessage": 200, - "syncgroup": 200, - "updategroupname": 200, - "removemembers": 200, - "addmembers": 200, - "receivegroupmessage": 200 - } - }, - "regionMultipliers": { - "us-east": 1, - "us-west": 1, - "europe": 1, - "asia": 1.5, - "south-america": 3 - }, - "GEO_TO_COUNTRY_CODE": { - "us-east": "US", - "us-west": "US", - "europe": "FR", - "asia": "JP", - "south-america": "BR" - }, - "reliability": 99.9 -} diff --git a/datadog/dashboards/workflows.json b/datadog/workflows.json similarity index 100% rename from datadog/dashboards/workflows.json rename to datadog/workflows.json diff --git a/helpers/client.ts b/helpers/client.ts index 6507886d..95d1cee7 100644 --- a/helpers/client.ts +++ b/helpers/client.ts @@ -9,7 +9,7 @@ import { createWalletClient, http, toBytes } from "viem"; import { privateKeyToAccount } from "viem/accounts"; import { sepolia } from "viem/chains"; import { b } from "vitest/dist/chunks/suite.qtkXWc6R.js"; -import { flushMetrics, initDataDog } from "../datadog/helper"; +import { flushMetrics, initDataDog } from "./datadog"; import { createLogger, flushLogger, overrideConsole } from "./logger"; interface User { diff --git a/helpers/datadog.ts b/helpers/datadog.ts new file mode 100644 index 00000000..29685ed6 --- /dev/null +++ b/helpers/datadog.ts @@ -0,0 +1,798 @@ +import { exec } from "child_process"; +import fs from "fs"; +import path from "path"; +import { promisify } from "util"; +import type { WorkerManager } from "@workers/manager"; +import metrics from "datadog-metrics"; + +// Types definitions +interface MemberThresholds { + creategroup: number; + creategroupbyidentifiers: number; + sendgroupmessage: number; + syncgroup: number; + updategroupname: number; + removemembers: number; + addmembers: number; + receivegroupmessage: number; + [key: string]: number; +} + +interface ThresholdsData { + core: { + [key: string]: number; + }; + network: { + [key: string]: number; + }; + memberBasedThresholds: { + [memberCount: string]: MemberThresholds; + }; + regionMultipliers: { + [region: string]: number; + }; + GEO_TO_COUNTRY_CODE: { + [region: string]: string; + }; + reliability: number; +} + +interface NetworkStats { + "DNS Lookup": number; + "TCP Connection": number; + "TLS Handshake": number; + Processing: number; + "Server Call": number; +} + +interface MetricData { + values: number[]; + threshold: number; + members?: string; +} + +interface ParsedTestName { + metricName: string; + metricDescription: string; + testNameExtracted: string; + operationType: string; + operationName: string; + members: string; +} + +type OperationType = "core" | "group" | "network"; +type MetricSubType = "stream" | "poll" | "recovery"; +type MetricType = "delivery" | "order"; + +// Constants +const THRESHOLDS: ThresholdsData = { + core: { + clientcreate: 350, + inboxstate: 200, + createdm: 200, + sendgm: 200, + receivegm: 200, + creategroup: 200, + creategroupbyidentifiers: 200, + sendgroupmessage: 200, + syncgroup: 200, + updategroupname: 200, + removemembers: 200, + addmembers: 200, + receivegroupmessage: 200, + }, + network: { + dns_lookup: 50, + tcp_connection: 50, + tls_handshake: 150, + processing: 100, + server_call: 250, + }, + memberBasedThresholds: { + "50": { + creategroup: 300, + creategroupbyidentifiers: 300, + receivegroupmessage: 300, + sendgroupmessage: 200, + syncgroup: 200, + updategroupname: 200, + removemembers: 200, + addmembers: 200, + }, + "100": { + creategroup: 300, + creategroupbyidentifiers: 300, + receivegroupmessage: 300, + sendgroupmessage: 200, + syncgroup: 200, + updategroupname: 200, + removemembers: 200, + addmembers: 200, + }, + "150": { + creategroup: 400, + creategroupbyidentifiers: 400, + receivegroupmessage: 400, + sendgroupmessage: 200, + syncgroup: 200, + updategroupname: 200, + removemembers: 200, + addmembers: 200, + }, + "200": { + creategroup: 500, + creategroupbyidentifiers: 500, + receivegroupmessage: 500, + sendgroupmessage: 200, + syncgroup: 200, + updategroupname: 200, + removemembers: 200, + addmembers: 200, + }, + "250": { + creategroup: 700, + creategroupbyidentifiers: 700, + receivegroupmessage: 700, + sendgroupmessage: 200, + syncgroup: 200, + updategroupname: 200, + removemembers: 200, + addmembers: 200, + }, + "300": { + creategroup: 1000, + creategroupbyidentifiers: 1000, + receivegroupmessage: 1000, + sendgroupmessage: 200, + syncgroup: 200, + updategroupname: 200, + removemembers: 200, + addmembers: 200, + }, + "350": { + creategroup: 1000, + creategroupbyidentifiers: 1000, + receivegroupmessage: 1000, + sendgroupmessage: 200, + syncgroup: 200, + updategroupname: 200, + removemembers: 200, + addmembers: 200, + }, + "400": { + creategroup: 1300, + creategroupbyidentifiers: 1300, + receivegroupmessage: 1300, + sendgroupmessage: 200, + syncgroup: 200, + updategroupname: 200, + removemembers: 200, + addmembers: 200, + }, + }, + regionMultipliers: { + "us-east": 1, + "us-west": 1, + europe: 1, + asia: 1.5, + "south-america": 3, + }, + GEO_TO_COUNTRY_CODE: { + "us-east": "US", + "us-west": "US", + europe: "FR", + asia: "JP", + "south-america": "BR", + }, + reliability: 99.9, +}; + +// Global state with proper initialization +const state = { + isInitialized: false, + currentGeo: "", + collectedMetrics: {} as Record, +}; + +// Helper functions +/** + * Find the appropriate member bucket for a given member count + */ +function findMemberBucket(members: number): string { + if (members <= 0) return "0"; + + const memberBuckets = Object.keys(THRESHOLDS.memberBasedThresholds) + .map(Number) + .sort((a, b) => a - b); + + // Find highest bucket <= member count + let applicableBucket = "0"; + for (const bucket of memberBuckets) { + if (members >= bucket) { + applicableBucket = bucket.toString(); + } else { + break; + } + } + + return applicableBucket; +} + +/** + * Get the appropriate threshold for an operation based on type, member count, and region + */ +export function getThresholdForOperation( + operation: string, + operationType: OperationType, + members: number = 0, + region: string = "us-east", +): number { + // Normalize inputs + const operationLower = operation.toLowerCase(); + const regionNormalized = region.toLowerCase().trim(); + const regionMultiplier = + THRESHOLDS.regionMultipliers[ + regionNormalized as keyof typeof THRESHOLDS.regionMultipliers + ] || 1.0; + + let baseThreshold = 0; + + if (operationType === "network") { + const networkThreshold = + THRESHOLDS.network[operationLower as keyof typeof THRESHOLDS.network]; + baseThreshold = + typeof networkThreshold === "number" + ? networkThreshold + : THRESHOLDS.network.server_call; + } else if (operationType === "core") { + baseThreshold = + THRESHOLDS.core[operationLower as keyof typeof THRESHOLDS.core] || 0; + } else if (operationType === "group" && members > 0) { + const applicableBucket = findMemberBucket(members); + const memberThresholds = + THRESHOLDS.memberBasedThresholds[ + applicableBucket as keyof typeof THRESHOLDS.memberBasedThresholds + ]; + + if (memberThresholds && operationLower in memberThresholds) { + baseThreshold = + memberThresholds[operationLower as keyof typeof memberThresholds]; + console.log( + `Operation: ${operation}, members: ${members}, bucket: ${applicableBucket}, threshold: ${baseThreshold}`, + ); + } else { + baseThreshold = + THRESHOLDS.core[operationLower as keyof typeof THRESHOLDS.core] || 0; + console.log( + `Operation: ${operation}, members: ${members}, using core threshold: ${baseThreshold}`, + ); + } + } else { + baseThreshold = + THRESHOLDS.core[operationLower as keyof typeof THRESHOLDS.core] || 0; + } + + return Math.round(baseThreshold * regionMultiplier); +} + +/** + * Calculate average of numeric values + */ +export function calculateAverage(values: number[]): number { + if (values.length === 0) return 0; + return values.reduce((sum, val) => sum + val, 0) / values.length; +} + +/** + * Group metrics by operation name and member count + */ +export function groupMetricsByOperation( + metrics: [string, MetricData][], +): Map< + string, + { operationName: string; members: string; operationData: MetricData } +> { + const groups = new Map(); + + for (const [operation, data] of metrics) { + // Use the operation parsing logic from parseTestName for consistency + const parts = operation.split(":"); + const operationPart = parts[0]; + const dashMatch = operationPart.match(/^([a-zA-Z]+)-(\d+)$/); + + const operationName = dashMatch ? dashMatch[1] : operationPart; + const memberCount = dashMatch ? dashMatch[2] : data.members || "-"; + const groupKey = `${operationName}-${memberCount}`; + + if (!groups.has(groupKey)) { + groups.set(groupKey, { + operationName, + members: memberCount, + operationData: null, + }); + } + + groups.get(groupKey).operationData = data; + } + + return groups; +} + +/** + * Get country code from geolocation + */ +function getCountryCodeFromGeo(geolocation: string): string { + return ( + THRESHOLDS.GEO_TO_COUNTRY_CODE[ + geolocation as keyof typeof THRESHOLDS.GEO_TO_COUNTRY_CODE + ] || "US" + ); +} + +// DataDog integration +/** + * Initialize DataDog metrics reporting + */ +export function initDataDog( + testName: string, + envValue: string, + geolocation: string, + apiKey: string, +): boolean { + if (state.isInitialized) { + return true; + } + + if (!testName.includes("ts_")) { + return true; + } + + if (!apiKey) { + console.warn("⚠️ DATADOG_API_KEY not found. Metrics will not be sent."); + return false; + } + + try { + const countryCode = getCountryCodeFromGeo(geolocation); + state.currentGeo = geolocation; + + const initConfig = { + apiKey: apiKey, + defaultTags: [ + `env:${envValue}`, + `test:${testName}`, + `region:${geolocation}`, + `country_iso_code:${countryCode}`, + ], + }; + + metrics.init(initConfig); + state.isInitialized = true; + return true; + } catch (error) { + console.error("❌ Failed to initialize DataDog metrics:", error); + return false; + } +} + +/** + * Send a metric to DataDog and collect for summary + */ +export function sendMetric( + metricName: string, + metricValue: number, + tags: Record, +): void { + if (!state.isInitialized) return; + + try { + const fullMetricName = `xmtp.sdk.${metricName}`; + const allTags = Object.entries({ ...tags }).map( + ([key, value]) => `${key}:${String(value)}`, + ); + + // Add environment tag if not already present + if (!allTags.some((tag) => tag.startsWith("env:"))) { + allTags.push(`env:${process.env.XMTP_ENV as string}`); + } + + // Add version tag if not already present + if (!allTags.some((tag) => tag.startsWith("xv:"))) { + allTags.push(`vm:${process.env.RAILWAY_SERVICE_ID || "unknown"}`); + } + + // Create a distinctive operation key that properly includes member count + // Format: operation_name-member_count (e.g., "createGroup-10") + const memberCount = tags.members || ""; + const operationKey = tags.operation + ? `${tags.operation}${memberCount ? `-${memberCount}` : ""}` + : metricName; + + if (!state.collectedMetrics[operationKey]) { + state.collectedMetrics[operationKey] = { + values: [], + threshold: tags.threshold || 0, + members: memberCount, + }; + } + + state.collectedMetrics[operationKey].values.push(metricValue); + + metrics.gauge(fullMetricName, Math.round(metricValue), allTags); + } catch (error) { + console.error( + `❌ Error sending metric '${metricName}':`, + error instanceof Error ? error.message : String(error), + ); + } +} + +/** + * Send test results metrics + */ +export function sendTestResults(hasFailures: boolean, testName: string): void { + if (!state.isInitialized) { + console.warn("Datadog metrics not initialized"); + return; + } + + try { + const metricValue = hasFailures ? 0 : 1; + sendMetric("workflow", metricValue, { + workflow: testName, + metric_type: "workflow", + }); + } catch (error) { + console.error("Error reporting to Datadog:", error); + } +} + +// Performance tracking +/** + * Send performance metrics for tests + */ +export const sendPerformanceResult = ( + expect: any, + workers: WorkerManager, + start: number, +) => { + const testName = expect.getState().currentTestName; + if (testName) { + console.timeEnd(testName as string); + expect(workers.getWorkers()).toBeDefined(); + expect(workers.getWorkers().length).toBeGreaterThan(0); + void sendPerformanceMetric( + performance.now() - start, + testName as string, + workers.getVersion(), + false, + ); + } +}; + +/** + * Extract operation details from test name + */ +export function parseTestName(testName: string): ParsedTestName { + const metricNameParts = testName.split(":")[0]; + const metricName = metricNameParts.replaceAll(" > ", "."); + const metricDescription = testName.split(":")[1] || ""; + const operationParts = metricName.split("."); + const testNameExtracted = operationParts[0]; + + // Extract operation name and member count + let operationName = ""; + let members = ""; + + if (operationParts[1]) { + // Check formats: "operation-10" and "operation10" + const dashMatch = operationParts[1].match(/^([a-zA-Z]+)-(\d+)$/); + const noSeparatorMatch = operationParts[1].match(/^([a-zA-Z]+)(\d+)$/); + + if (dashMatch) { + operationName = dashMatch[1]; + members = dashMatch[2]; + } else if (noSeparatorMatch) { + operationName = noSeparatorMatch[1]; + members = noSeparatorMatch[2]; + } else { + operationName = operationParts[1]; + } + } + + const operationType = operationName.toLowerCase().includes("group") + ? "group" + : "core"; + + return { + metricName, + metricDescription, + testNameExtracted, + operationType, + operationName, + members, + }; +} + +/** + * Send detailed performance metrics + */ +export async function sendPerformanceMetric( + metricValue: number, + testName: string, + libxmtpVersion: string, + skipNetworkStats: boolean = false, +): Promise { + if (!state.isInitialized) return; + + try { + const { + metricDescription, + testNameExtracted, + operationType, + operationName, + members, + } = parseTestName(testName); + + const threshold = getThresholdForOperation( + operationName, + operationType as OperationType, + parseInt(members) || 0, + state.currentGeo, + ); + + const isSuccess = metricValue <= threshold; + + sendMetric("duration", metricValue, { + libxmtp: libxmtpVersion, + operation: operationName, + test: testNameExtracted, + metric_type: "operation", + metric_subtype: operationType, + description: metricDescription, + members: members, + success: isSuccess, + threshold: threshold, + region: state.currentGeo, + }); + + // Network stats handling + if (!skipNetworkStats) { + const networkStats = await getNetworkStats(); + const countryCode = getCountryCodeFromGeo(state.currentGeo); + + for (const [statName, statValue] of Object.entries(networkStats)) { + const networkMetricValue = Math.round(statValue * 1000); + const networkPhase = statName.toLowerCase().replace(/\s+/g, "_"); + const networkThreshold = getThresholdForOperation( + networkPhase, + "network", + parseInt(members) || 0, + state.currentGeo, + ); + + sendMetric("duration", networkMetricValue, { + libxmtp: libxmtpVersion, + operation: operationName, + test: testNameExtracted, + metric_type: "network", + network_phase: networkPhase, + country_iso_code: countryCode, + members: members, + success: networkMetricValue <= networkThreshold, + threshold: networkThreshold, + region: state.currentGeo, + }); + } + } + } catch (error) { + console.error( + `❌ Error sending performance metric for '${testName}':`, + error, + ); + } +} + +/** + * Send delivery reliability metrics + */ +export function sendDeliveryMetric( + metricValue: number, + version: string, + testName: string, + metricSubType: MetricSubType, + metricType: MetricType, +): void { + const threshold = THRESHOLDS.reliability; + const isSuccess = metricValue >= threshold; + + sendMetric(metricType, Math.round(metricValue), { + libxmtp: version, + test: testName, + metric_type: metricType, + metric_subtype: metricSubType, + success: isSuccess, + threshold: threshold, + }); +} + +// Network performance +const execAsync = promisify(exec); + +/** + * Measure network performance to an endpoint + */ +export async function getNetworkStats( + endpoint = "https://grpc.dev.xmtp.network:443", +): Promise { + const curlCommand = `curl -s -w "\\n{\\"DNS Lookup\\": %{time_namelookup}, \\"TCP Connection\\": %{time_connect}, \\"TLS Handshake\\": %{time_appconnect}, \\"Server Call\\": %{time_starttransfer}}" -o /dev/null --max-time 10 ${endpoint}`; + + let stdout: string; + + try { + const result = await execAsync(curlCommand); + stdout = result.stdout; + } catch (error: any) { + // Even if curl returns an error, we might still have useful stdout + if (error.stdout) { + stdout = error.stdout; + console.warn( + `⚠️ Curl command returned error code ${error.code}, but stdout is available.`, + ); + } else { + console.error(`❌ Curl command failed without stdout:`, error); + throw error; // rethrow if no stdout is available + } + } + + // Parse the JSON response + const stats = JSON.parse(stdout.trim()) as NetworkStats; + + // Handle the case where Server Call time is 0 + if (stats["Server Call"] === 0) { + console.warn( + `Network request to ${endpoint} returned Server Call time of 0.`, + ); + + // Use a reasonable estimate + stats["Server Call"] = stats["TLS Handshake"] + 0.1; + } + + // Calculate processing time + stats["Processing"] = stats["Server Call"] - stats["TLS Handshake"]; + + // Ensure processing time is not negative + if (stats["Processing"] < 0) { + stats["Processing"] = 0; + } + + return stats; +} + +// Reporting and summary +/** + * Flush all metrics and generate summary report + */ +export function flushMetrics(testName: string): Promise { + return new Promise((resolve) => { + if (!state.isInitialized) { + resolve(); + return; + } + + logMetricsSummary(testName); + + void metrics.flush().then(() => { + resolve(); + }); + }); +} + +/** + * Creates and saves the metrics report + */ +function saveMetricsReport( + testName: string, + validMetrics: [string, MetricData][], +): void { + // Create directory for reports + const reportsDir = path.join(process.cwd(), "logs"); + if (!fs.existsSync(reportsDir)) { + fs.mkdirSync(reportsDir, { recursive: true }); + } + + // Create filename with environment info + const filename = path.join( + reportsDir, + `${testName}-${state.currentGeo}-${process.env.XMTP_ENV}.md`, + ); + + try { + fs.writeFileSync(filename, generateReportContent(validMetrics)); + console.log(`✅ Report saved to ${filename}`); + } catch (error) { + console.error(`❌ Error writing metrics summary to file:`, error); + } +} + +/** + * Generate and log a metrics summary report + */ +export function logMetricsSummary(testName: string): void { + if ( + !state.isInitialized || + Object.keys(state.collectedMetrics).length === 0 + ) { + console.log("No metrics collected to summarize"); + return; + } + + console.log("\n📊 Creating metrics summary report"); + + // Filter out workflow metrics and empty value arrays + const validMetrics = Object.entries(state.collectedMetrics).filter( + ([operation, data]) => operation !== "workflow" && data.values.length > 0, + ); + + // Count passed metrics + const passedMetrics = validMetrics.filter( + ([_, data]) => calculateAverage(data.values) <= data.threshold, + ).length; + + const totalMetrics = validMetrics.length; + const passRate = + totalMetrics > 0 ? Math.round((passedMetrics / totalMetrics) * 100) : 0; + + console.log( + `✅ Passed: ${passedMetrics}/${totalMetrics} metrics (${passRate}%)`, + ); + + saveMetricsReport(testName, validMetrics); + + // Reset metrics collection for next test run + state.collectedMetrics = {}; +} + +/** + * Generate the markdown report content + */ +function generateReportContent(validMetrics: [string, MetricData][]): string { + let content = "# METRICS SUMMARY\n\n"; + content += + "| Operation | Members | Avg (ms) | Min/Max (ms) | Threshold (ms) | Variance (ms) | Status |\n"; + content += + "|-----------|---------|----------|--------------|----------------|---------------|---------|\n"; + + // Group metrics by operation name and member count + const operationGroups = groupMetricsByOperation(validMetrics); + + // Generate table rows + for (const group of operationGroups.values()) { + if (!group.operationData) continue; + + const { operationName, members, operationData: data } = group; + const memberCount = members !== "-" ? parseInt(members) : 0; + const operationType = operationName.toLowerCase().includes("group") + ? "group" + : "core"; + + // Calculate metrics data + const threshold = getThresholdForOperation( + operationName, + operationType as OperationType, + memberCount, + state.currentGeo, + ); + data.threshold = threshold; + + const average = calculateAverage(data.values); + const variance = Math.round(average - threshold); + const varianceFormatted = + variance <= 0 ? variance.toString() : `+${variance}`; + + // Format table row + content += `| ${operationName} | ${members} | ${Math.round(average)} | ${Math.round(Math.min(...data.values))}/${Math.round(Math.max(...data.values))} | ${threshold} | ${varianceFormatted} | ${average <= threshold ? "PASS ✅" : "FAIL ❌"} |\n`; + } + + return content; +} diff --git a/scripts/railway-test.ts b/scripts/railway-test.ts index 074530e4..fd1ce6b0 100755 --- a/scripts/railway-test.ts +++ b/scripts/railway-test.ts @@ -1,5 +1,5 @@ import { execSync } from "child_process"; -import { sendTestResults } from "@datadog/helper"; +import { sendTestResults } from "@helpers/datadog"; function runTests(): void { let hasFailures: boolean = false; diff --git a/tests/TS_Delivery.test.ts b/tests/TS_Delivery.test.ts index 1b3c5063..d3cd3efe 100644 --- a/tests/TS_Delivery.test.ts +++ b/tests/TS_Delivery.test.ts @@ -1,5 +1,5 @@ -import { sendDeliveryMetric, sendTestResults } from "@datadog/helper"; import { closeEnv, loadEnv } from "@helpers/client"; +import { sendDeliveryMetric, sendTestResults } from "@helpers/datadog"; import { logError } from "@helpers/logger"; import { defaultValues, diff --git a/tests/TS_Gm.test.ts b/tests/TS_Gm.test.ts index e0d55c90..596e11b3 100644 --- a/tests/TS_Gm.test.ts +++ b/tests/TS_Gm.test.ts @@ -1,5 +1,5 @@ -import { sendTestResults } from "@datadog/helper"; import { closeEnv, loadEnv } from "@helpers/client"; +import { sendTestResults } from "@helpers/datadog"; import generatedInboxes from "@helpers/generated-inboxes.json"; import { logError } from "@helpers/logger"; import { createGroupAndReceiveGm } from "@helpers/playwright"; diff --git a/tests/TS_Performance.test.ts b/tests/TS_Performance.test.ts index d22ef3ba..e2fd6e77 100644 --- a/tests/TS_Performance.test.ts +++ b/tests/TS_Performance.test.ts @@ -1,5 +1,5 @@ -import { sendPerformanceResult, sendTestResults } from "@datadog/helper"; import { closeEnv, loadEnv } from "@helpers/client"; +import { sendPerformanceResult, sendTestResults } from "@helpers/datadog"; import generatedInboxes from "@helpers/generated-inboxes.json"; import { logError } from "@helpers/logger"; import { diff --git a/tests/dms.test.ts b/tests/dms.test.ts index 16bd0139..ee039a55 100644 --- a/tests/dms.test.ts +++ b/tests/dms.test.ts @@ -1,5 +1,5 @@ -import { sendPerformanceResult, sendTestResults } from "@datadog/helper"; import { closeEnv, loadEnv } from "@helpers/client"; +import { sendPerformanceResult, sendTestResults } from "@helpers/datadog"; import { logError } from "@helpers/logger"; import { type Conversation, type WorkerManager } from "@helpers/types"; import { verifyStream } from "@helpers/verify"; diff --git a/tests/groups.test.ts b/tests/groups.test.ts index 830940a4..93095abf 100644 --- a/tests/groups.test.ts +++ b/tests/groups.test.ts @@ -1,5 +1,5 @@ -import { sendPerformanceResult, sendTestResults } from "@datadog/helper"; import { closeEnv, loadEnv } from "@helpers/client"; +import { sendPerformanceResult, sendTestResults } from "@helpers/datadog"; import generatedInboxes from "@helpers/generated-inboxes.json"; import { logError } from "@helpers/logger"; import { diff --git a/tests/offline.test.ts b/tests/offline.test.ts index c07b1180..0d5fd063 100644 --- a/tests/offline.test.ts +++ b/tests/offline.test.ts @@ -1,5 +1,5 @@ -import { sendDeliveryMetric } from "@datadog/helper"; import { closeEnv, loadEnv } from "@helpers/client"; +import { sendDeliveryMetric } from "@helpers/datadog"; import { logError } from "@helpers/logger"; import { type Group, type WorkerManager } from "@helpers/types"; import { calculateMessageStats } from "@helpers/verify"; diff --git a/tsconfig.json b/tsconfig.json index 9d3b1cde..462745de 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -21,7 +21,6 @@ "paths": { "@helpers/*": ["./helpers/*"], "@scripts/*": ["./scripts/*"], - "@datadog/*": ["./datadog/*"], "@workers/*": ["./workers/*"] } }, diff --git a/vitest.config.ts b/vitest.config.ts index 1107724f..aa7ce759 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -8,7 +8,6 @@ export default defineConfig({ "@helpers": resolve(__dirname, "./helpers"), "@workers": resolve(__dirname, "./workers"), "@scripts": resolve(__dirname, "./scripts"), - "@datadog": resolve(__dirname, "./datadog"), }, }, test: { From f5662608a35f28c72728cd3b850a091f5ebd0aad Mon Sep 17 00:00:00 2001 From: fabri Date: Wed, 19 Mar 2025 01:56:13 -0300 Subject: [PATCH 44/57] fix --- helpers/datadog.ts | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/helpers/datadog.ts b/helpers/datadog.ts index 29685ed6..6b10f636 100644 --- a/helpers/datadog.ts +++ b/helpers/datadog.ts @@ -110,9 +110,9 @@ const THRESHOLDS: ThresholdsData = { addmembers: 200, }, "150": { - creategroup: 400, - creategroupbyidentifiers: 400, - receivegroupmessage: 400, + creategroup: 500, + creategroupbyidentifiers: 500, + receivegroupmessage: 500, sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, @@ -120,9 +120,9 @@ const THRESHOLDS: ThresholdsData = { addmembers: 200, }, "200": { - creategroup: 500, - creategroupbyidentifiers: 500, - receivegroupmessage: 500, + creategroup: 700, + creategroupbyidentifiers: 700, + receivegroupmessage: 700, sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, @@ -130,9 +130,9 @@ const THRESHOLDS: ThresholdsData = { addmembers: 200, }, "250": { - creategroup: 700, - creategroupbyidentifiers: 700, - receivegroupmessage: 700, + creategroup: 850, + creategroupbyidentifiers: 850, + receivegroupmessage: 850, sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, @@ -150,9 +150,9 @@ const THRESHOLDS: ThresholdsData = { addmembers: 200, }, "350": { - creategroup: 1000, - creategroupbyidentifiers: 1000, - receivegroupmessage: 1000, + creategroup: 1500, + creategroupbyidentifiers: 1500, + receivegroupmessage: 1500, sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, @@ -160,9 +160,9 @@ const THRESHOLDS: ThresholdsData = { addmembers: 200, }, "400": { - creategroup: 1300, - creategroupbyidentifiers: 1300, - receivegroupmessage: 1300, + creategroup: 1800, + creategroupbyidentifiers: 1800, + receivegroupmessage: 1800, sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, From 9b7e15f1160f6e1d95d50cde1fff26c17608eab2 Mon Sep 17 00:00:00 2001 From: fabri Date: Wed, 19 Mar 2025 01:58:59 -0300 Subject: [PATCH 45/57] fix --- helpers/datadog.ts | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/helpers/datadog.ts b/helpers/datadog.ts index 6b10f636..fbc1a621 100644 --- a/helpers/datadog.ts +++ b/helpers/datadog.ts @@ -82,11 +82,11 @@ const THRESHOLDS: ThresholdsData = { receivegroupmessage: 200, }, network: { - dns_lookup: 50, - tcp_connection: 50, - tls_handshake: 150, + dns_lookup: 100, + tcp_connection: 150, + tls_handshake: 250, processing: 100, - server_call: 250, + server_call: 350, }, memberBasedThresholds: { "50": { @@ -110,9 +110,9 @@ const THRESHOLDS: ThresholdsData = { addmembers: 200, }, "150": { - creategroup: 500, - creategroupbyidentifiers: 500, - receivegroupmessage: 500, + creategroup: 400, + creategroupbyidentifiers: 400, + receivegroupmessage: 400, sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, @@ -120,9 +120,9 @@ const THRESHOLDS: ThresholdsData = { addmembers: 200, }, "200": { - creategroup: 700, - creategroupbyidentifiers: 700, - receivegroupmessage: 700, + creategroup: 500, + creategroupbyidentifiers: 500, + receivegroupmessage: 500, sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, @@ -130,9 +130,9 @@ const THRESHOLDS: ThresholdsData = { addmembers: 200, }, "250": { - creategroup: 850, - creategroupbyidentifiers: 850, - receivegroupmessage: 850, + creategroup: 700, + creategroupbyidentifiers: 700, + receivegroupmessage: 700, sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, @@ -150,9 +150,9 @@ const THRESHOLDS: ThresholdsData = { addmembers: 200, }, "350": { - creategroup: 1500, - creategroupbyidentifiers: 1500, - receivegroupmessage: 1500, + creategroup: 1000, + creategroupbyidentifiers: 1000, + receivegroupmessage: 1000, sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, @@ -160,9 +160,9 @@ const THRESHOLDS: ThresholdsData = { addmembers: 200, }, "400": { - creategroup: 1800, - creategroupbyidentifiers: 1800, - receivegroupmessage: 1800, + creategroup: 1300, + creategroupbyidentifiers: 1300, + receivegroupmessage: 1300, sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, @@ -315,7 +315,10 @@ export function groupMetricsByOperation( groups.get(groupKey).operationData = data; } - return groups; + return groups as Map< + string, + { operationName: string; members: string; operationData: MetricData } + >; } /** From f04c7ccf930b5c54f3a65e6c494d29df04c5e84d Mon Sep 17 00:00:00 2001 From: fabri Date: Wed, 19 Mar 2025 02:01:29 -0300 Subject: [PATCH 46/57] fix --- .github/workflows/TS_Performance.yml | 56 ++++++++++++++-------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/TS_Performance.yml b/.github/workflows/TS_Performance.yml index fb59f27e..73b46ce8 100644 --- a/.github/workflows/TS_Performance.yml +++ b/.github/workflows/TS_Performance.yml @@ -9,34 +9,34 @@ on: workflow_dispatch: jobs: - # dev: - # runs-on: ubuntu-latest - # env: - # DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} - # LOGGING_LEVEL: ${{ vars.LOGGING_LEVEL }} - # XMTP_ENV: "dev" - # GEOLOCATION: ${{ vars.GEOLOCATION }} - # BATCH_SIZE: ${{ vars.BATCH_SIZE }} - # MAX_GROUP_SIZE: ${{ vars.MAX_GROUP_SIZE }} - # steps: - # - uses: actions/checkout@v4 - # - uses: actions/setup-node@v4 - # with: - # node-version-file: ".node-version" - # cache: "yarn" - # env: - # SKIP_YARN_COREPACK_CHECK: "1" - # - run: corepack enable - # - run: yarn - # - name: Make test script executable - # run: chmod +x ./scripts/run-test.sh - # - name: Run tests with retry - # run: ./scripts/run-test.sh TS_Performance - # - name: Upload reports artifacts - # uses: actions/upload-artifact@v4 - # with: - # name: ts_performance_${{ env.XMTP_ENV }} - # path: logs/ + dev: + runs-on: ubuntu-latest + env: + DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} + LOGGING_LEVEL: ${{ vars.LOGGING_LEVEL }} + XMTP_ENV: "dev" + GEOLOCATION: ${{ vars.GEOLOCATION }} + BATCH_SIZE: ${{ vars.BATCH_SIZE }} + MAX_GROUP_SIZE: ${{ vars.MAX_GROUP_SIZE }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: ".node-version" + cache: "yarn" + env: + SKIP_YARN_COREPACK_CHECK: "1" + - run: corepack enable + - run: yarn + - name: Make test script executable + run: chmod +x ./scripts/run-test.sh + - name: Run tests with retry + run: ./scripts/run-test.sh TS_Performance + - name: Upload reports artifacts + uses: actions/upload-artifact@v4 + with: + name: ts_performance_${{ env.XMTP_ENV }} + path: logs/ production: runs-on: ubuntu-latest env: From f73115590d5673c2fb452eb25fe34c94a1c9f5d0 Mon Sep 17 00:00:00 2001 From: fabri Date: Wed, 19 Mar 2025 02:02:47 -0300 Subject: [PATCH 47/57] fix --- .github/workflows/TS_Performance.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/TS_Performance.yml b/.github/workflows/TS_Performance.yml index 73b46ce8..ce964463 100644 --- a/.github/workflows/TS_Performance.yml +++ b/.github/workflows/TS_Performance.yml @@ -33,10 +33,10 @@ jobs: - name: Run tests with retry run: ./scripts/run-test.sh TS_Performance - name: Upload reports artifacts - uses: actions/upload-artifact@v4 - with: - name: ts_performance_${{ env.XMTP_ENV }} - path: logs/ + uses: actions/upload-artifact@v4 + with: + name: ts_performance_${{ env.XMTP_ENV }} + path: logs/ production: runs-on: ubuntu-latest env: From b398450ef72b078f8d517648ddeb019af78f0c5c Mon Sep 17 00:00:00 2001 From: fabri Date: Wed, 19 Mar 2025 02:04:10 -0300 Subject: [PATCH 48/57] fix --- .github/workflows/TS_Performance.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/TS_Performance.yml b/.github/workflows/TS_Performance.yml index ce964463..3466ca45 100644 --- a/.github/workflows/TS_Performance.yml +++ b/.github/workflows/TS_Performance.yml @@ -37,6 +37,7 @@ jobs: with: name: ts_performance_${{ env.XMTP_ENV }} path: logs/ + production: runs-on: ubuntu-latest env: From 119dadfa63a9fc67d624c7ed7abff939fb994ede Mon Sep 17 00:00:00 2001 From: fabri Date: Wed, 19 Mar 2025 02:06:01 -0300 Subject: [PATCH 49/57] fix --- .github/workflows/TS_Performance.yml | 56 ++++++++++++++-------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/TS_Performance.yml b/.github/workflows/TS_Performance.yml index 3466ca45..bf3ecff5 100644 --- a/.github/workflows/TS_Performance.yml +++ b/.github/workflows/TS_Performance.yml @@ -9,34 +9,34 @@ on: workflow_dispatch: jobs: - dev: - runs-on: ubuntu-latest - env: - DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} - LOGGING_LEVEL: ${{ vars.LOGGING_LEVEL }} - XMTP_ENV: "dev" - GEOLOCATION: ${{ vars.GEOLOCATION }} - BATCH_SIZE: ${{ vars.BATCH_SIZE }} - MAX_GROUP_SIZE: ${{ vars.MAX_GROUP_SIZE }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version-file: ".node-version" - cache: "yarn" - env: - SKIP_YARN_COREPACK_CHECK: "1" - - run: corepack enable - - run: yarn - - name: Make test script executable - run: chmod +x ./scripts/run-test.sh - - name: Run tests with retry - run: ./scripts/run-test.sh TS_Performance - - name: Upload reports artifacts - uses: actions/upload-artifact@v4 - with: - name: ts_performance_${{ env.XMTP_ENV }} - path: logs/ + # dev: + # runs-on: ubuntu-latest + # env: + # DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} + # LOGGING_LEVEL: ${{ vars.LOGGING_LEVEL }} + # XMTP_ENV: "dev" + # GEOLOCATION: ${{ vars.GEOLOCATION }} + # BATCH_SIZE: ${{ vars.BATCH_SIZE }} + # MAX_GROUP_SIZE: ${{ vars.MAX_GROUP_SIZE }} + # steps: + # - uses: actions/checkout@v4 + # - uses: actions/setup-node@v4 + # with: + # node-version-file: ".node-version" + # cache: "yarn" + # env: + # SKIP_YARN_COREPACK_CHECK: "1" + # - run: corepack enable + # - run: yarn + # - name: Make test script executable + # run: chmod +x ./scripts/run-test.sh + # - name: Run tests with retry + # run: ./scripts/run-test.sh TS_Performance + # - name: Upload reports artifacts + # uses: actions/upload-artifact@v4 + # with: + # name: ts_performance_${{ env.XMTP_ENV }} + # path: logs/ production: runs-on: ubuntu-latest From 60e1e4efae1577d51731be9bae914069089bc981 Mon Sep 17 00:00:00 2001 From: fabri Date: Wed, 19 Mar 2025 02:06:48 -0300 Subject: [PATCH 50/57] fix --- .github/workflows/TS_Performance.yml | 68 ---------------------------- 1 file changed, 68 deletions(-) diff --git a/.github/workflows/TS_Performance.yml b/.github/workflows/TS_Performance.yml index bf3ecff5..e69de29b 100644 --- a/.github/workflows/TS_Performance.yml +++ b/.github/workflows/TS_Performance.yml @@ -1,68 +0,0 @@ -name: TS_Performance -description: "should verify performance of the library in the dev network" - -on: - pull_request: - branches: [deploy] - schedule: - - cron: "30 * * * *" # Runs at 30 minutes past each hour - workflow_dispatch: - -jobs: - # dev: - # runs-on: ubuntu-latest - # env: - # DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} - # LOGGING_LEVEL: ${{ vars.LOGGING_LEVEL }} - # XMTP_ENV: "dev" - # GEOLOCATION: ${{ vars.GEOLOCATION }} - # BATCH_SIZE: ${{ vars.BATCH_SIZE }} - # MAX_GROUP_SIZE: ${{ vars.MAX_GROUP_SIZE }} - # steps: - # - uses: actions/checkout@v4 - # - uses: actions/setup-node@v4 - # with: - # node-version-file: ".node-version" - # cache: "yarn" - # env: - # SKIP_YARN_COREPACK_CHECK: "1" - # - run: corepack enable - # - run: yarn - # - name: Make test script executable - # run: chmod +x ./scripts/run-test.sh - # - name: Run tests with retry - # run: ./scripts/run-test.sh TS_Performance - # - name: Upload reports artifacts - # uses: actions/upload-artifact@v4 - # with: - # name: ts_performance_${{ env.XMTP_ENV }} - # path: logs/ - - production: - runs-on: ubuntu-latest - env: - DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} - LOGGING_LEVEL: ${{ vars.LOGGING_LEVEL }} - XMTP_ENV: "production" - GEOLOCATION: ${{ vars.GEOLOCATION }} - BATCH_SIZE: ${{ vars.BATCH_SIZE }} - MAX_GROUP_SIZE: ${{ vars.MAX_GROUP_SIZE }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version-file: ".node-version" - cache: "yarn" - env: - SKIP_YARN_COREPACK_CHECK: "1" - - run: corepack enable - - run: yarn - - name: Make test script executable - run: chmod +x ./scripts/run-test.sh - - name: Run tests with retry - run: ./scripts/run-test.sh TS_Performance - - name: Upload reports artifacts - uses: actions/upload-artifact@v4 - with: - name: ts_performance_${{ env.XMTP_ENV }} - path: logs/ From f5d5c2c2081e1ca9457dd0bc69a4a59293d6756b Mon Sep 17 00:00:00 2001 From: fabri Date: Wed, 19 Mar 2025 02:07:19 -0300 Subject: [PATCH 51/57] fix --- .github/workflows/TS_Performance.yml | 68 ++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/.github/workflows/TS_Performance.yml b/.github/workflows/TS_Performance.yml index e69de29b..3466ca45 100644 --- a/.github/workflows/TS_Performance.yml +++ b/.github/workflows/TS_Performance.yml @@ -0,0 +1,68 @@ +name: TS_Performance +description: "should verify performance of the library in the dev network" + +on: + pull_request: + branches: [deploy] + schedule: + - cron: "30 * * * *" # Runs at 30 minutes past each hour + workflow_dispatch: + +jobs: + dev: + runs-on: ubuntu-latest + env: + DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} + LOGGING_LEVEL: ${{ vars.LOGGING_LEVEL }} + XMTP_ENV: "dev" + GEOLOCATION: ${{ vars.GEOLOCATION }} + BATCH_SIZE: ${{ vars.BATCH_SIZE }} + MAX_GROUP_SIZE: ${{ vars.MAX_GROUP_SIZE }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: ".node-version" + cache: "yarn" + env: + SKIP_YARN_COREPACK_CHECK: "1" + - run: corepack enable + - run: yarn + - name: Make test script executable + run: chmod +x ./scripts/run-test.sh + - name: Run tests with retry + run: ./scripts/run-test.sh TS_Performance + - name: Upload reports artifacts + uses: actions/upload-artifact@v4 + with: + name: ts_performance_${{ env.XMTP_ENV }} + path: logs/ + + production: + runs-on: ubuntu-latest + env: + DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} + LOGGING_LEVEL: ${{ vars.LOGGING_LEVEL }} + XMTP_ENV: "production" + GEOLOCATION: ${{ vars.GEOLOCATION }} + BATCH_SIZE: ${{ vars.BATCH_SIZE }} + MAX_GROUP_SIZE: ${{ vars.MAX_GROUP_SIZE }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: ".node-version" + cache: "yarn" + env: + SKIP_YARN_COREPACK_CHECK: "1" + - run: corepack enable + - run: yarn + - name: Make test script executable + run: chmod +x ./scripts/run-test.sh + - name: Run tests with retry + run: ./scripts/run-test.sh TS_Performance + - name: Upload reports artifacts + uses: actions/upload-artifact@v4 + with: + name: ts_performance_${{ env.XMTP_ENV }} + path: logs/ From c2c5393d418495bb479934340e3bcceb166c9325 Mon Sep 17 00:00:00 2001 From: fabri Date: Wed, 19 Mar 2025 02:08:18 -0300 Subject: [PATCH 52/57] fix --- .github/workflows/TS_Performance.yml | 57 ++++++++++++++-------------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/.github/workflows/TS_Performance.yml b/.github/workflows/TS_Performance.yml index 3466ca45..fb59f27e 100644 --- a/.github/workflows/TS_Performance.yml +++ b/.github/workflows/TS_Performance.yml @@ -9,35 +9,34 @@ on: workflow_dispatch: jobs: - dev: - runs-on: ubuntu-latest - env: - DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} - LOGGING_LEVEL: ${{ vars.LOGGING_LEVEL }} - XMTP_ENV: "dev" - GEOLOCATION: ${{ vars.GEOLOCATION }} - BATCH_SIZE: ${{ vars.BATCH_SIZE }} - MAX_GROUP_SIZE: ${{ vars.MAX_GROUP_SIZE }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version-file: ".node-version" - cache: "yarn" - env: - SKIP_YARN_COREPACK_CHECK: "1" - - run: corepack enable - - run: yarn - - name: Make test script executable - run: chmod +x ./scripts/run-test.sh - - name: Run tests with retry - run: ./scripts/run-test.sh TS_Performance - - name: Upload reports artifacts - uses: actions/upload-artifact@v4 - with: - name: ts_performance_${{ env.XMTP_ENV }} - path: logs/ - + # dev: + # runs-on: ubuntu-latest + # env: + # DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} + # LOGGING_LEVEL: ${{ vars.LOGGING_LEVEL }} + # XMTP_ENV: "dev" + # GEOLOCATION: ${{ vars.GEOLOCATION }} + # BATCH_SIZE: ${{ vars.BATCH_SIZE }} + # MAX_GROUP_SIZE: ${{ vars.MAX_GROUP_SIZE }} + # steps: + # - uses: actions/checkout@v4 + # - uses: actions/setup-node@v4 + # with: + # node-version-file: ".node-version" + # cache: "yarn" + # env: + # SKIP_YARN_COREPACK_CHECK: "1" + # - run: corepack enable + # - run: yarn + # - name: Make test script executable + # run: chmod +x ./scripts/run-test.sh + # - name: Run tests with retry + # run: ./scripts/run-test.sh TS_Performance + # - name: Upload reports artifacts + # uses: actions/upload-artifact@v4 + # with: + # name: ts_performance_${{ env.XMTP_ENV }} + # path: logs/ production: runs-on: ubuntu-latest env: From 6be2359e9cf1cca5b279ea1ecf7e06376aa2d4bc Mon Sep 17 00:00:00 2001 From: fabri Date: Wed, 19 Mar 2025 02:09:24 -0300 Subject: [PATCH 53/57] fix --- .github/workflows/TS_Performance.yml | 67 ---------------------------- 1 file changed, 67 deletions(-) delete mode 100644 .github/workflows/TS_Performance.yml diff --git a/.github/workflows/TS_Performance.yml b/.github/workflows/TS_Performance.yml deleted file mode 100644 index fb59f27e..00000000 --- a/.github/workflows/TS_Performance.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: TS_Performance -description: "should verify performance of the library in the dev network" - -on: - pull_request: - branches: [deploy] - schedule: - - cron: "30 * * * *" # Runs at 30 minutes past each hour - workflow_dispatch: - -jobs: - # dev: - # runs-on: ubuntu-latest - # env: - # DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} - # LOGGING_LEVEL: ${{ vars.LOGGING_LEVEL }} - # XMTP_ENV: "dev" - # GEOLOCATION: ${{ vars.GEOLOCATION }} - # BATCH_SIZE: ${{ vars.BATCH_SIZE }} - # MAX_GROUP_SIZE: ${{ vars.MAX_GROUP_SIZE }} - # steps: - # - uses: actions/checkout@v4 - # - uses: actions/setup-node@v4 - # with: - # node-version-file: ".node-version" - # cache: "yarn" - # env: - # SKIP_YARN_COREPACK_CHECK: "1" - # - run: corepack enable - # - run: yarn - # - name: Make test script executable - # run: chmod +x ./scripts/run-test.sh - # - name: Run tests with retry - # run: ./scripts/run-test.sh TS_Performance - # - name: Upload reports artifacts - # uses: actions/upload-artifact@v4 - # with: - # name: ts_performance_${{ env.XMTP_ENV }} - # path: logs/ - production: - runs-on: ubuntu-latest - env: - DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} - LOGGING_LEVEL: ${{ vars.LOGGING_LEVEL }} - XMTP_ENV: "production" - GEOLOCATION: ${{ vars.GEOLOCATION }} - BATCH_SIZE: ${{ vars.BATCH_SIZE }} - MAX_GROUP_SIZE: ${{ vars.MAX_GROUP_SIZE }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version-file: ".node-version" - cache: "yarn" - env: - SKIP_YARN_COREPACK_CHECK: "1" - - run: corepack enable - - run: yarn - - name: Make test script executable - run: chmod +x ./scripts/run-test.sh - - name: Run tests with retry - run: ./scripts/run-test.sh TS_Performance - - name: Upload reports artifacts - uses: actions/upload-artifact@v4 - with: - name: ts_performance_${{ env.XMTP_ENV }} - path: logs/ From 517737a10832bc898da658e20700f9d8839e5489 Mon Sep 17 00:00:00 2001 From: fabri Date: Wed, 19 Mar 2025 02:10:47 -0300 Subject: [PATCH 54/57] fix --- .github/workflows/TS_Performance.yml | 68 ++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 .github/workflows/TS_Performance.yml diff --git a/.github/workflows/TS_Performance.yml b/.github/workflows/TS_Performance.yml new file mode 100644 index 00000000..3466ca45 --- /dev/null +++ b/.github/workflows/TS_Performance.yml @@ -0,0 +1,68 @@ +name: TS_Performance +description: "should verify performance of the library in the dev network" + +on: + pull_request: + branches: [deploy] + schedule: + - cron: "30 * * * *" # Runs at 30 minutes past each hour + workflow_dispatch: + +jobs: + dev: + runs-on: ubuntu-latest + env: + DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} + LOGGING_LEVEL: ${{ vars.LOGGING_LEVEL }} + XMTP_ENV: "dev" + GEOLOCATION: ${{ vars.GEOLOCATION }} + BATCH_SIZE: ${{ vars.BATCH_SIZE }} + MAX_GROUP_SIZE: ${{ vars.MAX_GROUP_SIZE }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: ".node-version" + cache: "yarn" + env: + SKIP_YARN_COREPACK_CHECK: "1" + - run: corepack enable + - run: yarn + - name: Make test script executable + run: chmod +x ./scripts/run-test.sh + - name: Run tests with retry + run: ./scripts/run-test.sh TS_Performance + - name: Upload reports artifacts + uses: actions/upload-artifact@v4 + with: + name: ts_performance_${{ env.XMTP_ENV }} + path: logs/ + + production: + runs-on: ubuntu-latest + env: + DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} + LOGGING_LEVEL: ${{ vars.LOGGING_LEVEL }} + XMTP_ENV: "production" + GEOLOCATION: ${{ vars.GEOLOCATION }} + BATCH_SIZE: ${{ vars.BATCH_SIZE }} + MAX_GROUP_SIZE: ${{ vars.MAX_GROUP_SIZE }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: ".node-version" + cache: "yarn" + env: + SKIP_YARN_COREPACK_CHECK: "1" + - run: corepack enable + - run: yarn + - name: Make test script executable + run: chmod +x ./scripts/run-test.sh + - name: Run tests with retry + run: ./scripts/run-test.sh TS_Performance + - name: Upload reports artifacts + uses: actions/upload-artifact@v4 + with: + name: ts_performance_${{ env.XMTP_ENV }} + path: logs/ From 64c4c810d3f02c9131234de364af8dc40cc6aee5 Mon Sep 17 00:00:00 2001 From: fabri Date: Wed, 19 Mar 2025 02:25:51 -0300 Subject: [PATCH 55/57] fix --- README.md | 25 ++++++++++++++----------- helpers/datadog.ts | 41 ++++++++++++++++++++--------------------- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index c1c2647e..df63fd27 100644 --- a/README.md +++ b/README.md @@ -105,17 +105,20 @@ We can test all XMTP bindings using three main applications. We use [xmtp.chat]( ### Core SDK Operations Performance -| Operation | Description | Avg (ms) | Target | Status | -| ------------------- | -------------------------------------- | -------- | ------ | ------------ | -| createDM | Creating a direct message conversation | 254-306 | <500ms | ✅ On Target | -| sendGM | Sending a group message | 123-132 | <200ms | ✅ On Target | -| receiveGM | Receiving a group message | 90-94 | <200ms | ✅ On Target | -| receiveGroupMessage | Processing group message streams | 119-127 | <200ms | ✅ On Target | -| updateGroupName | Updating group metadata | 105-108 | <200ms | ✅ On Target | -| syncGroup | Syncing group state | 78-89 | <200ms | ✅ On Target | -| addMembers | Adding participants to a group | 238-280 | <500ms | ✅ On Target | -| removeMembers | Removing participants from a group | 147-168 | <300ms | ✅ On Target | -| inboxState | Checking inbox state | 36 | <100ms | ✅ On Target | +| Operation | Description | Avg (ms) | Target | Status | +| ------------------------ | -------------------------------------- | -------- | ------ | ------------ | +| clientCreate | Creating a client | 254-306 | <350ms | ✅ On Target | +| inboxState | Checking inbox state | 300 | <350ms | ✅ On Target | +| createDM | Creating a direct message conversation | 200-250 | <350ms | ✅ On Target | +| sendGM | Sending a group message | 123-160 | <200ms | ✅ On Target | +| receiveGM | Receiving a group message | 90-140 | <200ms | ✅ On Target | +| createGroup | Creating a group | 254-306 | <350ms | ✅ On Target | +| createGroupByIdentifiers | Creating a group by address | 254-306 | <350ms | ✅ On Target | +| syncGroup | Syncing group state | 78-89 | <200ms | ✅ On Target | +| updateGroupName | Updating group metadata | 105-160 | <200ms | ✅ On Target | +| removeMembers | Removing participants from a group | 110-168 | <250ms | ✅ On Target | +| sendGroupMessage | Sending a group message | 100-127 | <200ms | ✅ On Target | +| receiveGroupMessage | Processing group message streams | 119-127 | <200ms | ✅ On Target | _Note: Based on data from 79 measured operations in the `us-east` region and `production` network._ diff --git a/helpers/datadog.ts b/helpers/datadog.ts index fbc1a621..3d3228b8 100644 --- a/helpers/datadog.ts +++ b/helpers/datadog.ts @@ -68,17 +68,16 @@ type MetricType = "delivery" | "order"; const THRESHOLDS: ThresholdsData = { core: { clientcreate: 350, - inboxstate: 200, - createdm: 200, + inboxstate: 350, + createdm: 350, sendgm: 200, receivegm: 200, - creategroup: 200, - creategroupbyidentifiers: 200, - sendgroupmessage: 200, + creategroup: 350, + creategroupbyidentifiers: 350, syncgroup: 200, updategroupname: 200, - removemembers: 200, - addmembers: 200, + removemembers: 250, + sendgroupmessage: 200, receivegroupmessage: 200, }, network: { @@ -90,23 +89,23 @@ const THRESHOLDS: ThresholdsData = { }, memberBasedThresholds: { "50": { - creategroup: 300, - creategroupbyidentifiers: 300, - receivegroupmessage: 300, + creategroup: 350, + creategroupbyidentifiers: 350, + receivegroupmessage: 350, sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, - removemembers: 200, + removemembers: 250, addmembers: 200, }, "100": { - creategroup: 300, - creategroupbyidentifiers: 300, - receivegroupmessage: 300, + creategroup: 400, + creategroupbyidentifiers: 400, + receivegroupmessage: 400, sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, - removemembers: 200, + removemembers: 300, addmembers: 200, }, "150": { @@ -116,7 +115,7 @@ const THRESHOLDS: ThresholdsData = { sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, - removemembers: 200, + removemembers: 300, addmembers: 200, }, "200": { @@ -126,7 +125,7 @@ const THRESHOLDS: ThresholdsData = { sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, - removemembers: 200, + removemembers: 300, addmembers: 200, }, "250": { @@ -136,7 +135,7 @@ const THRESHOLDS: ThresholdsData = { sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, - removemembers: 200, + removemembers: 400, addmembers: 200, }, "300": { @@ -146,7 +145,7 @@ const THRESHOLDS: ThresholdsData = { sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, - removemembers: 200, + removemembers: 400, addmembers: 200, }, "350": { @@ -156,7 +155,7 @@ const THRESHOLDS: ThresholdsData = { sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, - removemembers: 200, + removemembers: 400, addmembers: 200, }, "400": { @@ -166,7 +165,7 @@ const THRESHOLDS: ThresholdsData = { sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, - removemembers: 200, + removemembers: 500, addmembers: 200, }, }, From ff5b58d6b6f83bd0220c8d1ed91082fc5c59850d Mon Sep 17 00:00:00 2001 From: fabri Date: Wed, 19 Mar 2025 02:32:15 -0300 Subject: [PATCH 56/57] fix --- README.md | 56 ++++++++++++++-------------------------------- helpers/datadog.ts | 30 ++++++++++++------------- 2 files changed, 32 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index df63fd27..e58e615d 100644 --- a/README.md +++ b/README.md @@ -126,14 +126,14 @@ _Note: Based on data from 79 measured operations in the `us-east` region and `pr | Size | Create(ms) | Send(ms) | Sync(ms) | Update(ms) | Remove(ms) | Target(Create) | Status | | ---- | ---------- | -------- | -------- | ---------- | ---------- | -------------- | ---------------------- | -| 50 | 990 | 71 | 61 | 81 | 140 | <2,000ms | ✅ On Target | -| 100 | 1,599 | 67 | 66 | 91 | 182 | <2,000ms | ✅ On Target | -| 150 | 2,956 | 72 | 85 | 104 | 183 | <4,000ms | ✅ On Target | -| 200 | 4,598 | 73 | 103 | 139 | 211 | <5,000ms | ✅ On Target | -| 250 | 5,983 | 76 | 120 | 164 | 234 | <7,000ms | ✅ On Target | -| 300 | 8,707 | 81 | 321 | 255 | 309 | <9,000ms | ✅ On Target | -| 350 | 9,826 | 79 | 132 | 228 | 368 | <11,000ms | ⚠️ Performance Concern | -| 400 | 11,451 | 84 | 170 | 427 | 501 | <15,000ms | ⚠️ Performance Concern | +| 50 | 323 | 71 | 61 | 81 | 140 | <350ms | ✅ On Target | +| 100 | 289 | 67 | 66 | 91 | 182 | <350ms | ✅ On Target | +| 150 | 330 | 72 | 85 | 104 | 183 | <400ms | ✅ On Target | +| 200 | 474 | 73 | 103 | 139 | 211 | <500ms | ✅ On Target | +| 250 | 654 | 76 | 120 | 164 | 234 | <700ms | ✅ On Target | +| 300 | 758 | 81 | 321 | 255 | 309 | <1000ms | ✅ On Target | +| 350 | 1064 | 79 | 132 | 228 | 368 | <1000ms | ⚠️ Performance Concern | +| 400 | 1305 | 84 | 170 | 427 | 501 | <1300ms | ⚠️ Performance Concern | | 450 | - | - | - | - | - | - | ❌ Severe impact | _Note: Performance increases significantly beyond `350` members, which represents a hard limit on the protocol._ @@ -148,7 +148,7 @@ _Note: Performance increases significantly beyond `350` members, which represent | TCP Connection | 105.6ms avg | <200ms | ✅ On Target | | TLS Handshake | 238.9ms avg | <300ms | ✅ On Target | | Processing | 30ms avg | <100ms | ✅ On Target | -| Server Call | 238.9ms avg | <400ms | ✅ On Target | +| Server Call | 238.9ms avg | <300ms | ✅ On Target | _Note: Performance metrics based on `us-east` testing on `production` network._ @@ -180,14 +180,14 @@ _Note: `Production` network consistently shows better network performance across ### Message delivery testing -| Test Area | Current Performance | Target | Status | -| ---------------------- | ------------------- | --------------- | ------------ | -| Stream Delivery Rate | 100% successful | 99.9% minimum | ✅ On Target | -| Poll Delivery Rate | 100% successful | 99.9% minimum | ✅ On Target | -| Stream Order | 100% in order | 100% in order | ✅ On Target | -| Poll Order | 100% in order | 100% in order | ✅ On Target | -| Offline Recovery Rate | 100% successful | 100% successful | ✅ On Target | -| Offline Recovery Order | 100% in order | 100% in order | ✅ On Target | +| Test Area | Current Performance | Target | Status | +| -------------------- | ------------------- | -------------- | ------------ | +| Stream Delivery Rate | 100% successful | 99.9% minimum | ✅ On Target | +| Poll Delivery Rate | 100% successful | 99.9% minimum | ✅ On Target | +| Recovery Rate | 100% successful | 99.9% minimum | ✅ On Target | +| Stream Order | 100% in order | 99.9% in order | ✅ On Target | +| Poll Order | 100% in order | 99.9% in order | ✅ On Target | +| Recovery Order | 100% in order | 99.9% in order | ✅ On Target | _Note: Testing regularly in groups of `40` active members listening to one user sending 100 messages_ @@ -222,28 +222,6 @@ _Note: A hybrid approach using `stream` and `poll`-based verification provides t - **Node-sdk only**: Metrics are based on node-sdk only operations and are not covering performance across all SDKs. - **Pre-Release Status**: This assessment reflects the current development version targeting the `4.0.0` stable release. Optimizations and improvements are ongoing. -## Other - -### Cross-SDK Testing - -| SDK Combination | Test Focus | Status | -| ------------------------ | ----------------------------- | ----------- | -| Node SDK ↔ Node SDK | Agent-to-Agent communication | ✅ Verified | -| Web ↔ Node SDK | Client-to-Agent communication | ✅ Verified | -| React Native ↔ Node SDK | Client-to-Agent communication | ✅ Verified | - -_Note: Cross-SDK was tested using the `operations` describe above and is not covering all edge cases._ - -### Package Manager Test Results - -| Package Manager | Node 20 | Node 21 | Node 22 | Node 23 | -| --------------- | ------- | ------- | ------- | ------- | -| pnpm | ✅ | ✅ | ✅ | ✅ | -| npm | ✅ | ✅ | ✅ | ✅ | -| yarn@4.6.0 | ✅ | ✅ | ✅ | ✅ | -| yarn@1.22.19 | ✅ | ✅ | ✅ | ✅ | -| bun | ✅ | ✅ | ✅ | ✅ | - ## Tools & Utilities - **Repository:** [xmtp-qa-testing](https://github.com/xmtp/xmtp-qa-testing): This monorepo contains multiple tools for testing and monitoring diff --git a/helpers/datadog.ts b/helpers/datadog.ts index 3d3228b8..1c49e90b 100644 --- a/helpers/datadog.ts +++ b/helpers/datadog.ts @@ -119,9 +119,9 @@ const THRESHOLDS: ThresholdsData = { addmembers: 200, }, "200": { - creategroup: 500, - creategroupbyidentifiers: 500, - receivegroupmessage: 500, + creategroup: 700, + creategroupbyidentifiers: 700, + receivegroupmessage: 700, sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, @@ -129,9 +129,9 @@ const THRESHOLDS: ThresholdsData = { addmembers: 200, }, "250": { - creategroup: 700, - creategroupbyidentifiers: 700, - receivegroupmessage: 700, + creategroup: 900, + creategroupbyidentifiers: 900, + receivegroupmessage: 900, sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, @@ -139,9 +139,9 @@ const THRESHOLDS: ThresholdsData = { addmembers: 200, }, "300": { - creategroup: 1000, - creategroupbyidentifiers: 1000, - receivegroupmessage: 1000, + creategroup: 1100, + creategroupbyidentifiers: 1100, + receivegroupmessage: 1100, sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, @@ -149,9 +149,9 @@ const THRESHOLDS: ThresholdsData = { addmembers: 200, }, "350": { - creategroup: 1000, - creategroupbyidentifiers: 1000, - receivegroupmessage: 1000, + creategroup: 1300, + creategroupbyidentifiers: 1300, + receivegroupmessage: 1300, sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, @@ -159,9 +159,9 @@ const THRESHOLDS: ThresholdsData = { addmembers: 200, }, "400": { - creategroup: 1300, - creategroupbyidentifiers: 1300, - receivegroupmessage: 1300, + creategroup: 1500, + creategroupbyidentifiers: 1500, + receivegroupmessage: 1500, sendgroupmessage: 200, syncgroup: 200, updategroupname: 200, From ef9f73249e4d19d85f919a7a61bcfbccefa52d8f Mon Sep 17 00:00:00 2001 From: fabri Date: Wed, 19 Mar 2025 02:32:49 -0300 Subject: [PATCH 57/57] fix --- README.md | 14 +++++++------- helpers/datadog.ts | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index e58e615d..9364a931 100644 --- a/README.md +++ b/README.md @@ -127,13 +127,13 @@ _Note: Based on data from 79 measured operations in the `us-east` region and `pr | Size | Create(ms) | Send(ms) | Sync(ms) | Update(ms) | Remove(ms) | Target(Create) | Status | | ---- | ---------- | -------- | -------- | ---------- | ---------- | -------------- | ---------------------- | | 50 | 323 | 71 | 61 | 81 | 140 | <350ms | ✅ On Target | -| 100 | 289 | 67 | 66 | 91 | 182 | <350ms | ✅ On Target | -| 150 | 330 | 72 | 85 | 104 | 183 | <400ms | ✅ On Target | -| 200 | 474 | 73 | 103 | 139 | 211 | <500ms | ✅ On Target | -| 250 | 654 | 76 | 120 | 164 | 234 | <700ms | ✅ On Target | -| 300 | 758 | 81 | 321 | 255 | 309 | <1000ms | ✅ On Target | -| 350 | 1064 | 79 | 132 | 228 | 368 | <1000ms | ⚠️ Performance Concern | -| 400 | 1305 | 84 | 170 | 427 | 501 | <1300ms | ⚠️ Performance Concern | +| 100 | 289 | 67 | 66 | 91 | 182 | <400ms | ✅ On Target | +| 150 | 330 | 72 | 85 | 104 | 183 | <500ms | ✅ On Target | +| 200 | 474 | 73 | 103 | 139 | 211 | <700ms | ✅ On Target | +| 250 | 654 | 76 | 120 | 164 | 234 | <900ms | ✅ On Target | +| 300 | 758 | 81 | 321 | 255 | 309 | <1100ms | ✅ On Target | +| 350 | 1064 | 79 | 132 | 228 | 368 | <1300ms | ⚠️ Performance Concern | +| 400 | 1305 | 84 | 170 | 427 | 501 | <1400ms | ⚠️ Performance Concern | | 450 | - | - | - | - | - | - | ❌ Severe impact | _Note: Performance increases significantly beyond `350` members, which represents a hard limit on the protocol._ diff --git a/helpers/datadog.ts b/helpers/datadog.ts index 1c49e90b..1012f1e7 100644 --- a/helpers/datadog.ts +++ b/helpers/datadog.ts @@ -109,9 +109,9 @@ const THRESHOLDS: ThresholdsData = { addmembers: 200, }, "150": { - creategroup: 400, - creategroupbyidentifiers: 400, - receivegroupmessage: 400, + creategroup: 500, + creategroupbyidentifiers: 500, + receivegroupmessage: 500, sendgroupmessage: 200, syncgroup: 200, updategroupname: 200,