diff --git a/.github/workflows/TS_Delivery.yml b/.github/workflows/TS_Delivery.yml index 2d4a95b5..504be80f 100644 --- a/.github/workflows/TS_Delivery.yml +++ b/.github/workflows/TS_Delivery.yml @@ -34,10 +34,10 @@ jobs: run: chmod +x ./scripts/run-test.sh - name: Run tests with retry run: ./scripts/run-test.sh TS_Delivery - - name: Upload log files + - name: Upload reports artifacts uses: actions/upload-artifact@v4 with: - name: delivery_logs_dev + name: ts_delivery_${{ env.XMTP_ENV }} path: logs/ production: runs-on: ubuntu-latest @@ -66,8 +66,8 @@ jobs: run: chmod +x ./scripts/run-test.sh - name: Run tests with retry run: ./scripts/run-test.sh TS_Delivery - - name: Upload log files + - name: Upload reports artifacts uses: actions/upload-artifact@v4 with: - name: delivery_logs_production + name: ts_delivery_${{ env.XMTP_ENV }} path: logs/ diff --git a/.github/workflows/TS_Geolocation.yml b/.github/workflows/TS_Geolocation.yml index 311d61e8..0ca725a9 100644 --- a/.github/workflows/TS_Geolocation.yml +++ b/.github/workflows/TS_Geolocation.yml @@ -71,7 +71,7 @@ jobs: - name: Upload log files uses: actions/upload-artifact@v4 with: - name: geolocation_logs_dev + name: ts_geolocation_${{ env.XMTP_ENV }} path: logs/ - name: Check for failures @@ -154,7 +154,7 @@ jobs: - name: Upload log files uses: actions/upload-artifact@v4 with: - name: geolocation_logs_production + name: ts_geolocation_${{ env.XMTP_ENV }} path: logs/ - name: Check for failures diff --git a/.github/workflows/TS_Gm.yml b/.github/workflows/TS_Gm.yml index 9badd501..ebef9e6d 100644 --- a/.github/workflows/TS_Gm.yml +++ b/.github/workflows/TS_Gm.yml @@ -38,18 +38,11 @@ jobs: run: chmod +x ./scripts/run-test.sh - name: Run tests with retry run: ./scripts/run-test.sh TS_Gm - - name: Upload log files + - name: Upload reports artifacts uses: actions/upload-artifact@v4 - if: always() with: - name: gm_logs_dev + name: ts_gm_${{ env.XMTP_ENV }} path: logs/ - - name: Upload snapshots - uses: actions/upload-artifact@v4 - if: always() - with: - name: playwright_snapshots - path: logs/snapshots/*.png production: runs-on: ubuntu-latest env: @@ -83,13 +76,6 @@ jobs: run: ./scripts/run-test.sh TS_Gm - name: Upload log files uses: actions/upload-artifact@v4 - if: always() with: - name: gm_logs_production + name: ts_gm_${{ env.XMTP_ENV }} path: logs/ - - name: Upload snapshots - uses: actions/upload-artifact@v4 - if: always() - with: - name: playwright_snapshots - path: logs/snapshots/*.png diff --git a/.github/workflows/TS_Performance.yml b/.github/workflows/TS_Performance.yml index 16355068..3466ca45 100644 --- a/.github/workflows/TS_Performance.yml +++ b/.github/workflows/TS_Performance.yml @@ -32,11 +32,12 @@ jobs: run: chmod +x ./scripts/run-test.sh - name: Run tests with retry run: ./scripts/run-test.sh TS_Performance - - name: Upload log files + - name: Upload reports artifacts uses: actions/upload-artifact@v4 with: - name: performance_logs_dev + name: ts_performance_${{ env.XMTP_ENV }} path: logs/ + production: runs-on: ubuntu-latest env: @@ -60,8 +61,8 @@ jobs: run: chmod +x ./scripts/run-test.sh - name: Run tests with retry run: ./scripts/run-test.sh TS_Performance - - name: Upload log files + - name: Upload reports artifacts uses: actions/upload-artifact@v4 with: - name: performance_logs_production + name: ts_performance_${{ env.XMTP_ENV }} path: logs/ diff --git a/README.md b/README.md index 3bb8cdaa..9364a931 100644 --- a/README.md +++ b/README.md @@ -105,17 +105,20 @@ We can test all XMTP bindings using three main applications. We use [xmtp.chat]( ### Core SDK Operations Performance -| Operation | Description | Avg (ms) | Target | Status | -| ------------------- | -------------------------------------- | -------- | ------ | ------------ | -| createDM | Creating a direct message conversation | 254-306 | <500ms | ✅ On Target | -| sendGM | Sending a group message | 123-132 | <200ms | ✅ On Target | -| receiveGM | Receiving a group message | 90-94 | <200ms | ✅ On Target | -| receiveGroupMessage | Processing group message streams | 119-127 | <200ms | ✅ On Target | -| updateGroupName | Updating group metadata | 105-108 | <200ms | ✅ On Target | -| syncGroup | Syncing group state | 78-89 | <200ms | ✅ On Target | -| addMembers | Adding participants to a group | 238-280 | <500ms | ✅ On Target | -| removeMembers | Removing participants from a group | 147-168 | <300ms | ✅ On Target | -| inboxState | Checking inbox state | 36 | <100ms | ✅ On Target | +| Operation | Description | Avg (ms) | Target | Status | +| ------------------------ | -------------------------------------- | -------- | ------ | ------------ | +| clientCreate | Creating a client | 254-306 | <350ms | ✅ On Target | +| inboxState | Checking inbox state | 300 | <350ms | ✅ On Target | +| createDM | Creating a direct message conversation | 200-250 | <350ms | ✅ On Target | +| sendGM | Sending a group message | 123-160 | <200ms | ✅ On Target | +| receiveGM | Receiving a group message | 90-140 | <200ms | ✅ On Target | +| createGroup | Creating a group | 254-306 | <350ms | ✅ On Target | +| createGroupByIdentifiers | Creating a group by address | 254-306 | <350ms | ✅ On Target | +| syncGroup | Syncing group state | 78-89 | <200ms | ✅ On Target | +| updateGroupName | Updating group metadata | 105-160 | <200ms | ✅ On Target | +| removeMembers | Removing participants from a group | 110-168 | <250ms | ✅ On Target | +| sendGroupMessage | Sending a group message | 100-127 | <200ms | ✅ On Target | +| receiveGroupMessage | Processing group message streams | 119-127 | <200ms | ✅ On Target | _Note: Based on data from 79 measured operations in the `us-east` region and `production` network._ @@ -123,14 +126,14 @@ _Note: Based on data from 79 measured operations in the `us-east` region and `pr | Size | Create(ms) | Send(ms) | Sync(ms) | Update(ms) | Remove(ms) | Target(Create) | Status | | ---- | ---------- | -------- | -------- | ---------- | ---------- | -------------- | ---------------------- | -| 50 | 990 | 71 | 61 | 81 | 140 | <2,000ms | ✅ On Target | -| 100 | 1,599 | 67 | 66 | 91 | 182 | <2,000ms | ✅ On Target | -| 150 | 2,956 | 72 | 85 | 104 | 183 | <4,000ms | ✅ On Target | -| 200 | 4,598 | 73 | 103 | 139 | 211 | <5,000ms | ✅ On Target | -| 250 | 5,983 | 76 | 120 | 164 | 234 | <7,000ms | ✅ On Target | -| 300 | 8,707 | 81 | 321 | 255 | 309 | <9,000ms | ✅ On Target | -| 350 | 9,826 | 79 | 132 | 228 | 368 | <11,000ms | ⚠️ Performance Concern | -| 400 | 11,451 | 84 | 170 | 427 | 501 | <15,000ms | ⚠️ Performance Concern | +| 50 | 323 | 71 | 61 | 81 | 140 | <350ms | ✅ On Target | +| 100 | 289 | 67 | 66 | 91 | 182 | <400ms | ✅ On Target | +| 150 | 330 | 72 | 85 | 104 | 183 | <500ms | ✅ On Target | +| 200 | 474 | 73 | 103 | 139 | 211 | <700ms | ✅ On Target | +| 250 | 654 | 76 | 120 | 164 | 234 | <900ms | ✅ On Target | +| 300 | 758 | 81 | 321 | 255 | 309 | <1100ms | ✅ On Target | +| 350 | 1064 | 79 | 132 | 228 | 368 | <1300ms | ⚠️ Performance Concern | +| 400 | 1305 | 84 | 170 | 427 | 501 | <1400ms | ⚠️ Performance Concern | | 450 | - | - | - | - | - | - | ❌ Severe impact | _Note: Performance increases significantly beyond `350` members, which represents a hard limit on the protocol._ @@ -145,7 +148,7 @@ _Note: Performance increases significantly beyond `350` members, which represent | TCP Connection | 105.6ms avg | <200ms | ✅ On Target | | TLS Handshake | 238.9ms avg | <300ms | ✅ On Target | | Processing | 30ms avg | <100ms | ✅ On Target | -| Server Call | 238.9ms avg | <400ms | ✅ On Target | +| Server Call | 238.9ms avg | <300ms | ✅ On Target | _Note: Performance metrics based on `us-east` testing on `production` network._ @@ -177,14 +180,14 @@ _Note: `Production` network consistently shows better network performance across ### Message delivery testing -| Test Area | Current Performance | Target | Status | -| ---------------------- | ------------------- | --------------- | ------------ | -| Stream Delivery Rate | 100% successful | 99.9% minimum | ✅ On Target | -| Poll Delivery Rate | 100% successful | 99.9% minimum | ✅ On Target | -| Stream Order | 100% in order | 100% in order | ✅ On Target | -| Poll Order | 100% in order | 100% in order | ✅ On Target | -| Offline Recovery Rate | 100% successful | 100% successful | ✅ On Target | -| Offline Recovery Order | 100% in order | 100% in order | ✅ On Target | +| Test Area | Current Performance | Target | Status | +| -------------------- | ------------------- | -------------- | ------------ | +| Stream Delivery Rate | 100% successful | 99.9% minimum | ✅ On Target | +| Poll Delivery Rate | 100% successful | 99.9% minimum | ✅ On Target | +| Recovery Rate | 100% successful | 99.9% minimum | ✅ On Target | +| Stream Order | 100% in order | 99.9% in order | ✅ On Target | +| Poll Order | 100% in order | 99.9% in order | ✅ On Target | +| Recovery Order | 100% in order | 99.9% in order | ✅ On Target | _Note: Testing regularly in groups of `40` active members listening to one user sending 100 messages_ @@ -219,34 +222,12 @@ _Note: A hybrid approach using `stream` and `poll`-based verification provides t - **Node-sdk only**: Metrics are based on node-sdk only operations and are not covering performance across all SDKs. - **Pre-Release Status**: This assessment reflects the current development version targeting the `4.0.0` stable release. Optimizations and improvements are ongoing. -## Other - -### Cross-SDK Testing - -| SDK Combination | Test Focus | Status | -| ------------------------ | ----------------------------- | ----------- | -| Node SDK ↔ Node SDK | Agent-to-Agent communication | ✅ Verified | -| Web ↔ Node SDK | Client-to-Agent communication | ✅ Verified | -| React Native ↔ Node SDK | Client-to-Agent communication | ✅ Verified | - -_Note: Cross-SDK was tested using the `operations` describe above and is not covering all edge cases._ - -### Package Manager Test Results - -| Package Manager | Node 20 | Node 21 | Node 22 | Node 23 | -| --------------- | ------- | ------- | ------- | ------- | -| pnpm | ✅ | ✅ | ✅ | ✅ | -| npm | ✅ | ✅ | ✅ | ✅ | -| yarn@4.6.0 | ✅ | ✅ | ✅ | ✅ | -| yarn@1.22.19 | ✅ | ✅ | ✅ | ✅ | -| bun | ✅ | ✅ | ✅ | ✅ | - ## Tools & Utilities - **Repository:** [xmtp-qa-testing](https://github.com/xmtp/xmtp-qa-testing): This monorepo contains multiple tools for testing and monitoring - **Test bot:** Bot for testing with multiple agents - [see section](https://github.com/xmtp/xmtp-qa-testing/tree/main/bots/test/) - **Workflows:** See our CI/CD pipeline configuration - [see section](https://github.com/xmtp/xmtp-qa-testing/tree/main/.github/workflows) -- **Vitest:** We use Vitest for running tests with an interactive UI - [see section](https://xmtp-qa-testing.up.railway.app/__vitest__/#/) +- **Vitest:** We use Vitest for running tests with an interactive UI - [see section](https://xmtp-qa-testingus-east-production.up.railway.app/__vitest__/#/) - **Railway:** Visit our Railway project with all our services - [see section](https://railway.com/project/cc97c743-1be5-4ca3-a41d-0109e41ca1fd) - **Gm bot:** Bot for testing with older version of the protocol - [see section](https://github.com/xmtp/gm-bot) diff --git a/datadog/README.md b/datadog/README.md new file mode 100644 index 00000000..50e3af21 --- /dev/null +++ b/datadog/README.md @@ -0,0 +1,47 @@ +# XMTP Datadog integration + +This module provides tools for monitoring XMTP performance metrics via Datadog integration. + +## Overview + +The XMTP Datadog integration allows you to track crucial performance metrics, message delivery rates, and network statistics for your XMTP implementation. This data helps maintain optimal service quality and quickly identify potential issues. + +## Key components + +- `summary.ts` - Aggregates and formats metrics data +- `helper.ts` - Utility functions for Datadog integration +- `network.ts` - Network performance monitoring tools +- `thresholds.json` - Configuration for alert thresholds +- `dashboards/` - Datadog dashboard configuration files + +## Usage examples + +```typescript +// Initialize Datadog metrics +initDataDog(testName, envValue, geolocation, apiKey); + +// Send delivery rate metrics +sendDeliveryMetric(deliveryRate, testName, libxmtpVersion); + +// Send performance metrics +sendPerformanceMetric(durationMs, testName, libxmtpVersion); + +// Measure network performance +const networkStats = await getNetworkStats(); +``` + +## Common metrics + +- Message delivery rates +- End-to-end message latency +- Network performance statistics +- API response times +- Error rates and types + +## Dashboard integration + +The `dashboards/` directory contains configuration for visualizing your XMTP metrics in Datadog. These dashboards provide real-time monitoring of your application's performance. + +- [Performance Dashboard](./dashboards/performance.json) +- [Delivery Rate Dashboard](./dashboards/delivery-rate.json) +- [Network Performance Dashboard](./dashboards/network-performance.json) diff --git a/datadog/extract-2025-03-14T18_10_16.493Z-Core SDK operations performance.csv b/datadog/extract-2025-03-14T18_10_16.493Z-Core SDK operations performance.csv deleted file mode 100644 index 737a1f70..00000000 --- a/datadog/extract-2025-03-14T18_10_16.493Z-Core SDK operations performance.csv +++ /dev/null @@ -1,61 +0,0 @@ -"OPERATION","MEMBERS","AVG:XMTP.SDK.DURATION{METRIC_TYPE:OPERATION,ENV:PRODUCTION,GEO:US-EAST,TEST:TS_PERFORMANCE}" -creategroupbyidentifiers,400,7293.44 -creategroup,400,7172.52 -creategroupbyidentifiers,350,6502.96 -creategroup,350,6457.56 -creategroupbyidentifiers,300,5726.98 -creategroup,300,5657.68 -creategroupbyidentifiers,250,4526.802857142858 -creategroup,250,4310.269523809524 -creategroupbyidentifiers,200,3161.3219047619045 -creategroup,200,2967.5857142857144 -creategroupbyidentifiers,150,2203.7285714285713 -creategroup,150,1996.9409523809525 -creategroup,100,1273.5847619047618 -creategroupbyidentifiers,100,1200.4352380952382 -creategroup,50,1081.152380952381 -creategroupbyidentifiers,50,681.1723809523809 -removemembers,400,295.98 -removemembers,250,275.2142857142857 -removemembers,350,260.5 -createdm,,238.44285714285715 -updategroupname,400,234.38 -removemembers,300,230.22 -updategroupname,350,227.6 -updategroupname,300,214.68 -updategroupname,250,208.2952380952381 -removemembers,200,187.73142857142858 -receivegroupmessage,400,187.22 -receivegroupmessage,350,174.44 -removemembers,150,165.25333333333336 -receivegroupmessage,300,162.72 -updategroupname,200,157.90571428571428 -syncgroup,400,157.44 -receivegroupmessage,250,155.24571428571429 -syncgroup,250,151.98761904761903 -updategroupname,150,147.18857142857144 -removemembers,100,146.58285714285714 -syncgroup,350,146.38 -receivegroupmessage,200,145.02 -syncgroup,300,138.02 -updategroupname,100,135.80285714285714 -receivegroupmessage,150,133.98857142857142 -removemembers,50,131.87333333333333 -updategroupname,50,129.19714285714286 -receivegroupmessage,100,125.77333333333334 -syncgroup,200,119.56380952380952 -receivegroupmessage,50,118.44380952380952 -sendgm,,115.38857142857142 -sendgroupmessage,400,97.88 -sendgroupmessage,350,92.86 -sendgroupmessage,300,92.32 -syncgroup,150,89.00952380952381 -sendgroupmessage,200,88.7247619047619 -sendgroupmessage,250,87.93523809523809 -sendgroupmessage,150,84.53523809523809 -receivegm,,82.28 -sendgroupmessage,100,81.56761904761905 -sendgroupmessage,50,81.22095238095238 -syncgroup,100,78.81809523809524 -syncgroup,50,74.64190476190475 -inboxstate,,42.72380952380952 diff --git a/helpers/README.md b/helpers/README.md index c234a978..47c67d40 100644 --- a/helpers/README.md +++ b/helpers/README.md @@ -7,7 +7,6 @@ This directory contains utility modules that power the XMTP testing framework. T | Module | Purpose | | ------------------------------ | ------------------------------------------------- | | [client.ts](#clientts) | Creates signers and manages keys for test workers | -| [datadog.ts](#datadogts) | Sends performance metrics to Datadog | | [group.ts](#groupts) | Creates test groups with specified participants | | [logger.ts](#loggerts) | Logging utilities for test output | | [test.ts](#testts) | Test utilities for creating and managing tests | @@ -32,24 +31,6 @@ const dbPath = getDbPath(workerName, accountAddress, testName); const encryptionKey = generateEncryptionKeyHex(); ``` -### datadog.ts - -Sends performance metrics to Datadog for monitoring: - -```typescript -// Initialize Datadog metrics -initDataDog(testName, envValue, geolocation, apiKey); - -// Send delivery rate metrics -sendDeliveryMetric(deliveryRate, testName, libxmtpVersion); - -// Send performance metrics -sendPerformanceMetric(durationMs, testName, libxmtpVersion); - -// Measure network performance -const networkStats = await getNetworkStats(); -``` - ### group.ts Utilities for creating and managing test groups: diff --git a/helpers/client.ts b/helpers/client.ts index 2a7f7f52..95d1cee7 100644 --- a/helpers/client.ts +++ b/helpers/client.ts @@ -8,6 +8,7 @@ import { fromString, toString } from "uint8arrays"; import { createWalletClient, http, toBytes } from "viem"; import { privateKeyToAccount } from "viem/accounts"; import { sepolia } from "viem/chains"; +import { b } from "vitest/dist/chunks/suite.qtkXWc6R.js"; import { flushMetrics, initDataDog } from "./datadog"; import { createLogger, flushLogger, overrideConsole } from "./logger"; @@ -150,7 +151,7 @@ export function getAddressOfMember(members: GroupMember[], inboxId: string) { export async function closeEnv(testName: string, workers: WorkerManager) { flushLogger(testName); - await flushMetrics(); + await flushMetrics(testName); if (workers && typeof workers.getWorkers === "function") { for (const worker of workers.getWorkers()) { await worker.worker.terminate(); diff --git a/helpers/datadog.ts b/helpers/datadog.ts index 0a804b8a..1012f1e7 100644 --- a/helpers/datadog.ts +++ b/helpers/datadog.ts @@ -1,101 +1,180 @@ import { exec } from "child_process"; +import fs from "fs"; +import path from "path"; import { promisify } from "util"; import type { WorkerManager } from "@workers/manager"; import metrics from "datadog-metrics"; -// Global state variables -let isInitialized = false; -let currentGeo: string = ""; +// Types definitions +interface MemberThresholds { + creategroup: number; + creategroupbyidentifiers: number; + sendgroupmessage: number; + syncgroup: number; + updategroupname: number; + removemembers: number; + addmembers: number; + receivegroupmessage: number; + [key: string]: number; +} -// Refactored thresholds into a single configuration object -const THRESHOLDS = { +interface ThresholdsData { core: { - clientcreate: 3000, - createdm: 500, + [key: string]: number; + }; + network: { + [key: string]: number; + }; + memberBasedThresholds: { + [memberCount: string]: MemberThresholds; + }; + regionMultipliers: { + [region: string]: number; + }; + GEO_TO_COUNTRY_CODE: { + [region: string]: string; + }; + reliability: number; +} + +interface NetworkStats { + "DNS Lookup": number; + "TCP Connection": number; + "TLS Handshake": number; + Processing: number; + "Server Call": number; +} + +interface MetricData { + values: number[]; + threshold: number; + members?: string; +} + +interface ParsedTestName { + metricName: string; + metricDescription: string; + testNameExtracted: string; + operationType: string; + operationName: string; + members: string; +} + +type OperationType = "core" | "group" | "network"; +type MetricSubType = "stream" | "poll" | "recovery"; +type MetricType = "delivery" | "order"; + +// Constants +const THRESHOLDS: ThresholdsData = { + core: { + clientcreate: 350, + inboxstate: 350, + createdm: 350, sendgm: 200, receivegm: 200, - receivegroupmessage: 200, - updategroupname: 200, + creategroup: 350, + creategroupbyidentifiers: 350, syncgroup: 200, - addmembers: 500, - removemembers: 300, - inboxstate: 100, + updategroupname: 200, + removemembers: 250, + sendgroupmessage: 200, + receivegroupmessage: 200, }, network: { dns_lookup: 100, - tcp_connection: 200, - tls_handshake: 300, + tcp_connection: 150, + tls_handshake: 250, processing: 100, - server_call: 400, + server_call: 350, }, - group: { - createGroup: { - "50": 2000, - "100": 2000, - "150": 4000, - "200": 5000, - "250": 7000, - "300": 9000, - "350": 11000, - "400": 13000, + memberBasedThresholds: { + "50": { + creategroup: 350, + creategroupbyidentifiers: 350, + receivegroupmessage: 350, + sendgroupmessage: 200, + syncgroup: 200, + updategroupname: 200, + removemembers: 250, + addmembers: 200, }, - createGroupByIdentifiers: { - "50": 2300, - "100": 2500, - "150": 4500, - "200": 5500, - "250": 7500, - "300": 9500, - "350": 11500, - "400": 15000, + "100": { + creategroup: 400, + creategroupbyidentifiers: 400, + receivegroupmessage: 400, + sendgroupmessage: 200, + syncgroup: 200, + updategroupname: 200, + removemembers: 300, + addmembers: 200, }, - sendGroupMessage: { - "50": 100, - "100": 100, - "150": 100, - "200": 150, - "250": 200, - "300": 300, - "350": 350, - "400": 500, + "150": { + creategroup: 500, + creategroupbyidentifiers: 500, + receivegroupmessage: 500, + sendgroupmessage: 200, + syncgroup: 200, + updategroupname: 200, + removemembers: 300, + addmembers: 200, }, - syncGroup: { - "50": 100, - "100": 100, - "150": 100, - "200": 150, - "250": 200, - "300": 350, - "350": 350, - "400": 500, + "200": { + creategroup: 700, + creategroupbyidentifiers: 700, + receivegroupmessage: 700, + sendgroupmessage: 200, + syncgroup: 200, + updategroupname: 200, + removemembers: 300, + addmembers: 200, }, - updateGroupName: { - "50": 300, - "100": 300, - "150": 300, - "200": 300, - "250": 300, - "300": 300, - "350": 1500, - "400": 2000, + "250": { + creategroup: 900, + creategroupbyidentifiers: 900, + receivegroupmessage: 900, + sendgroupmessage: 200, + syncgroup: 200, + updategroupname: 200, + removemembers: 400, + addmembers: 200, }, - removeMembers: { - "50": 300, - "100": 300, - "150": 300, - "200": 300, - "250": 300, - "300": 300, - "350": 300, - "400": 300, + "300": { + creategroup: 1100, + creategroupbyidentifiers: 1100, + receivegroupmessage: 1100, + sendgroupmessage: 200, + syncgroup: 200, + updategroupname: 200, + removemembers: 400, + addmembers: 200, + }, + "350": { + creategroup: 1300, + creategroupbyidentifiers: 1300, + receivegroupmessage: 1300, + sendgroupmessage: 200, + syncgroup: 200, + updategroupname: 200, + removemembers: 400, + addmembers: 200, + }, + "400": { + creategroup: 1500, + creategroupbyidentifiers: 1500, + receivegroupmessage: 1500, + sendgroupmessage: 200, + syncgroup: 200, + updategroupname: 200, + removemembers: 500, + addmembers: 200, }, }, regionMultipliers: { - "us-east": 1.0, - "us-west": 1.0, - europe: 1.0, + "us-east": 1, + "us-west": 1, + europe: 1, asia: 1.5, - "south-america": 2.6, + "south-america": 3, }, GEO_TO_COUNTRY_CODE: { "us-east": "US", @@ -107,99 +186,169 @@ const THRESHOLDS = { reliability: 99.9, }; -function getCountryCodeFromGeo(geolocation: string): string { - return ( - THRESHOLDS.GEO_TO_COUNTRY_CODE[ - geolocation as keyof typeof THRESHOLDS.GEO_TO_COUNTRY_CODE - ] || "US" - ); +// Global state with proper initialization +const state = { + isInitialized: false, + currentGeo: "", + collectedMetrics: {} as Record, +}; + +// Helper functions +/** + * Find the appropriate member bucket for a given member count + */ +function findMemberBucket(members: number): string { + if (members <= 0) return "0"; + + const memberBuckets = Object.keys(THRESHOLDS.memberBasedThresholds) + .map(Number) + .sort((a, b) => a - b); + + // Find highest bucket <= member count + let applicableBucket = "0"; + for (const bucket of memberBuckets) { + if (members >= bucket) { + applicableBucket = bucket.toString(); + } else { + break; + } + } + + return applicableBucket; } -// Simplified threshold function +/** + * Get the appropriate threshold for an operation based on type, member count, and region + */ export function getThresholdForOperation( operation: string, - operationType: string = "core", - members: string = "", + operationType: OperationType, + members: number = 0, region: string = "us-east", ): number { + // Normalize inputs + const operationLower = operation.toLowerCase(); + const regionNormalized = region.toLowerCase().trim(); + const regionMultiplier = + THRESHOLDS.regionMultipliers[ + regionNormalized as keyof typeof THRESHOLDS.regionMultipliers + ] || 1.0; + + let baseThreshold = 0; + if (operationType === "network") { const networkThreshold = - THRESHOLDS.network[ - operation.toLowerCase() as keyof typeof THRESHOLDS.network + THRESHOLDS.network[operationLower as keyof typeof THRESHOLDS.network]; + baseThreshold = + typeof networkThreshold === "number" + ? networkThreshold + : THRESHOLDS.network.server_call; + } else if (operationType === "core") { + baseThreshold = + THRESHOLDS.core[operationLower as keyof typeof THRESHOLDS.core] || 0; + } else if (operationType === "group" && members > 0) { + const applicableBucket = findMemberBucket(members); + const memberThresholds = + THRESHOLDS.memberBasedThresholds[ + applicableBucket as keyof typeof THRESHOLDS.memberBasedThresholds ]; - return typeof networkThreshold === "number" ? networkThreshold : 200; + + if (memberThresholds && operationLower in memberThresholds) { + baseThreshold = + memberThresholds[operationLower as keyof typeof memberThresholds]; + console.log( + `Operation: ${operation}, members: ${members}, bucket: ${applicableBucket}, threshold: ${baseThreshold}`, + ); + } else { + baseThreshold = + THRESHOLDS.core[operationLower as keyof typeof THRESHOLDS.core] || 0; + console.log( + `Operation: ${operation}, members: ${members}, using core threshold: ${baseThreshold}`, + ); + } + } else { + baseThreshold = + THRESHOLDS.core[operationLower as keyof typeof THRESHOLDS.core] || 0; } - if (operationType === "group") { - const size = members || "50"; + return Math.round(baseThreshold * regionMultiplier); +} - // Get the operation-specific thresholds object - const groupThresholds = - THRESHOLDS.group[operation as keyof typeof THRESHOLDS.group]; +/** + * Calculate average of numeric values + */ +export function calculateAverage(values: number[]): number { + if (values.length === 0) return 0; + return values.reduce((sum, val) => sum + val, 0) / values.length; +} - // Safely check if this size exists, defaulting to 2000 if not - let baseThreshold = 2000; - if (groupThresholds) { - // Use type assertion to tell TypeScript this is a valid lookup - const sizeKey = size as unknown as keyof typeof groupThresholds; - if (sizeKey in groupThresholds) { - baseThreshold = groupThresholds[sizeKey]; - } +/** + * Group metrics by operation name and member count + */ +export function groupMetricsByOperation( + metrics: [string, MetricData][], +): Map< + string, + { operationName: string; members: string; operationData: MetricData } +> { + const groups = new Map(); + + for (const [operation, data] of metrics) { + // Use the operation parsing logic from parseTestName for consistency + const parts = operation.split(":"); + const operationPart = parts[0]; + const dashMatch = operationPart.match(/^([a-zA-Z]+)-(\d+)$/); + + const operationName = dashMatch ? dashMatch[1] : operationPart; + const memberCount = dashMatch ? dashMatch[2] : data.members || "-"; + const groupKey = `${operationName}-${memberCount}`; + + if (!groups.has(groupKey)) { + groups.set(groupKey, { + operationName, + members: memberCount, + operationData: null, + }); } - return Math.round( - baseThreshold * - (THRESHOLDS.regionMultipliers[ - region as keyof typeof THRESHOLDS.regionMultipliers - ] || 1.0), - ); + groups.get(groupKey).operationData = data; } - const coreOp = operation.toLowerCase(); - const baseThreshold = - coreOp in THRESHOLDS.core - ? THRESHOLDS.core[coreOp as keyof typeof THRESHOLDS.core] - : 300; - - return Math.round( - baseThreshold * - (THRESHOLDS.regionMultipliers[ - region as keyof typeof THRESHOLDS.regionMultipliers - ] || 1.0), - ); + return groups as Map< + string, + { operationName: string; members: string; operationData: MetricData } + >; } -export const sendPerformanceResult = ( - expect: any, - workers: WorkerManager, - start: number, -) => { - const testName = expect.getState().currentTestName; - if (testName) { - console.timeEnd(testName as string); - expect(workers.getWorkers()).toBeDefined(); - expect(workers.getWorkers().length).toBeGreaterThan(0); - void sendPerformanceMetric( - performance.now() - start, - testName as string, - workers.getVersion(), - ); - } -}; -// Add success status tag to duration metrics +/** + * Get country code from geolocation + */ +function getCountryCodeFromGeo(geolocation: string): string { + return ( + THRESHOLDS.GEO_TO_COUNTRY_CODE[ + geolocation as keyof typeof THRESHOLDS.GEO_TO_COUNTRY_CODE + ] || "US" + ); +} +// DataDog integration +/** + * Initialize DataDog metrics reporting + */ export function initDataDog( testName: string, envValue: string, geolocation: string, apiKey: string, ): boolean { - if (isInitialized) { + if (state.isInitialized) { return true; } + if (!testName.includes("ts_")) { return true; } + if (!apiKey) { console.warn("⚠️ DATADOG_API_KEY not found. Metrics will not be sent."); return false; @@ -207,7 +356,8 @@ export function initDataDog( try { const countryCode = getCountryCodeFromGeo(geolocation); - currentGeo = geolocation; + state.currentGeo = geolocation; + const initConfig = { apiKey: apiKey, defaultTags: [ @@ -217,8 +367,9 @@ export function initDataDog( `country_iso_code:${countryCode}`, ], }; + metrics.init(initConfig); - isInitialized = true; + state.isInitialized = true; return true; } catch (error) { console.error("❌ Failed to initialize DataDog metrics:", error); @@ -226,27 +377,49 @@ export function initDataDog( } } -// Combined metric sending function to reduce duplication +/** + * Send a metric to DataDog and collect for summary + */ export function sendMetric( metricName: string, metricValue: number, tags: Record, ): void { - if (!isInitialized) return; + if (!state.isInitialized) return; try { const fullMetricName = `xmtp.sdk.${metricName}`; const allTags = Object.entries({ ...tags }).map( ([key, value]) => `${key}:${String(value)}`, ); + // Add environment tag if not already present if (!allTags.some((tag) => tag.startsWith("env:"))) { allTags.push(`env:${process.env.XMTP_ENV as string}`); } - if (allTags.includes("success:false")) { - console.debug(fullMetricName, Math.round(metricValue), allTags); + // Add version tag if not already present + if (!allTags.some((tag) => tag.startsWith("xv:"))) { + allTags.push(`vm:${process.env.RAILWAY_SERVICE_ID || "unknown"}`); + } + + // Create a distinctive operation key that properly includes member count + // Format: operation_name-member_count (e.g., "createGroup-10") + const memberCount = tags.members || ""; + const operationKey = tags.operation + ? `${tags.operation}${memberCount ? `-${memberCount}` : ""}` + : metricName; + + if (!state.collectedMetrics[operationKey]) { + state.collectedMetrics[operationKey] = { + values: [], + threshold: tags.threshold || 0, + members: memberCount, + }; } + + state.collectedMetrics[operationKey].values.push(metricValue); + metrics.gauge(fullMetricName, Math.round(metricValue), allTags); } catch (error) { console.error( @@ -256,53 +429,120 @@ export function sendMetric( } } +/** + * Send test results metrics + */ export function sendTestResults(hasFailures: boolean, testName: string): void { - if (!isInitialized) { + if (!state.isInitialized) { console.warn("Datadog metrics not initialized"); return; } + try { const metricValue = hasFailures ? 0 : 1; - const metricName = `workflow`; - sendMetric(metricName, metricValue, { + sendMetric("workflow", metricValue, { workflow: testName, metric_type: "workflow", }); - console.log( - `The tests indicated that the test ${testName} was ${hasFailures}`, - ); } catch (error) { console.error("Error reporting to Datadog:", error); } } -// Simplified version of sendPerformanceMetric +// Performance tracking +/** + * Send performance metrics for tests + */ +export const sendPerformanceResult = ( + expect: any, + workers: WorkerManager, + start: number, +) => { + const testName = expect.getState().currentTestName; + if (testName) { + console.timeEnd(testName as string); + expect(workers.getWorkers()).toBeDefined(); + expect(workers.getWorkers().length).toBeGreaterThan(0); + void sendPerformanceMetric( + performance.now() - start, + testName as string, + workers.getVersion(), + false, + ); + } +}; + +/** + * Extract operation details from test name + */ +export function parseTestName(testName: string): ParsedTestName { + const metricNameParts = testName.split(":")[0]; + const metricName = metricNameParts.replaceAll(" > ", "."); + const metricDescription = testName.split(":")[1] || ""; + const operationParts = metricName.split("."); + const testNameExtracted = operationParts[0]; + + // Extract operation name and member count + let operationName = ""; + let members = ""; + + if (operationParts[1]) { + // Check formats: "operation-10" and "operation10" + const dashMatch = operationParts[1].match(/^([a-zA-Z]+)-(\d+)$/); + const noSeparatorMatch = operationParts[1].match(/^([a-zA-Z]+)(\d+)$/); + + if (dashMatch) { + operationName = dashMatch[1]; + members = dashMatch[2]; + } else if (noSeparatorMatch) { + operationName = noSeparatorMatch[1]; + members = noSeparatorMatch[2]; + } else { + operationName = operationParts[1]; + } + } + + const operationType = operationName.toLowerCase().includes("group") + ? "group" + : "core"; + + return { + metricName, + metricDescription, + testNameExtracted, + operationType, + operationName, + members, + }; +} + +/** + * Send detailed performance metrics + */ export async function sendPerformanceMetric( metricValue: number, testName: string, libxmtpVersion: string, skipNetworkStats: boolean = false, ): Promise { - if (!isInitialized) return; + if (!state.isInitialized) return; try { - const metricNameParts = testName.split(":")[0]; - const metricName = metricNameParts.replaceAll(" > ", "."); - const metricDescription = testName.split(":")[1] || ""; - const operationParts = metricName.split("."); - const testNameExtracted = operationParts[0]; - const operationName = operationParts[1]?.split("-")[0] || ""; - const members = operationParts[1]?.split("-")[1] || ""; + const { + metricDescription, + testNameExtracted, + operationType, + operationName, + members, + } = parseTestName(testName); - const operationType = operationName.toLowerCase().includes("group") - ? "group" - : "core"; const threshold = getThresholdForOperation( operationName, - operationType, - members, - currentGeo, + operationType as OperationType, + parseInt(members) || 0, + state.currentGeo, ); + const isSuccess = metricValue <= threshold; sendMetric("duration", metricValue, { @@ -315,13 +555,13 @@ export async function sendPerformanceMetric( members: members, success: isSuccess, threshold: threshold, - region: currentGeo, + region: state.currentGeo, }); // Network stats handling if (!skipNetworkStats) { const networkStats = await getNetworkStats(); - const countryCode = getCountryCodeFromGeo(currentGeo); + const countryCode = getCountryCodeFromGeo(state.currentGeo); for (const [statName, statValue] of Object.entries(networkStats)) { const networkMetricValue = Math.round(statValue * 1000); @@ -329,6 +569,8 @@ export async function sendPerformanceMetric( const networkThreshold = getThresholdForOperation( networkPhase, "network", + parseInt(members) || 0, + state.currentGeo, ); sendMetric("duration", networkMetricValue, { @@ -341,7 +583,7 @@ export async function sendPerformanceMetric( members: members, success: networkMetricValue <= networkThreshold, threshold: networkThreshold, - region: currentGeo, + region: state.currentGeo, }); } } @@ -354,42 +596,34 @@ export async function sendPerformanceMetric( } /** - * Explicitly flush all buffered metrics to DataDog - * Call this at the end of your test suite + * Send delivery reliability metrics */ -export function flushMetrics(): Promise { - return new Promise((resolve) => { - if (!isInitialized) { - resolve(); - return; - } - - //console.log("🔄 Flushing DataDog metrics..."); +export function sendDeliveryMetric( + metricValue: number, + version: string, + testName: string, + metricSubType: MetricSubType, + metricType: MetricType, +): void { + const threshold = THRESHOLDS.reliability; + const isSuccess = metricValue >= threshold; - void metrics.flush().then(() => { - //console.log("✅ DataDog metrics flushed successfully"); - resolve(); - }); + sendMetric(metricType, Math.round(metricValue), { + libxmtp: version, + test: testName, + metric_type: metricType, + metric_subtype: metricSubType, + success: isSuccess, + threshold: threshold, }); } +// Network performance const execAsync = promisify(exec); /** - * Get network performance statistics for a specific endpoint - * @param endpoint The endpoint to monitor (defaults to XMTP gRPC endpoint) - * @returns Object containing timing information in seconds + * Measure network performance to an endpoint */ -let firstLogShared = false; - -interface NetworkStats { - "DNS Lookup": number; - "TCP Connection": number; - "TLS Handshake": number; - Processing: number; - "Server Call": number; -} - export async function getNetworkStats( endpoint = "https://grpc.dev.xmtp.network:443", ): Promise { @@ -434,42 +668,133 @@ export async function getNetworkStats( stats["Processing"] = 0; } - if ( - stats["Processing"] * 1000 > 300 || - stats["TLS Handshake"] * 1000 > 300 || - stats["Server Call"] * 1000 > 300 - ) { - if (!firstLogShared) { - firstLogShared = true; - console.warn( - `Slow connection detected - total: ${stats["Server Call"] * 1000}ms, TLS: ${stats["TLS Handshake"] * 1000}ms, processing: ${stats["Processing"] * 1000}ms`, - ); + return stats; +} + +// Reporting and summary +/** + * Flush all metrics and generate summary report + */ +export function flushMetrics(testName: string): Promise { + return new Promise((resolve) => { + if (!state.isInitialized) { + resolve(); + return; } - } - return stats; + logMetricsSummary(testName); + + void metrics.flush().then(() => { + resolve(); + }); + }); } -// Unified delivery metrics function -export function sendDeliveryMetric( - metricValue: number, - version: string, +/** + * Creates and saves the metrics report + */ +function saveMetricsReport( testName: string, - metricSubType: "stream" | "poll" | "recovery", - metricType: "delivery" | "order", + validMetrics: [string, MetricData][], ): void { - // Determine success based on the metric subtype - const threshold = THRESHOLDS.reliability; + // Create directory for reports + const reportsDir = path.join(process.cwd(), "logs"); + if (!fs.existsSync(reportsDir)) { + fs.mkdirSync(reportsDir, { recursive: true }); + } - const isSuccess = metricValue >= threshold; + // Create filename with environment info + const filename = path.join( + reportsDir, + `${testName}-${state.currentGeo}-${process.env.XMTP_ENV}.md`, + ); - // Send primary metric - sendMetric(metricType, Math.round(metricValue), { - libxmtp: version, - test: testName, - metric_type: metricType, - metric_subtype: metricSubType, - success: isSuccess, - threshold: threshold, - }); + try { + fs.writeFileSync(filename, generateReportContent(validMetrics)); + console.log(`✅ Report saved to ${filename}`); + } catch (error) { + console.error(`❌ Error writing metrics summary to file:`, error); + } +} + +/** + * Generate and log a metrics summary report + */ +export function logMetricsSummary(testName: string): void { + if ( + !state.isInitialized || + Object.keys(state.collectedMetrics).length === 0 + ) { + console.log("No metrics collected to summarize"); + return; + } + + console.log("\n📊 Creating metrics summary report"); + + // Filter out workflow metrics and empty value arrays + const validMetrics = Object.entries(state.collectedMetrics).filter( + ([operation, data]) => operation !== "workflow" && data.values.length > 0, + ); + + // Count passed metrics + const passedMetrics = validMetrics.filter( + ([_, data]) => calculateAverage(data.values) <= data.threshold, + ).length; + + const totalMetrics = validMetrics.length; + const passRate = + totalMetrics > 0 ? Math.round((passedMetrics / totalMetrics) * 100) : 0; + + console.log( + `✅ Passed: ${passedMetrics}/${totalMetrics} metrics (${passRate}%)`, + ); + + saveMetricsReport(testName, validMetrics); + + // Reset metrics collection for next test run + state.collectedMetrics = {}; +} + +/** + * Generate the markdown report content + */ +function generateReportContent(validMetrics: [string, MetricData][]): string { + let content = "# METRICS SUMMARY\n\n"; + content += + "| Operation | Members | Avg (ms) | Min/Max (ms) | Threshold (ms) | Variance (ms) | Status |\n"; + content += + "|-----------|---------|----------|--------------|----------------|---------------|---------|\n"; + + // Group metrics by operation name and member count + const operationGroups = groupMetricsByOperation(validMetrics); + + // Generate table rows + for (const group of operationGroups.values()) { + if (!group.operationData) continue; + + const { operationName, members, operationData: data } = group; + const memberCount = members !== "-" ? parseInt(members) : 0; + const operationType = operationName.toLowerCase().includes("group") + ? "group" + : "core"; + + // Calculate metrics data + const threshold = getThresholdForOperation( + operationName, + operationType as OperationType, + memberCount, + state.currentGeo, + ); + data.threshold = threshold; + + const average = calculateAverage(data.values); + const variance = Math.round(average - threshold); + const varianceFormatted = + variance <= 0 ? variance.toString() : `+${variance}`; + + // Format table row + content += `| ${operationName} | ${members} | ${Math.round(average)} | ${Math.round(Math.min(...data.values))}/${Math.round(Math.max(...data.values))} | ${threshold} | ${varianceFormatted} | ${average <= threshold ? "PASS ✅" : "FAIL ❌"} |\n`; + } + + return content; } diff --git a/helpers/logger.ts b/helpers/logger.ts index 351097b0..58d492da 100644 --- a/helpers/logger.ts +++ b/helpers/logger.ts @@ -102,22 +102,29 @@ function filterLog(args: any[]): string { } // Check for the console.time/timeEnd pattern: where args[0] is "%s: %s" - if (args.length >= 2 && args[0] === "%s: %s") { - // Join the remaining parts into one message - const message = args.slice(1).join(" "); - const timePattern = /(\d+(\.\d+)?)(ms|s)\b/; - const match = message.match(timePattern); - if (match) { - const timeValue = parseFloat(match[1]); - const unit = match[3]; - const timeInMs = unit === "ms" ? timeValue : timeValue * 1000; - // Skip logs for durations less than or equal to 300ms - if (timeInMs <= 300) { - return ""; + + if (!process.env.CI) { + if (args.length >= 2 && args[0] === "%s: %s") { + // Join the remaining parts into one message + const message = args.slice(1).join(" "); + const timePattern = /(\d+(\.\d+)?)(ms|s)\b/; + const match = message.match(timePattern); + if (match) { + const timeValue = parseFloat(match[1]); + const unit = match[3]; + const timeInMs = unit === "ms" ? timeValue : timeValue * 1000; + // Skip logs for durations less than or equal to 300ms + if (timeInMs <= 300) { + return ""; + } } + // Remove any "%s" placeholders from the message. + return message.replace(/%s/g, "").trim(); + } + } else { + if (args.length >= 2 && args[0] === "%s: %s") { + return ""; } - // Remove any "%s" placeholders from the message. - return message.replace(/%s/g, "").trim(); } return ( @@ -145,7 +152,6 @@ function filterLog(args: any[]): string { const getLogFilePath = (testName: string): string => { const env = process.env.XMTP_ENV as string; const logName = testName + "_" + env; - console.log("logName", logName); const sanitizedName = logName.replace(/[^a-zA-Z0-9-_]/g, "_"); const fileName = `${sanitizedName}.log`; @@ -190,16 +196,16 @@ export const overrideConsole = (logger: winston.Logger) => { logger.log("error", message); } }; + console.debug = (...args: any[]) => { - // Only show debug logs when not in CI - //if (!process.env.CI) { - // Use the original console.debug function - const originalConsoleDebug = Function.prototype.bind.call( - console.constructor.prototype.debug, - console, - ); - originalConsoleDebug(...args); - //} + if (!process.env.CI) { + // Using a specific function type for console.debug + const originalConsoleDebug = Function.prototype.bind.call( + console.constructor.prototype.debug as (...args: any[]) => void, + console, + ); + originalConsoleDebug(...args); + } }; } catch (error) { console.error("Error overriding console", error); diff --git a/helpers/playwright.ts b/helpers/playwright.ts index 555272c2..4acd85fe 100644 --- a/helpers/playwright.ts +++ b/helpers/playwright.ts @@ -8,7 +8,7 @@ import { type Page, } from "playwright-chromium"; -const snapshotDir = path.join(process.cwd(), "./logs/snapshots"); +const snapshotDir = path.join(process.cwd(), "./logs"); let browser: Browser | null = null; if (!fs.existsSync(snapshotDir)) { fs.mkdirSync(snapshotDir, { recursive: true }); diff --git a/package.json b/package.json index 42d830d2..0f874aa4 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "bot:simple": "tsx bots/simple/index.ts", "bot:test": "tsx bots/test/index.ts", "build": "tsc", - "clean": "rimraf .data/ ||:", + "clean": "rimraf .data/ ||: && rimraf logs/ ||:", "datadog": "tsx scripts/send-to-datadog.ts", "format": "prettier -w .", "format:check": "prettier -c .", diff --git a/scripts/railway.sh b/scripts/railway.sh index 1b607ccb..0d3d4374 100755 --- a/scripts/railway.sh +++ b/scripts/railway.sh @@ -1,7 +1,23 @@ #!/bin/bash -# Set your Railway token -export RAILWAY_TOKEN="73286d48-1ca0-469e-b9b1-fca1f5011ae8" +# Load environment variables from .env file +if [ -f .env ]; then + echo "Loading environment variables from .env file" + export $(grep -v '^#' .env | xargs) +else + echo "Error: .env file not found" + exit 1 +fi + +# Check if RAILWAY_PROJECT_TOKEN is set +if [ -z "$RAILWAY_PROJECT_TOKEN" ]; then + echo "Error: RAILWAY_PROJECT_TOKEN is not set in .env file" + exit 1 +fi + +# Set Railway token directly +export RAILWAY_TOKEN="$RAILWAY_PROJECT_TOKEN" +echo "Using Railway project token: ${RAILWAY_PROJECT_TOKEN:0:8}..." # Check CLI version echo "Railway CLI version:" @@ -11,7 +27,27 @@ railway --version echo "Checking Railway status:" railway status +# Important: The key issue is understanding what 'railway run' does +# It executes a command locally but with Railway environment variables +# To truly run on Railway infrastructure, we need to use railway exec -# Run your performance test -echo "Running performance test on service:" -railway run -s xmtp-qa-testing:us-east yarn test ts_delivery \ No newline at end of file +echo "Running test on Railway remote infrastructure..." +# Use railway exec to run a command on the remote service +railway run --service "xmtp-qa-testing:us-east" -- bash -c ' + echo "Running on Railway remote infrastructure" + echo "Service details:" + echo "- Project name: $RAILWAY_PROJECT_NAME" + echo "- Environment: $RAILWAY_ENVIRONMENT_NAME" + echo "- Service name: $RAILWAY_SERVICE_NAME" + echo "- Service ID: $RAILWAY_SERVICE_ID" + + # Set XMTP_ENV to production manually + export XMTP_ENV=production + export GEOLOCATION=us-east + + echo "- XMTP environment: $XMTP_ENV" + echo "- Geolocation: $GEOLOCATION" + + # Run the test + yarn test ts_performance +' \ No newline at end of file diff --git a/tests/TS_Performance.test.ts b/tests/TS_Performance.test.ts index 1f8f2b06..e2fd6e77 100644 --- a/tests/TS_Performance.test.ts +++ b/tests/TS_Performance.test.ts @@ -147,6 +147,97 @@ describe(testName, () => { throw e; } }); + let i = 4; + let newGroup: Conversation; + it(`createGroup: should create a large group of ${i} participants ${i}`, async () => { + try { + const sliced = generatedInboxes.slice(0, i); + newGroup = await workers + .get("henry")! + .client.conversations.newGroup(sliced.map((inbox) => inbox.inboxId)); + console.log("New group created", newGroup.id); + expect(newGroup.id).toBeDefined(); + } catch (e) { + hasFailures = logError(e, expect); + throw e; + } + }); + it(`createGroupByIdentifiers: should create a large group of ${i} participants ${i}`, async () => { + try { + const sliced = generatedInboxes.slice(0, i); + const newGroupByIdentifier = await workers + .get("henry")! + .client.conversations.newGroupWithIdentifiers( + sliced.map((inbox) => ({ + identifier: inbox.accountAddress, + identifierKind: IdentifierKind.Ethereum, + })), + ); + expect(newGroupByIdentifier.id).toBeDefined(); + } catch (e) { + hasFailures = logError(e, expect); + throw e; + } + }); + it(`syncGroup: should sync a large group of ${i} participants ${i}`, async () => { + try { + await newGroup.sync(); + const members = await newGroup.members(); + expect(members.length).toBe(i + 1); + } catch (e) { + hasFailures = logError(e, expect); + throw e; + } + }); + it(`updateGroupName: should update the group name`, async () => { + try { + const newName = "Large Group"; + await (newGroup as Group).updateName(newName); + await newGroup.sync(); + const name = (newGroup as Group).name; + expect(name).toBe(newName); + } catch (e) { + hasFailures = logError(e, expect); + throw e; + } + }); + it(`removeMembers: should remove a participant from a group`, async () => { + try { + const previousMembers = await newGroup.members(); + await (newGroup as Group).removeMembers([ + previousMembers.filter( + (member) => member.inboxId !== (newGroup as Group).addedByInboxId, + )[0].inboxId, + ]); + + const members = await newGroup.members(); + expect(members.length).toBe(previousMembers.length - 1); + } catch (e) { + hasFailures = logError(e, expect); + throw e; + } + }); + it(`sendGroupMessage: should measure sending a gm in a group of ${i} participants`, async () => { + try { + const groupMessage = "gm-" + Math.random().toString(36).substring(2, 15); + + await newGroup.send(groupMessage); + console.log("GM Message sent in group", groupMessage); + expect(groupMessage).toBeDefined(); + } catch (e) { + hasFailures = logError(e, expect); + throw e; + } + }); + it(`receiveGroupMessage: should create a group and measure all streams`, async () => { + try { + const verifyResult = await verifyStreamAll(newGroup, workers); + expect(verifyResult.allReceived).toBe(true); + } catch (e) { + hasFailures = logError(e, expect); + throw e; + } + }); for (let i = batchSize; i <= total; i += batchSize) { let newGroup: Conversation; diff --git a/tests/groups.test.ts b/tests/groups.test.ts index 6bf60b65..93095abf 100644 --- a/tests/groups.test.ts +++ b/tests/groups.test.ts @@ -83,9 +83,11 @@ describe(testName, () => { it(`createGroup-${i}: should create a large group of ${i} participants ${i}`, async () => { try { const sliced = generatedInboxes.slice(0, i); + console.log("Creating group with", sliced.length, "participants"); newGroup = await workers .get("henry")! .client.conversations.newGroup(sliced.map((inbox) => inbox.inboxId)); + console.log("Group created", newGroup.id); expect(newGroup.id).toBeDefined(); } catch (e) { hasFailures = logError(e, expect); diff --git a/workers/main.ts b/workers/main.ts index 8cdd4e69..9cbab733 100644 --- a/workers/main.ts +++ b/workers/main.ts @@ -210,13 +210,13 @@ export class WorkerClient extends Worker { const installationId = this.client.installationId; - console.debug({ - inboxId: this.client.inboxId, - dbPath, - version, - address: this.address, - installationId, - }); + // console.debug({ + // inboxId: this.client.inboxId, + // dbPath, + // version, + // address: this.address, + // installationId, + // }); return { client: this.client, dbPath, diff --git a/yarn.lock b/yarn.lock index 841fbafd..3b74d7c2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1357,9 +1357,9 @@ __metadata: linkType: hard "@xmtp/node-bindings@npm:^1.0.0-rc2": - version: 1.0.0-rc3 - resolution: "@xmtp/node-bindings@npm:1.0.0-rc3" - checksum: 10/b9afdfd4ca409778c6845acf342ba53220829be2e1b1c818cdc57fd6611c39930a07038d78ff31f8e06c6756bcd3996dfae04b695064639541295d112b977ae8 + version: 1.0.0 + resolution: "@xmtp/node-bindings@npm:1.0.0" + checksum: 10/8bf91003ea3859f1c535f871013d13eb1096728f51e3c81471eb5ad9a27de65a19a9817eb8efbd56e257c07076af94771845643211764ff028103e202779f6ce languageName: node linkType: hard