Skip to content

Commit f99aff4

Browse files
humanagenthumanagent
authored
railway script update (#83)
* railway script update * push summary * fix lint * updload summary * us-east tressholds * improved artifacts * fix log * dont log time * dont log time * dont log time * new tessolds * artifcacts * fix lint * fix lint * variance * artifact locations * better tressholds * better tressholds * group miltipliers * group miltipliers * group miltipliers * group miltipliers * group miltipliers * mutlipliers * better artifacts * better artifacts * better artifacts * better artifacts * fix * fi bug * fi bug * fix tresshold * test * adjusted batch sixe * adjust * adjust * adjust * adjust * adjust * adjust * adjust * adjust * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix
1 parent c33fde9 commit f99aff4

18 files changed

+819
-423
lines changed

‎.github/workflows/TS_Delivery.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,10 @@ jobs:
3434
run: chmod +x ./scripts/run-test.sh
3535
- name: Run tests with retry
3636
run: ./scripts/run-test.sh TS_Delivery
37-
- name: Upload log files
37+
- name: Upload reports artifacts
3838
uses: actions/upload-artifact@v4
3939
with:
40-
name: delivery_logs_dev
40+
name: ts_delivery_${{ env.XMTP_ENV }}
4141
path: logs/
4242
production:
4343
runs-on: ubuntu-latest
@@ -66,8 +66,8 @@ jobs:
6666
run: chmod +x ./scripts/run-test.sh
6767
- name: Run tests with retry
6868
run: ./scripts/run-test.sh TS_Delivery
69-
- name: Upload log files
69+
- name: Upload reports artifacts
7070
uses: actions/upload-artifact@v4
7171
with:
72-
name: delivery_logs_production
72+
name: ts_delivery_${{ env.XMTP_ENV }}
7373
path: logs/

‎.github/workflows/TS_Geolocation.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ jobs:
7171
- name: Upload log files
7272
uses: actions/upload-artifact@v4
7373
with:
74-
name: geolocation_logs_dev
74+
name: ts_geolocation_${{ env.XMTP_ENV }}
7575
path: logs/
7676

7777
- name: Check for failures
@@ -154,7 +154,7 @@ jobs:
154154
- name: Upload log files
155155
uses: actions/upload-artifact@v4
156156
with:
157-
name: geolocation_logs_production
157+
name: ts_geolocation_${{ env.XMTP_ENV }}
158158
path: logs/
159159

160160
- name: Check for failures

‎.github/workflows/TS_Gm.yml

+3-17
Original file line numberDiff line numberDiff line change
@@ -38,18 +38,11 @@ jobs:
3838
run: chmod +x ./scripts/run-test.sh
3939
- name: Run tests with retry
4040
run: ./scripts/run-test.sh TS_Gm
41-
- name: Upload log files
41+
- name: Upload reports artifacts
4242
uses: actions/upload-artifact@v4
43-
if: always()
4443
with:
45-
name: gm_logs_dev
44+
name: ts_gm_${{ env.XMTP_ENV }}
4645
path: logs/
47-
- name: Upload snapshots
48-
uses: actions/upload-artifact@v4
49-
if: always()
50-
with:
51-
name: playwright_snapshots
52-
path: logs/snapshots/*.png
5346
production:
5447
runs-on: ubuntu-latest
5548
env:
@@ -83,13 +76,6 @@ jobs:
8376
run: ./scripts/run-test.sh TS_Gm
8477
- name: Upload log files
8578
uses: actions/upload-artifact@v4
86-
if: always()
8779
with:
88-
name: gm_logs_production
80+
name: ts_gm_${{ env.XMTP_ENV }}
8981
path: logs/
90-
- name: Upload snapshots
91-
uses: actions/upload-artifact@v4
92-
if: always()
93-
with:
94-
name: playwright_snapshots
95-
path: logs/snapshots/*.png

‎.github/workflows/TS_Performance.yml

+5-4
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,12 @@ jobs:
3232
run: chmod +x ./scripts/run-test.sh
3333
- name: Run tests with retry
3434
run: ./scripts/run-test.sh TS_Performance
35-
- name: Upload log files
35+
- name: Upload reports artifacts
3636
uses: actions/upload-artifact@v4
3737
with:
38-
name: performance_logs_dev
38+
name: ts_performance_${{ env.XMTP_ENV }}
3939
path: logs/
40+
4041
production:
4142
runs-on: ubuntu-latest
4243
env:
@@ -60,8 +61,8 @@ jobs:
6061
run: chmod +x ./scripts/run-test.sh
6162
- name: Run tests with retry
6263
run: ./scripts/run-test.sh TS_Performance
63-
- name: Upload log files
64+
- name: Upload reports artifacts
6465
uses: actions/upload-artifact@v4
6566
with:
66-
name: performance_logs_production
67+
name: ts_performance_${{ env.XMTP_ENV }}
6768
path: logs/

‎README.md

+32-51
Original file line numberDiff line numberDiff line change
@@ -105,32 +105,35 @@ We can test all XMTP bindings using three main applications. We use [xmtp.chat](
105105

106106
### Core SDK Operations Performance
107107

108-
| Operation | Description | Avg (ms) | Target | Status |
109-
| ------------------- | -------------------------------------- | -------- | ------ | ------------ |
110-
| createDM | Creating a direct message conversation | 254-306 | <500ms | ✅ On Target |
111-
| sendGM | Sending a group message | 123-132 | <200ms | ✅ On Target |
112-
| receiveGM | Receiving a group message | 90-94 | <200ms | ✅ On Target |
113-
| receiveGroupMessage | Processing group message streams | 119-127 | <200ms | ✅ On Target |
114-
| updateGroupName | Updating group metadata | 105-108 | <200ms | ✅ On Target |
115-
| syncGroup | Syncing group state | 78-89 | <200ms | ✅ On Target |
116-
| addMembers | Adding participants to a group | 238-280 | <500ms | ✅ On Target |
117-
| removeMembers | Removing participants from a group | 147-168 | <300ms | ✅ On Target |
118-
| inboxState | Checking inbox state | 36 | <100ms | ✅ On Target |
108+
| Operation | Description | Avg (ms) | Target | Status |
109+
| ------------------------ | -------------------------------------- | -------- | ------ | ------------ |
110+
| clientCreate | Creating a client | 254-306 | <350ms | ✅ On Target |
111+
| inboxState | Checking inbox state | 300 | <350ms | ✅ On Target |
112+
| createDM | Creating a direct message conversation | 200-250 | <350ms | ✅ On Target |
113+
| sendGM | Sending a group message | 123-160 | <200ms | ✅ On Target |
114+
| receiveGM | Receiving a group message | 90-140 | <200ms | ✅ On Target |
115+
| createGroup | Creating a group | 254-306 | <350ms | ✅ On Target |
116+
| createGroupByIdentifiers | Creating a group by address | 254-306 | <350ms | ✅ On Target |
117+
| syncGroup | Syncing group state | 78-89 | <200ms | ✅ On Target |
118+
| updateGroupName | Updating group metadata | 105-160 | <200ms | ✅ On Target |
119+
| removeMembers | Removing participants from a group | 110-168 | <250ms | ✅ On Target |
120+
| sendGroupMessage | Sending a group message | 100-127 | <200ms | ✅ On Target |
121+
| receiveGroupMessage | Processing group message streams | 119-127 | <200ms | ✅ On Target |
119122

120123
_Note: Based on data from 79 measured operations in the `us-east` region and `production` network._
121124

122125
### Group Operations Performance by Size
123126

124127
| Size | Create(ms) | Send(ms) | Sync(ms) | Update(ms) | Remove(ms) | Target(Create) | Status |
125128
| ---- | ---------- | -------- | -------- | ---------- | ---------- | -------------- | ---------------------- |
126-
| 50 | 990 | 71 | 61 | 81 | 140 | <2,000ms | ✅ On Target |
127-
| 100 | 1,599 | 67 | 66 | 91 | 182 | <2,000ms | ✅ On Target |
128-
| 150 | 2,956 | 72 | 85 | 104 | 183 | <4,000ms | ✅ On Target |
129-
| 200 | 4,598 | 73 | 103 | 139 | 211 | <5,000ms | ✅ On Target |
130-
| 250 | 5,983 | 76 | 120 | 164 | 234 | <7,000ms | ✅ On Target |
131-
| 300 | 8,707 | 81 | 321 | 255 | 309 | <9,000ms | ✅ On Target |
132-
| 350 | 9,826 | 79 | 132 | 228 | 368 | <11,000ms | ⚠️ Performance Concern |
133-
| 400 | 11,451 | 84 | 170 | 427 | 501 | <15,000ms | ⚠️ Performance Concern |
129+
| 50 | 323 | 71 | 61 | 81 | 140 | <350ms | ✅ On Target |
130+
| 100 | 289 | 67 | 66 | 91 | 182 | <400ms | ✅ On Target |
131+
| 150 | 330 | 72 | 85 | 104 | 183 | <500ms | ✅ On Target |
132+
| 200 | 474 | 73 | 103 | 139 | 211 | <700ms | ✅ On Target |
133+
| 250 | 654 | 76 | 120 | 164 | 234 | <900ms | ✅ On Target |
134+
| 300 | 758 | 81 | 321 | 255 | 309 | <1100ms | ✅ On Target |
135+
| 350 | 1064 | 79 | 132 | 228 | 368 | <1300ms | ⚠️ Performance Concern |
136+
| 400 | 1305 | 84 | 170 | 427 | 501 | <1400ms | ⚠️ Performance Concern |
134137
| 450 | - | - | - | - | - | - | ❌ Severe impact |
135138

136139
_Note: Performance increases significantly beyond `350` members, which represents a hard limit on the protocol._
@@ -145,7 +148,7 @@ _Note: Performance increases significantly beyond `350` members, which represent
145148
| TCP Connection | 105.6ms avg | <200ms | ✅ On Target |
146149
| TLS Handshake | 238.9ms avg | <300ms | ✅ On Target |
147150
| Processing | 30ms avg | <100ms | ✅ On Target |
148-
| Server Call | 238.9ms avg | <400ms | ✅ On Target |
151+
| Server Call | 238.9ms avg | <300ms | ✅ On Target |
149152

150153
_Note: Performance metrics based on `us-east` testing on `production` network._
151154

@@ -177,14 +180,14 @@ _Note: `Production` network consistently shows better network performance across
177180

178181
### Message delivery testing
179182

180-
| Test Area | Current Performance | Target | Status |
181-
| ---------------------- | ------------------- | --------------- | ------------ |
182-
| Stream Delivery Rate | 100% successful | 99.9% minimum | ✅ On Target |
183-
| Poll Delivery Rate | 100% successful | 99.9% minimum | ✅ On Target |
184-
| Stream Order | 100% in order | 100% in order | ✅ On Target |
185-
| Poll Order | 100% in order | 100% in order | ✅ On Target |
186-
| Offline Recovery Rate | 100% successful | 100% successful | ✅ On Target |
187-
| Offline Recovery Order | 100% in order | 100% in order | ✅ On Target |
183+
| Test Area | Current Performance | Target | Status |
184+
| -------------------- | ------------------- | -------------- | ------------ |
185+
| Stream Delivery Rate | 100% successful | 99.9% minimum | ✅ On Target |
186+
| Poll Delivery Rate | 100% successful | 99.9% minimum | ✅ On Target |
187+
| Recovery Rate | 100% successful | 99.9% minimum | ✅ On Target |
188+
| Stream Order | 100% in order | 99.9% in order | ✅ On Target |
189+
| Poll Order | 100% in order | 99.9% in order | ✅ On Target |
190+
| Recovery Order | 100% in order | 99.9% in order | ✅ On Target |
188191

189192
_Note: Testing regularly in groups of `40` active members listening to one user sending 100 messages_
190193

@@ -219,34 +222,12 @@ _Note: A hybrid approach using `stream` and `poll`-based verification provides t
219222
- **Node-sdk only**: Metrics are based on node-sdk only operations and are not covering performance across all SDKs.
220223
- **Pre-Release Status**: This assessment reflects the current development version targeting the `4.0.0` stable release. Optimizations and improvements are ongoing.
221224

222-
## Other
223-
224-
### Cross-SDK Testing
225-
226-
| SDK Combination | Test Focus | Status |
227-
| ------------------------ | ----------------------------- | ----------- |
228-
| Node SDK ↔ Node SDK | Agent-to-Agent communication | ✅ Verified |
229-
| Web ↔ Node SDK | Client-to-Agent communication | ✅ Verified |
230-
| React Native ↔ Node SDK | Client-to-Agent communication | ✅ Verified |
231-
232-
_Note: Cross-SDK was tested using the `operations` describe above and is not covering all edge cases._
233-
234-
### Package Manager Test Results
235-
236-
| Package Manager | Node 20 | Node 21 | Node 22 | Node 23 |
237-
| --------------- | ------- | ------- | ------- | ------- |
238-
| pnpm | ✅ | ✅ | ✅ | ✅ |
239-
| npm | ✅ | ✅ | ✅ | ✅ |
240-
| yarn@4.6.0 | ✅ | ✅ | ✅ | ✅ |
241-
| yarn@1.22.19 | ✅ | ✅ | ✅ | ✅ |
242-
| bun | ✅ | ✅ | ✅ | ✅ |
243-
244225
## Tools & Utilities
245226

246227
- **Repository:** [xmtp-qa-testing](https://github.com/xmtp/xmtp-qa-testing): This monorepo contains multiple tools for testing and monitoring
247228
- **Test bot:** Bot for testing with multiple agents - [see section](https://github.com/xmtp/xmtp-qa-testing/tree/main/bots/test/)
248229
- **Workflows:** See our CI/CD pipeline configuration - [see section](https://github.com/xmtp/xmtp-qa-testing/tree/main/.github/workflows)
249-
- **Vitest:** We use Vitest for running tests with an interactive UI - [see section](https://xmtp-qa-testing.up.railway.app/__vitest__/#/)
230+
- **Vitest:** We use Vitest for running tests with an interactive UI - [see section](https://xmtp-qa-testingus-east-production.up.railway.app/__vitest__/#/)
250231
- **Railway:** Visit our Railway project with all our services - [see section](https://railway.com/project/cc97c743-1be5-4ca3-a41d-0109e41ca1fd)
251232
- **Gm bot:** Bot for testing with older version of the protocol - [see section](https://github.com/xmtp/gm-bot)
252233

‎datadog/README.md

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# XMTP Datadog integration
2+
3+
This module provides tools for monitoring XMTP performance metrics via Datadog integration.
4+
5+
## Overview
6+
7+
The XMTP Datadog integration allows you to track crucial performance metrics, message delivery rates, and network statistics for your XMTP implementation. This data helps maintain optimal service quality and quickly identify potential issues.
8+
9+
## Key components
10+
11+
- `summary.ts` - Aggregates and formats metrics data
12+
- `helper.ts` - Utility functions for Datadog integration
13+
- `network.ts` - Network performance monitoring tools
14+
- `thresholds.json` - Configuration for alert thresholds
15+
- `dashboards/` - Datadog dashboard configuration files
16+
17+
## Usage examples
18+
19+
```typescript
20+
// Initialize Datadog metrics
21+
initDataDog(testName, envValue, geolocation, apiKey);
22+
23+
// Send delivery rate metrics
24+
sendDeliveryMetric(deliveryRate, testName, libxmtpVersion);
25+
26+
// Send performance metrics
27+
sendPerformanceMetric(durationMs, testName, libxmtpVersion);
28+
29+
// Measure network performance
30+
const networkStats = await getNetworkStats();
31+
```
32+
33+
## Common metrics
34+
35+
- Message delivery rates
36+
- End-to-end message latency
37+
- Network performance statistics
38+
- API response times
39+
- Error rates and types
40+
41+
## Dashboard integration
42+
43+
The `dashboards/` directory contains configuration for visualizing your XMTP metrics in Datadog. These dashboards provide real-time monitoring of your application's performance.
44+
45+
- [Performance Dashboard](./dashboards/performance.json)
46+
- [Delivery Rate Dashboard](./dashboards/delivery-rate.json)
47+
- [Network Performance Dashboard](./dashboards/network-performance.json)

‎datadog/extract-2025-03-14T18_10_16.493Z-Core SDK operations performance.csv

-61
This file was deleted.

‎helpers/README.md

-19
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ This directory contains utility modules that power the XMTP testing framework. T
77
| Module | Purpose |
88
| ------------------------------ | ------------------------------------------------- |
99
| [client.ts](#clientts) | Creates signers and manages keys for test workers |
10-
| [datadog.ts](#datadogts) | Sends performance metrics to Datadog |
1110
| [group.ts](#groupts) | Creates test groups with specified participants |
1211
| [logger.ts](#loggerts) | Logging utilities for test output |
1312
| [test.ts](#testts) | Test utilities for creating and managing tests |
@@ -32,24 +31,6 @@ const dbPath = getDbPath(workerName, accountAddress, testName);
3231
const encryptionKey = generateEncryptionKeyHex();
3332
```
3433

35-
### datadog.ts
36-
37-
Sends performance metrics to Datadog for monitoring:
38-
39-
```typescript
40-
// Initialize Datadog metrics
41-
initDataDog(testName, envValue, geolocation, apiKey);
42-
43-
// Send delivery rate metrics
44-
sendDeliveryMetric(deliveryRate, testName, libxmtpVersion);
45-
46-
// Send performance metrics
47-
sendPerformanceMetric(durationMs, testName, libxmtpVersion);
48-
49-
// Measure network performance
50-
const networkStats = await getNetworkStats();
51-
```
52-
5334
### group.ts
5435

5536
Utilities for creating and managing test groups:

‎helpers/client.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import { fromString, toString } from "uint8arrays";
88
import { createWalletClient, http, toBytes } from "viem";
99
import { privateKeyToAccount } from "viem/accounts";
1010
import { sepolia } from "viem/chains";
11+
import { b } from "vitest/dist/chunks/suite.qtkXWc6R.js";
1112
import { flushMetrics, initDataDog } from "./datadog";
1213
import { createLogger, flushLogger, overrideConsole } from "./logger";
1314

@@ -150,7 +151,7 @@ export function getAddressOfMember(members: GroupMember[], inboxId: string) {
150151
export async function closeEnv(testName: string, workers: WorkerManager) {
151152
flushLogger(testName);
152153

153-
await flushMetrics();
154+
await flushMetrics(testName);
154155
if (workers && typeof workers.getWorkers === "function") {
155156
for (const worker of workers.getWorkers()) {
156157
await worker.worker.terminate();

0 commit comments

Comments
 (0)