From 2a80ebea32861e652cea79a20782e4a5c1d35efb Mon Sep 17 00:00:00 2001 From: Kasper Ziemianek Date: Thu, 7 Dec 2023 15:45:03 +0100 Subject: [PATCH 01/19] resuming worker tests --- .github/workflows/ci.yml | 3 +-- tee-worker/docker/lit-resume-worker.yml | 3 +-- .../integration-tests/resuming_worker.test.ts | 15 +++++++++------ 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 838fbd9f06..ca7d477f11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -624,8 +624,6 @@ jobs: - test_name: lit-parentchain-nonce - test_name: lit-ii-batch-test - test_name: lit-test-stress-script - # disabled due to P-208 - # - test_name: lit-resume-worker steps: - uses: actions/checkout@v4 @@ -708,6 +706,7 @@ jobs: - test_name: lit-ii-batch-test-multiworker - test_name: lit-ii-identity-multiworker-test - test_name: lit-ii-vc-multiworker-test + - test_name: lit-resume-worker steps: - uses: actions/checkout@v4 diff --git a/tee-worker/docker/lit-resume-worker.yml b/tee-worker/docker/lit-resume-worker.yml index c1beb8c6ce..6e23ed6ebb 100644 --- a/tee-worker/docker/lit-resume-worker.yml +++ b/tee-worker/docker/lit-resume-worker.yml @@ -19,5 +19,4 @@ services: restart: "no" networks: litentry-test-network: - driver: bridge -# docker compose -f docker-compose.yml -f resume-worker.yml up relaychain-alice relaychain-bob litentry-node --no-build --exit-code-from resume-worker -- resume-worker + driver: bridge \ No newline at end of file diff --git a/tee-worker/ts-tests/integration-tests/resuming_worker.test.ts b/tee-worker/ts-tests/integration-tests/resuming_worker.test.ts index 535ebb5399..6698e16a64 100644 --- a/tee-worker/ts-tests/integration-tests/resuming_worker.test.ts +++ b/tee-worker/ts-tests/integration-tests/resuming_worker.test.ts @@ -3,12 +3,14 @@ import * as readline from 'readline'; import fs from 'fs'; import * as path from 'path'; import * as process from 'process'; -import { step, xstep } from 'mocha-steps'; +import { step } from 'mocha-steps'; import WebSocketAsPromised from 'websocket-as-promised'; import os from 'os'; import { initWorkerConnection, sleep } from './common/utils'; import { assert } from 'chai'; import type { HexString } from '@polkadot/util/types'; +import * as base58 from 'micro-base58'; +import { u8aToHex } from '@polkadot/util'; type WorkerConfig = { name: string; @@ -168,7 +170,7 @@ async function spawnWorkerJob( const errorStream = readline.createInterface(job.stderr); errorStream.on('line', (line: string) => { console.warn(name, line); - if (line.includes('lock file: sidechain_db/LOCK: Resource temporarily unavailable')) { + if (line.includes('sidechain_db/LOCK: Resource temporarily unavailable')) { reject(new SidechainDbLockUnavailable()); } }); @@ -178,7 +180,7 @@ async function spawnWorkerJob( outputStream.on('line', (line: string) => { console.log(name, line); - const match = line.match(/^Successfully initialized shard (?0x[\w\d]{64}).*/); + const match = line.match(/^Successfully initialized shard "(?[\w]{44}).*"/); if (match !== null) { /** * Assertions needed because regex contents aren't reflected in function typing; @@ -188,7 +190,8 @@ async function spawnWorkerJob( * as well as the corresponding named capturing groups. See * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/match */ - shard = match.groups!.shard as HexString; + const base58shard = match.groups!.shard; + shard = u8aToHex(base58.decode(base58shard)) as HexString; return; } @@ -390,7 +393,7 @@ describe('Resume worker', function () { let worker1State: WorkerState | undefined = undefined; // #fixme #1524 multiworker not supported - xstep('Two workers & resume worker1', async function () { + step('Two workers & resume worker1', async function () { assert(worker0State); // first launch worker1 @@ -423,7 +426,7 @@ describe('Resume worker', function () { }); // #fixme #1524 multiworker not supported - xstep('Kill and resume both workers', async function () { + step('Kill and resume both workers', async function () { assert(worker0State); assert(worker1State); From 399831b182ebfa3b69d1e1b140547b6d76f6909c Mon Sep 17 00:00:00 2001 From: Kasper Ziemianek Date: Thu, 7 Dec 2023 17:07:41 +0100 Subject: [PATCH 02/19] move resuming worker test to separate workspace --- ..._ts_test.sh => lit_ts_integration_test.sh} | 2 +- tee-worker/cli/lit_ts_worker_test.sh | 18 +++++++ tee-worker/ts-tests/worker/.env.local.example | 3 ++ tee-worker/ts-tests/worker/.env.staging | 4 ++ tee-worker/ts-tests/worker/.eslintrc.json | 40 +++++++++++++++ tee-worker/ts-tests/worker/package.json | 49 +++++++++++++++++++ .../resuming_worker.test.ts | 0 tee-worker/ts-tests/worker/tsconfig.json | 24 +++++++++ 8 files changed, 139 insertions(+), 1 deletion(-) rename tee-worker/cli/{lit_ts_test.sh => lit_ts_integration_test.sh} (94%) create mode 100755 tee-worker/cli/lit_ts_worker_test.sh create mode 100644 tee-worker/ts-tests/worker/.env.local.example create mode 100644 tee-worker/ts-tests/worker/.env.staging create mode 100644 tee-worker/ts-tests/worker/.eslintrc.json create mode 100644 tee-worker/ts-tests/worker/package.json rename tee-worker/ts-tests/{integration-tests => worker}/resuming_worker.test.ts (100%) create mode 100644 tee-worker/ts-tests/worker/tsconfig.json diff --git a/tee-worker/cli/lit_ts_test.sh b/tee-worker/cli/lit_ts_integration_test.sh similarity index 94% rename from tee-worker/cli/lit_ts_test.sh rename to tee-worker/cli/lit_ts_integration_test.sh index 718cf86cc4..bdce639176 100755 --- a/tee-worker/cli/lit_ts_test.sh +++ b/tee-worker/cli/lit_ts_integration_test.sh @@ -40,7 +40,7 @@ CLIENT="${CLIENT_BIN} -p ${NPORT} -P ${WORKER1PORT} -u ${NODEURL} -U ${WORKER1UR function usage() { echo "" - echo "This is a script for tee-worker ts-test. Preparing to test: $1" + echo "This is a script for tee-worker integration ts tests. Pass test name as first argument" echo "" } diff --git a/tee-worker/cli/lit_ts_worker_test.sh b/tee-worker/cli/lit_ts_worker_test.sh new file mode 100755 index 0000000000..d3323d95ba --- /dev/null +++ b/tee-worker/cli/lit_ts_worker_test.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Copyright 2020-2023 Trust Computing GmbH. + +set -euo pipefail + +function usage() { + echo "" + echo "This is a script for tee-worker worker ts-test. Pass test name as first argument" + echo "" +} + +[ $# -ne 1 ] && (usage; exit 1) +TEST=$1 + +cd /ts-tests +pnpm install +pnpm --filter worker run $TEST:staging diff --git a/tee-worker/ts-tests/worker/.env.local.example b/tee-worker/ts-tests/worker/.env.local.example new file mode 100644 index 0000000000..fc6c3f0a50 --- /dev/null +++ b/tee-worker/ts-tests/worker/.env.local.example @@ -0,0 +1,3 @@ +NODE_ENV = local +WORKER_ENDPOINT = wss://localhost:2000 +NODE_ENDPOINT = "ws://localhost:9944" \ No newline at end of file diff --git a/tee-worker/ts-tests/worker/.env.staging b/tee-worker/ts-tests/worker/.env.staging new file mode 100644 index 0000000000..dbf117bf8f --- /dev/null +++ b/tee-worker/ts-tests/worker/.env.staging @@ -0,0 +1,4 @@ +NODE_ENV=staging +WORKER_ENDPOINT=wss://litentry-worker-1:2011 +NODE_ENDPOINT="ws://litentry-node:9912" +NODE_TLS_REJECT_UNAUTHORIZED=0 diff --git a/tee-worker/ts-tests/worker/.eslintrc.json b/tee-worker/ts-tests/worker/.eslintrc.json new file mode 100644 index 0000000000..0e93a1188c --- /dev/null +++ b/tee-worker/ts-tests/worker/.eslintrc.json @@ -0,0 +1,40 @@ +{ + "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended"], + "parser": "@typescript-eslint/parser", + "plugins": ["@typescript-eslint"], + "root": true, + "rules": { + /** + It's a temporary solution, folks. We had no choice but to shut it off, + because there's just a liiittle bit too much "any" lurking around in the code. + But fear not, my friends, for this is not the end of the story. + We shall return, armed with determination and resolve, + to tackle those "any" types head-on in the near future. + **/ + "@typescript-eslint/no-explicit-any": ["off"], + "@typescript-eslint/no-non-null-assertion": ["off"], + "@typescript-eslint/no-var-requires": ["off"], + + // explanation: https://typescript-eslint.io/rules/naming-convention/ + "@typescript-eslint/naming-convention": [ + "error", + { + "selector": "typeLike", + "format": ["StrictPascalCase"] + }, + { + "selector": "variable", + "modifiers": ["const"], + "format": ["strictCamelCase", "UPPER_CASE"] + }, + { + "selector": "function", + "format": ["strictCamelCase", "StrictPascalCase"] + }, + { + "selector": "parameter", + "format": ["strictCamelCase"] + } + ] + } +} diff --git a/tee-worker/ts-tests/worker/package.json b/tee-worker/ts-tests/worker/package.json new file mode 100644 index 0000000000..1173ee80c6 --- /dev/null +++ b/tee-worker/ts-tests/worker/package.json @@ -0,0 +1,49 @@ +{ + "name": "stress", + "license": "ISC", + "type": "module", + "scripts": { + "format": "pnpm exec prettier --write .", + "check-format": "pnpm exec prettier --check .", + "run-script": "NODE_TLS_REJECT_UNAUTHORIZED=0 pnpm exec ts-node bin/stress.ts", + "test-stress-script:local": "pnpm exec cross-env NODE_ENV=local mocha --exit --sort -r ts-node/register --loader=ts-node/esm 'test/test.ts'", + "test-stress-script:staging": "pnpm exec cross-env NODE_ENV=staging mocha --exit --sort -r ts-node/register --loader=ts-node/esm 'test/test.ts'" + }, + "dependencies": { + "@noble/ed25519": "^1.7.3", + "@polkadot/api": "^10.9.1", + "@polkadot/keyring": "^12.2.1", + "@polkadot/typegen": "^10.9.1", + "@polkadot/types": "^10.9.1", + "add": "^2.0.6", + "ajv": "^8.12.0", + "colors": "^1.4.0", + "js-base64": "^3.7.5", + "micro-base58": "^0.5.1", + "scale-ts": "^0.2.11", + "websocket-as-promised": "^2.0.1", + "ws": "^8.8.1", + "zod": "^3.22.3" + }, + "devDependencies": { + "@ethersproject/providers": "^5.7.2", + "@types/chai": "^4.3.3", + "@types/mocha": "^10.0.0", + "@types/node": "^20.4.4", + "@types/ws": "^8.5.3", + "@typescript-eslint/eslint-plugin": "^5.60.0", + "@typescript-eslint/parser": "^5.60.0", + "chai": "^4.3.6", + "cross-env": "^7.0.3", + "dotenv": "^16.3.1", + "eslint": "^8.43.0", + "ethers": "^5.7.2", + "mocha": "^10.1.0", + "parachain-api": "../../client-api/parachain-api", + "prettier": "2.8.1", + "sidechain-api": "../../client-api/sidechain-api", + "ts-node": "^10.9.1", + "typescript": "5.0.4" + }, + "packageManager": "pnpm@8.7.6" +} diff --git a/tee-worker/ts-tests/integration-tests/resuming_worker.test.ts b/tee-worker/ts-tests/worker/resuming_worker.test.ts similarity index 100% rename from tee-worker/ts-tests/integration-tests/resuming_worker.test.ts rename to tee-worker/ts-tests/worker/resuming_worker.test.ts diff --git a/tee-worker/ts-tests/worker/tsconfig.json b/tee-worker/ts-tests/worker/tsconfig.json new file mode 100644 index 0000000000..23863b8768 --- /dev/null +++ b/tee-worker/ts-tests/worker/tsconfig.json @@ -0,0 +1,24 @@ +{ + "compilerOptions": { + "target": "ESNext", + "module": "ESNext", + "moduleResolution": "Node", + "declaration": true, + "strict": true, + "skipLibCheck": true, + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "resolveJsonModule": true, + "baseUrl": "." + }, + "paths": { + "parachain-api/*": ["../parachain-api/dist/*", "../parachain-api/build/*"], + "sidechain-api/*": ["../sidechain-api/src/*", "../sidechain-api/build/*"] + }, + "references": [{ "path": "../parachain-api" }, { "path": "../sidechain-api" }], + "ts-node": { + "esm": true, + "experimentalResolver": true, + "experimentalSpecifierResolution": "node" + } +} From 7b508c78b81d947c2f73f54778b2128317b32846 Mon Sep 17 00:00:00 2001 From: Kasper Ziemianek Date: Thu, 7 Dec 2023 21:29:08 +0100 Subject: [PATCH 03/19] correct script name --- tee-worker/docker/lit-di-evm-identity-multiworker-test.yml | 2 +- tee-worker/docker/lit-di-evm-identity-test.yml | 2 +- .../docker/lit-di-substrate-identity-multiworker-test.yml | 2 +- tee-worker/docker/lit-di-substrate-identity-test.yml | 2 +- tee-worker/docker/lit-di-vc-multiworker-test.yml | 2 +- tee-worker/docker/lit-di-vc-test.yml | 2 +- tee-worker/docker/lit-ii-batch-test-multiworker.yml | 2 +- tee-worker/docker/lit-ii-batch-test.yml | 2 +- tee-worker/docker/lit-ii-identity-multiworker-test.yml | 2 +- tee-worker/docker/lit-ii-identity-test.yml | 2 +- tee-worker/docker/lit-ii-vc-multiworker-test.yml | 2 +- tee-worker/docker/lit-ii-vc-test.yml | 2 +- tee-worker/docker/lit-resume-worker.yml | 2 +- tee-worker/docker/lit-test-stress-script.yml | 2 +- tee-worker/ts-tests/worker/.env.local.example | 1 - tee-worker/ts-tests/worker/.env.staging | 1 - 16 files changed, 14 insertions(+), 16 deletions(-) diff --git a/tee-worker/docker/lit-di-evm-identity-multiworker-test.yml b/tee-worker/docker/lit-di-evm-identity-multiworker-test.yml index dbceb71a7a..4649e93fdb 100644 --- a/tee-worker/docker/lit-di-evm-identity-multiworker-test.yml +++ b/tee-worker/docker/lit-di-evm-identity-multiworker-test.yml @@ -21,7 +21,7 @@ services: condition: service_healthy networks: - litentry-test-network - entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_test.sh test-di-evm-identity 2>&1' " + entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_integration_test.sh test-di-evm-identity 2>&1' " restart: "no" networks: litentry-test-network: diff --git a/tee-worker/docker/lit-di-evm-identity-test.yml b/tee-worker/docker/lit-di-evm-identity-test.yml index 970580e7a7..7938ca67a8 100644 --- a/tee-worker/docker/lit-di-evm-identity-test.yml +++ b/tee-worker/docker/lit-di-evm-identity-test.yml @@ -17,7 +17,7 @@ services: condition: service_healthy networks: - litentry-test-network - entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_test.sh test-di-evm-identity 2>&1' " + entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_integration_test.sh test-di-evm-identity 2>&1' " restart: "no" networks: litentry-test-network: diff --git a/tee-worker/docker/lit-di-substrate-identity-multiworker-test.yml b/tee-worker/docker/lit-di-substrate-identity-multiworker-test.yml index 53eb542803..f5647624b9 100644 --- a/tee-worker/docker/lit-di-substrate-identity-multiworker-test.yml +++ b/tee-worker/docker/lit-di-substrate-identity-multiworker-test.yml @@ -21,7 +21,7 @@ services: condition: service_healthy networks: - litentry-test-network - entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_test.sh test-di-substrate-identity 2>&1' " + entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_integration_test.sh test-di-substrate-identity 2>&1' " restart: "no" networks: litentry-test-network: diff --git a/tee-worker/docker/lit-di-substrate-identity-test.yml b/tee-worker/docker/lit-di-substrate-identity-test.yml index 3cb262127f..3f1c8250c7 100644 --- a/tee-worker/docker/lit-di-substrate-identity-test.yml +++ b/tee-worker/docker/lit-di-substrate-identity-test.yml @@ -17,7 +17,7 @@ services: condition: service_healthy networks: - litentry-test-network - entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_test.sh test-di-substrate-identity 2>&1' " + entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_integration_test.sh test-di-substrate-identity 2>&1' " restart: "no" networks: litentry-test-network: diff --git a/tee-worker/docker/lit-di-vc-multiworker-test.yml b/tee-worker/docker/lit-di-vc-multiworker-test.yml index 18a694893c..08bb708212 100644 --- a/tee-worker/docker/lit-di-vc-multiworker-test.yml +++ b/tee-worker/docker/lit-di-vc-multiworker-test.yml @@ -21,7 +21,7 @@ services: condition: service_healthy networks: - litentry-test-network - entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_test.sh test-di-vc 2>&1' " + entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_integration_test.sh test-di-vc 2>&1' " restart: "no" networks: litentry-test-network: diff --git a/tee-worker/docker/lit-di-vc-test.yml b/tee-worker/docker/lit-di-vc-test.yml index 8a5fb637d2..a41ebfc457 100644 --- a/tee-worker/docker/lit-di-vc-test.yml +++ b/tee-worker/docker/lit-di-vc-test.yml @@ -17,7 +17,7 @@ services: condition: service_healthy networks: - litentry-test-network - entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_test.sh test-di-vc 2>&1' " + entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_integration_test.sh test-di-vc 2>&1' " restart: "no" networks: litentry-test-network: diff --git a/tee-worker/docker/lit-ii-batch-test-multiworker.yml b/tee-worker/docker/lit-ii-batch-test-multiworker.yml index e182d454af..6d26ecea5d 100644 --- a/tee-worker/docker/lit-ii-batch-test-multiworker.yml +++ b/tee-worker/docker/lit-ii-batch-test-multiworker.yml @@ -21,7 +21,7 @@ services: condition: service_healthy networks: - litentry-test-network - entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_test.sh test-ii-batch 2>&1' " + entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_integration_test.sh test-ii-batch 2>&1' " restart: "no" networks: litentry-test-network: diff --git a/tee-worker/docker/lit-ii-batch-test.yml b/tee-worker/docker/lit-ii-batch-test.yml index f5fb42b428..f0a6ee9fbf 100644 --- a/tee-worker/docker/lit-ii-batch-test.yml +++ b/tee-worker/docker/lit-ii-batch-test.yml @@ -17,7 +17,7 @@ services: condition: service_healthy networks: - litentry-test-network - entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_test.sh test-ii-batch 2>&1' " + entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_integration_test.sh test-ii-batch 2>&1' " restart: "no" networks: litentry-test-network: diff --git a/tee-worker/docker/lit-ii-identity-multiworker-test.yml b/tee-worker/docker/lit-ii-identity-multiworker-test.yml index 041e20ebd6..684f832367 100644 --- a/tee-worker/docker/lit-ii-identity-multiworker-test.yml +++ b/tee-worker/docker/lit-ii-identity-multiworker-test.yml @@ -21,7 +21,7 @@ services: condition: service_healthy networks: - litentry-test-network - entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_test.sh test-ii-identity 2>&1' " + entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_integration_test.sh test-ii-identity 2>&1' " restart: "no" networks: litentry-test-network: diff --git a/tee-worker/docker/lit-ii-identity-test.yml b/tee-worker/docker/lit-ii-identity-test.yml index 5f2011f1ed..874b3dad67 100644 --- a/tee-worker/docker/lit-ii-identity-test.yml +++ b/tee-worker/docker/lit-ii-identity-test.yml @@ -17,7 +17,7 @@ services: condition: service_healthy networks: - litentry-test-network - entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_test.sh test-ii-identity 2>&1' " + entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_integration_test.sh test-ii-identity 2>&1' " restart: "no" networks: litentry-test-network: diff --git a/tee-worker/docker/lit-ii-vc-multiworker-test.yml b/tee-worker/docker/lit-ii-vc-multiworker-test.yml index f762375df9..f84a6e52a2 100644 --- a/tee-worker/docker/lit-ii-vc-multiworker-test.yml +++ b/tee-worker/docker/lit-ii-vc-multiworker-test.yml @@ -21,7 +21,7 @@ services: condition: service_healthy networks: - litentry-test-network - entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_test.sh test-ii-vc 2>&1' " + entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_integration_test.sh test-ii-vc 2>&1' " restart: "no" networks: litentry-test-network: diff --git a/tee-worker/docker/lit-ii-vc-test.yml b/tee-worker/docker/lit-ii-vc-test.yml index d72852b81b..f530e499f8 100644 --- a/tee-worker/docker/lit-ii-vc-test.yml +++ b/tee-worker/docker/lit-ii-vc-test.yml @@ -17,7 +17,7 @@ services: condition: service_healthy networks: - litentry-test-network - entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_test.sh test-ii-vc 2>&1' " + entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_integration_test.sh test-ii-vc 2>&1' " restart: "no" networks: litentry-test-network: diff --git a/tee-worker/docker/lit-resume-worker.yml b/tee-worker/docker/lit-resume-worker.yml index 6e23ed6ebb..319d426d2f 100644 --- a/tee-worker/docker/lit-resume-worker.yml +++ b/tee-worker/docker/lit-resume-worker.yml @@ -15,7 +15,7 @@ services: condition: service_healthy networks: - litentry-test-network - entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_test.sh test-resuming-worker 2>&1' " + entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_worker_test.sh test-resuming-worker 2>&1' " restart: "no" networks: litentry-test-network: diff --git a/tee-worker/docker/lit-test-stress-script.yml b/tee-worker/docker/lit-test-stress-script.yml index 00b08d3d68..735670a852 100644 --- a/tee-worker/docker/lit-test-stress-script.yml +++ b/tee-worker/docker/lit-test-stress-script.yml @@ -17,7 +17,7 @@ services: condition: service_healthy networks: - litentry-test-network - entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_test.sh test-stress-script 2>&1' " + entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_integration_test.sh test-stress-script 2>&1' " restart: "no" networks: litentry-test-network: diff --git a/tee-worker/ts-tests/worker/.env.local.example b/tee-worker/ts-tests/worker/.env.local.example index fc6c3f0a50..7c704c45fc 100644 --- a/tee-worker/ts-tests/worker/.env.local.example +++ b/tee-worker/ts-tests/worker/.env.local.example @@ -1,3 +1,2 @@ NODE_ENV = local -WORKER_ENDPOINT = wss://localhost:2000 NODE_ENDPOINT = "ws://localhost:9944" \ No newline at end of file diff --git a/tee-worker/ts-tests/worker/.env.staging b/tee-worker/ts-tests/worker/.env.staging index dbf117bf8f..4c93f40871 100644 --- a/tee-worker/ts-tests/worker/.env.staging +++ b/tee-worker/ts-tests/worker/.env.staging @@ -1,4 +1,3 @@ NODE_ENV=staging -WORKER_ENDPOINT=wss://litentry-worker-1:2011 NODE_ENDPOINT="ws://litentry-node:9912" NODE_TLS_REJECT_UNAUTHORIZED=0 From 8a928050022d92ce2a174e1488289c77345bf850 Mon Sep 17 00:00:00 2001 From: Kasper Ziemianek Date: Thu, 7 Dec 2023 23:14:09 +0100 Subject: [PATCH 04/19] add workspace --- tee-worker/ts-tests/pnpm-workspace.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tee-worker/ts-tests/pnpm-workspace.yaml b/tee-worker/ts-tests/pnpm-workspace.yaml index 7b9194120a..5593dc92b9 100644 --- a/tee-worker/ts-tests/pnpm-workspace.yaml +++ b/tee-worker/ts-tests/pnpm-workspace.yaml @@ -1 +1 @@ -packages: ["integration-tests", "stress"] +packages: ["integration-tests", "stress", "worker"] From b5c4df29bf6da6070321cadeface602031197ee8 Mon Sep 17 00:00:00 2001 From: Kasper Ziemianek Date: Fri, 8 Dec 2023 10:17:27 +0100 Subject: [PATCH 05/19] fix evm build --- tee-worker/cli/src/evm/commands/evm_read.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tee-worker/cli/src/evm/commands/evm_read.rs b/tee-worker/cli/src/evm/commands/evm_read.rs index 4cc3c17556..b863533860 100644 --- a/tee-worker/cli/src/evm/commands/evm_read.rs +++ b/tee-worker/cli/src/evm/commands/evm_read.rs @@ -17,7 +17,7 @@ use crate::{ trusted_cli::TrustedCli, trusted_command_utils::get_pair_from_str, - trusted_operation::perform_trusted_operation, Cli, CliResult, CliResultOk, + trusted_operation::perform_trusted_operation, Cli, CliError, CliResult, CliResultOk, }; use ita_stf::{Getter, TrustedCallSigned, TrustedGetter}; use itp_stf_primitives::types::{KeyPair, TrustedOperation}; From 5e0e9499cde66e82892d4e3b16dfe90b51b647bc Mon Sep 17 00:00:00 2001 From: Kasper Ziemianek Date: Fri, 8 Dec 2023 10:22:37 +0100 Subject: [PATCH 06/19] lock file --- tee-worker/ts-tests/pnpm-lock.yaml | 100 +++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/tee-worker/ts-tests/pnpm-lock.yaml b/tee-worker/ts-tests/pnpm-lock.yaml index 53a61d3a2b..5ffbace37d 100644 --- a/tee-worker/ts-tests/pnpm-lock.yaml +++ b/tee-worker/ts-tests/pnpm-lock.yaml @@ -238,6 +238,106 @@ importers: specifier: 5.0.4 version: 5.0.4 + worker: + dependencies: + '@noble/ed25519': + specifier: ^1.7.3 + version: 1.7.3 + '@polkadot/api': + specifier: ^10.9.1 + version: 10.9.1 + '@polkadot/keyring': + specifier: ^12.2.1 + version: 12.5.1(@polkadot/util-crypto@12.5.1)(@polkadot/util@12.5.1) + '@polkadot/typegen': + specifier: ^10.9.1 + version: 10.9.1 + '@polkadot/types': + specifier: ^10.9.1 + version: 10.9.1 + add: + specifier: ^2.0.6 + version: 2.0.6 + ajv: + specifier: ^8.12.0 + version: 8.12.0 + colors: + specifier: ^1.4.0 + version: 1.4.0 + js-base64: + specifier: ^3.7.5 + version: 3.7.5 + micro-base58: + specifier: ^0.5.1 + version: 0.5.1 + scale-ts: + specifier: ^0.2.11 + version: 0.2.12 + websocket-as-promised: + specifier: ^2.0.1 + version: 2.0.1 + ws: + specifier: ^8.8.1 + version: 8.14.2 + zod: + specifier: ^3.22.3 + version: 3.22.3 + devDependencies: + '@ethersproject/providers': + specifier: ^5.7.2 + version: 5.7.2 + '@types/chai': + specifier: ^4.3.3 + version: 4.3.6 + '@types/mocha': + specifier: ^10.0.0 + version: 10.0.2 + '@types/node': + specifier: ^20.4.4 + version: 20.7.1 + '@types/ws': + specifier: ^8.5.3 + version: 8.5.6 + '@typescript-eslint/eslint-plugin': + specifier: ^5.60.0 + version: 5.62.0(@typescript-eslint/parser@5.62.0)(eslint@8.50.0)(typescript@5.0.4) + '@typescript-eslint/parser': + specifier: ^5.60.0 + version: 5.62.0(eslint@8.50.0)(typescript@5.0.4) + chai: + specifier: ^4.3.6 + version: 4.3.10 + cross-env: + specifier: ^7.0.3 + version: 7.0.3 + dotenv: + specifier: ^16.3.1 + version: 16.3.1 + eslint: + specifier: ^8.43.0 + version: 8.50.0 + ethers: + specifier: ^5.7.2 + version: 5.7.2 + mocha: + specifier: ^10.1.0 + version: 10.2.0 + parachain-api: + specifier: ../../client-api/parachain-api + version: link:../../client-api/parachain-api + prettier: + specifier: 2.8.1 + version: 2.8.1 + sidechain-api: + specifier: ../../client-api/sidechain-api + version: link:../../client-api/sidechain-api + ts-node: + specifier: ^10.9.1 + version: 10.9.1(@types/node@20.7.1)(typescript@5.0.4) + typescript: + specifier: 5.0.4 + version: 5.0.4 + packages: /@aashutoshrathi/word-wrap@1.2.6: From 4da0d0571ba5a309c91ceaea50ac21de5506c457 Mon Sep 17 00:00:00 2001 From: Kasper Ziemianek Date: Mon, 11 Dec 2023 13:15:31 +0100 Subject: [PATCH 07/19] fix package.json --- .../ts-tests/integration-tests/package.json | 2 - tee-worker/ts-tests/pnpm-lock.yaml | 99 ++----------------- tee-worker/ts-tests/worker/package.json | 42 ++------ .../ts-tests/worker/resuming_worker.test.ts | 2 +- 4 files changed, 13 insertions(+), 132 deletions(-) diff --git a/tee-worker/ts-tests/integration-tests/package.json b/tee-worker/ts-tests/integration-tests/package.json index a4fd1b74c1..083e8dd871 100644 --- a/tee-worker/ts-tests/integration-tests/package.json +++ b/tee-worker/ts-tests/integration-tests/package.json @@ -13,8 +13,6 @@ "test-di-evm-identity:staging": "pnpm exec eslint . && pnpm exec cross-env NODE_ENV=staging mocha --exit --sort -r ts-node/register --loader=ts-node/esm 'di_evm_identity.test.ts'", "test-di-vc:local": "pnpm exec eslint . && pnpm exec cross-env NODE_ENV=local mocha --exit --sort -r ts-node/register --loader=ts-node/esm 'di_vc.test.ts'", "test-di-vc:staging": "pnpm exec eslint . && pnpm exec cross-env NODE_ENV=staging mocha --exit --sort -r ts-node/register --loader=ts-node/esm 'di_vc.test.ts'", - "test-resuming-worker:staging": "pnpm exec eslint . && pnpm exec cross-env NODE_ENV=staging mocha --exit --sort -r ts-node/register --loader=ts-node/esm 'resuming_worker.test.ts'", - "test-resuming-worker:local": "pnpm exec eslint . && pnpm exec cross-env NODE_ENV=local mocha --exit --sort -r ts-node/register --loader=ts-node/esm 'resuming_worker.test.ts'", "test-ii-vc:local": "pnpm exec eslint . && pnpm exec cross-env NODE_ENV=local mocha --exit --sort -r ts-node/register --loader=ts-node/esm 'ii_vc.test.ts'", "test-ii-vc:staging": "pnpm exec eslint . && pnpm exec cross-env NODE_ENV=staging mocha --exit --sort -r ts-node/register --loader=ts-node/esm 'ii_vc.test.ts'", "test-ii-batch:local": "pnpm exec eslint . && pnpm exec cross-env NODE_ENV=local mocha --exit --sort -r ts-node/register --loader=ts-node/esm 'ii_batch.test.ts'", diff --git a/tee-worker/ts-tests/pnpm-lock.yaml b/tee-worker/ts-tests/pnpm-lock.yaml index 5ffbace37d..9426deb13e 100644 --- a/tee-worker/ts-tests/pnpm-lock.yaml +++ b/tee-worker/ts-tests/pnpm-lock.yaml @@ -240,103 +240,16 @@ importers: worker: dependencies: - '@noble/ed25519': - specifier: ^1.7.3 - version: 1.7.3 - '@polkadot/api': - specifier: ^10.9.1 - version: 10.9.1 - '@polkadot/keyring': - specifier: ^12.2.1 - version: 12.5.1(@polkadot/util-crypto@12.5.1)(@polkadot/util@12.5.1) - '@polkadot/typegen': - specifier: ^10.9.1 - version: 10.9.1 - '@polkadot/types': - specifier: ^10.9.1 - version: 10.9.1 - add: - specifier: ^2.0.6 - version: 2.0.6 - ajv: - specifier: ^8.12.0 - version: 8.12.0 - colors: - specifier: ^1.4.0 - version: 1.4.0 - js-base64: - specifier: ^3.7.5 - version: 3.7.5 - micro-base58: - specifier: ^0.5.1 - version: 0.5.1 - scale-ts: - specifier: ^0.2.11 - version: 0.2.12 - websocket-as-promised: - specifier: ^2.0.1 - version: 2.0.1 - ws: - specifier: ^8.8.1 - version: 8.14.2 - zod: - specifier: ^3.22.3 - version: 3.22.3 + '@polkadot/util': + specifier: ^12.5.1 + version: 12.5.1 + mocha-steps: + specifier: ^1.3.0 + version: 1.3.0 devDependencies: - '@ethersproject/providers': - specifier: ^5.7.2 - version: 5.7.2 - '@types/chai': - specifier: ^4.3.3 - version: 4.3.6 - '@types/mocha': - specifier: ^10.0.0 - version: 10.0.2 - '@types/node': - specifier: ^20.4.4 - version: 20.7.1 - '@types/ws': - specifier: ^8.5.3 - version: 8.5.6 - '@typescript-eslint/eslint-plugin': - specifier: ^5.60.0 - version: 5.62.0(@typescript-eslint/parser@5.62.0)(eslint@8.50.0)(typescript@5.0.4) - '@typescript-eslint/parser': - specifier: ^5.60.0 - version: 5.62.0(eslint@8.50.0)(typescript@5.0.4) - chai: - specifier: ^4.3.6 - version: 4.3.10 cross-env: specifier: ^7.0.3 version: 7.0.3 - dotenv: - specifier: ^16.3.1 - version: 16.3.1 - eslint: - specifier: ^8.43.0 - version: 8.50.0 - ethers: - specifier: ^5.7.2 - version: 5.7.2 - mocha: - specifier: ^10.1.0 - version: 10.2.0 - parachain-api: - specifier: ../../client-api/parachain-api - version: link:../../client-api/parachain-api - prettier: - specifier: 2.8.1 - version: 2.8.1 - sidechain-api: - specifier: ../../client-api/sidechain-api - version: link:../../client-api/sidechain-api - ts-node: - specifier: ^10.9.1 - version: 10.9.1(@types/node@20.7.1)(typescript@5.0.4) - typescript: - specifier: 5.0.4 - version: 5.0.4 packages: diff --git a/tee-worker/ts-tests/worker/package.json b/tee-worker/ts-tests/worker/package.json index 1173ee80c6..e3cbd5d0dd 100644 --- a/tee-worker/ts-tests/worker/package.json +++ b/tee-worker/ts-tests/worker/package.json @@ -1,49 +1,19 @@ { - "name": "stress", + "name": "worker", "license": "ISC", "type": "module", "scripts": { "format": "pnpm exec prettier --write .", "check-format": "pnpm exec prettier --check .", - "run-script": "NODE_TLS_REJECT_UNAUTHORIZED=0 pnpm exec ts-node bin/stress.ts", - "test-stress-script:local": "pnpm exec cross-env NODE_ENV=local mocha --exit --sort -r ts-node/register --loader=ts-node/esm 'test/test.ts'", - "test-stress-script:staging": "pnpm exec cross-env NODE_ENV=staging mocha --exit --sort -r ts-node/register --loader=ts-node/esm 'test/test.ts'" + "test-resuming-worker:staging": "pnpm exec eslint . && pnpm exec cross-env NODE_ENV=staging mocha --exit --sort -r ts-node/register --loader=ts-node/esm 'resuming_worker.test.ts'", + "test-resuming-worker:local": "pnpm exec eslint . && pnpm exec cross-env NODE_ENV=local mocha --exit --sort -r ts-node/register --loader=ts-node/esm 'resuming_worker.test.ts'" }, "dependencies": { - "@noble/ed25519": "^1.7.3", - "@polkadot/api": "^10.9.1", - "@polkadot/keyring": "^12.2.1", - "@polkadot/typegen": "^10.9.1", - "@polkadot/types": "^10.9.1", - "add": "^2.0.6", - "ajv": "^8.12.0", - "colors": "^1.4.0", - "js-base64": "^3.7.5", - "micro-base58": "^0.5.1", - "scale-ts": "^0.2.11", - "websocket-as-promised": "^2.0.1", - "ws": "^8.8.1", - "zod": "^3.22.3" + "mocha-steps": "^1.3.0", + "@polkadot/util": "^12.5.1" }, "devDependencies": { - "@ethersproject/providers": "^5.7.2", - "@types/chai": "^4.3.3", - "@types/mocha": "^10.0.0", - "@types/node": "^20.4.4", - "@types/ws": "^8.5.3", - "@typescript-eslint/eslint-plugin": "^5.60.0", - "@typescript-eslint/parser": "^5.60.0", - "chai": "^4.3.6", - "cross-env": "^7.0.3", - "dotenv": "^16.3.1", - "eslint": "^8.43.0", - "ethers": "^5.7.2", - "mocha": "^10.1.0", - "parachain-api": "../../client-api/parachain-api", - "prettier": "2.8.1", - "sidechain-api": "../../client-api/sidechain-api", - "ts-node": "^10.9.1", - "typescript": "5.0.4" + "cross-env": "^7.0.3" }, "packageManager": "pnpm@8.7.6" } diff --git a/tee-worker/ts-tests/worker/resuming_worker.test.ts b/tee-worker/ts-tests/worker/resuming_worker.test.ts index 6698e16a64..537c7a0bf5 100644 --- a/tee-worker/ts-tests/worker/resuming_worker.test.ts +++ b/tee-worker/ts-tests/worker/resuming_worker.test.ts @@ -6,7 +6,7 @@ import * as process from 'process'; import { step } from 'mocha-steps'; import WebSocketAsPromised from 'websocket-as-promised'; import os from 'os'; -import { initWorkerConnection, sleep } from './common/utils'; +import { initWorkerConnection, sleep } from './../integration-tests/common/utils'; import { assert } from 'chai'; import type { HexString } from '@polkadot/util/types'; import * as base58 from 'micro-base58'; From 87c8d5c22414cbde3b91b8186db28b8cb5b45aa4 Mon Sep 17 00:00:00 2001 From: Kasper Ziemianek Date: Tue, 12 Dec 2023 09:26:54 +0100 Subject: [PATCH 08/19] missing worker workspace deps --- tee-worker/ts-tests/pnpm-lock.yaml | 15 +++++++++++++++ tee-worker/ts-tests/worker/package.json | 9 +++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/tee-worker/ts-tests/pnpm-lock.yaml b/tee-worker/ts-tests/pnpm-lock.yaml index 9426deb13e..0362cc4e18 100644 --- a/tee-worker/ts-tests/pnpm-lock.yaml +++ b/tee-worker/ts-tests/pnpm-lock.yaml @@ -243,13 +243,28 @@ importers: '@polkadot/util': specifier: ^12.5.1 version: 12.5.1 + chai: + specifier: ^4.3.6 + version: 4.3.10 + micro-base58: + specifier: ^0.5.1 + version: 0.5.1 + mocha: + specifier: ^10.1.0 + version: 10.2.0 mocha-steps: specifier: ^1.3.0 version: 1.3.0 + websocket-as-promised: + specifier: ^2.0.1 + version: 2.0.1 devDependencies: cross-env: specifier: ^7.0.3 version: 7.0.3 + ts-node: + specifier: ^10.9.1 + version: 10.9.1(@types/node@20.7.1)(typescript@5.0.4) packages: diff --git a/tee-worker/ts-tests/worker/package.json b/tee-worker/ts-tests/worker/package.json index e3cbd5d0dd..24c02faeab 100644 --- a/tee-worker/ts-tests/worker/package.json +++ b/tee-worker/ts-tests/worker/package.json @@ -9,11 +9,16 @@ "test-resuming-worker:local": "pnpm exec eslint . && pnpm exec cross-env NODE_ENV=local mocha --exit --sort -r ts-node/register --loader=ts-node/esm 'resuming_worker.test.ts'" }, "dependencies": { + "chai": "^4.3.6", + "micro-base58": "^0.5.1", + "mocha": "^10.1.0", "mocha-steps": "^1.3.0", - "@polkadot/util": "^12.5.1" + "@polkadot/util": "^12.5.1", + "websocket-as-promised": "^2.0.1" }, "devDependencies": { - "cross-env": "^7.0.3" + "cross-env": "^7.0.3", + "ts-node": "^10.9.1" }, "packageManager": "pnpm@8.7.6" } From 51bbbfa7069bad31b7a99c50bfef6e187e00d3f1 Mon Sep 17 00:00:00 2001 From: Kasper Ziemianek Date: Wed, 20 Dec 2023 19:42:40 +0100 Subject: [PATCH 09/19] adjust bitcoin identity test docker compose --- tee-worker/docker/lit-di-bitcoin-identity-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tee-worker/docker/lit-di-bitcoin-identity-test.yml b/tee-worker/docker/lit-di-bitcoin-identity-test.yml index 8a4b5a2c59..9cc7e41c35 100644 --- a/tee-worker/docker/lit-di-bitcoin-identity-test.yml +++ b/tee-worker/docker/lit-di-bitcoin-identity-test.yml @@ -17,7 +17,7 @@ services: condition: service_healthy networks: - litentry-test-network - entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_test.sh test-di-bitcoin-identity 2>&1' " + entrypoint: "bash -c '/usr/local/worker-cli/lit_ts_integration_test.sh test-di-bitcoin-identity 2>&1' " restart: "no" networks: litentry-test-network: From a6f9fc655f3b2315b2caab4f818fd917b2752fb8 Mon Sep 17 00:00:00 2001 From: Kasper Ziemianek Date: Mon, 15 Jan 2024 14:49:13 +0100 Subject: [PATCH 10/19] fix test --- tee-worker/ts-tests/integration-tests/common/utils/crypto.ts | 2 +- tee-worker/ts-tests/worker/resuming_worker.test.ts | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tee-worker/ts-tests/integration-tests/common/utils/crypto.ts b/tee-worker/ts-tests/integration-tests/common/utils/crypto.ts index d6c6b4e306..333c637fc4 100644 --- a/tee-worker/ts-tests/integration-tests/common/utils/crypto.ts +++ b/tee-worker/ts-tests/integration-tests/common/utils/crypto.ts @@ -7,7 +7,7 @@ import { KeyringPair } from '@polkadot/keyring/types'; import { ethers } from 'ethers'; import { blake2AsU8a } from '@polkadot/util-crypto'; import bitcore from 'bitcore-lib'; -import { IntegrationTestContext } from 'common/common-types'; +import { IntegrationTestContext } from './../common-types'; import { buildIdentityHelper } from './identity-helper'; export type KeypairType = 'ed25519' | 'sr25519' | 'ecdsa' | 'ethereum' | 'bitcoin'; diff --git a/tee-worker/ts-tests/worker/resuming_worker.test.ts b/tee-worker/ts-tests/worker/resuming_worker.test.ts index 537c7a0bf5..ecafccf6ad 100644 --- a/tee-worker/ts-tests/worker/resuming_worker.test.ts +++ b/tee-worker/ts-tests/worker/resuming_worker.test.ts @@ -392,7 +392,6 @@ describe('Resume worker', function () { let worker1State: WorkerState | undefined = undefined; - // #fixme #1524 multiworker not supported step('Two workers & resume worker1', async function () { assert(worker0State); @@ -425,7 +424,6 @@ describe('Resume worker', function () { console.log('=========== worker1 produced blocks =================='); }); - // #fixme #1524 multiworker not supported step('Kill and resume both workers', async function () { assert(worker0State); assert(worker1State); From 5be490ba2309645f1a328450510a6ae8391c3bc6 Mon Sep 17 00:00:00 2001 From: Kasper Ziemianek Date: Tue, 16 Jan 2024 09:53:20 +0100 Subject: [PATCH 11/19] remove unwanted paths from tsconfig.json --- tee-worker/ts-tests/worker/tsconfig.json | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tee-worker/ts-tests/worker/tsconfig.json b/tee-worker/ts-tests/worker/tsconfig.json index 23863b8768..160c8db3b8 100644 --- a/tee-worker/ts-tests/worker/tsconfig.json +++ b/tee-worker/ts-tests/worker/tsconfig.json @@ -11,11 +11,6 @@ "resolveJsonModule": true, "baseUrl": "." }, - "paths": { - "parachain-api/*": ["../parachain-api/dist/*", "../parachain-api/build/*"], - "sidechain-api/*": ["../sidechain-api/src/*", "../sidechain-api/build/*"] - }, - "references": [{ "path": "../parachain-api" }, { "path": "../sidechain-api" }], "ts-node": { "esm": true, "experimentalResolver": true, From d7fbea94eab72d245b59c527f5882ce52524b79b Mon Sep 17 00:00:00 2001 From: Kasper Ziemianek Date: Tue, 16 Jan 2024 16:35:17 +0100 Subject: [PATCH 12/19] add config --- tee-worker/cli/lit_ts_worker_test.sh | 7 +++++++ tee-worker/ts-tests/worker/.env.staging | 1 + 2 files changed, 8 insertions(+) diff --git a/tee-worker/cli/lit_ts_worker_test.sh b/tee-worker/cli/lit_ts_worker_test.sh index d3323d95ba..8528b59691 100755 --- a/tee-worker/cli/lit_ts_worker_test.sh +++ b/tee-worker/cli/lit_ts_worker_test.sh @@ -13,6 +13,13 @@ function usage() { [ $# -ne 1 ] && (usage; exit 1) TEST=$1 + +BINARY_DIR="/usr/local/bin" +NODE_ENDPOINT="ws://litentry-node:9912" + +echo "Using binary dir: $BINARY_DIR" +echo "Using node endpoint: $NODE_ENDPOINT" + cd /ts-tests pnpm install pnpm --filter worker run $TEST:staging diff --git a/tee-worker/ts-tests/worker/.env.staging b/tee-worker/ts-tests/worker/.env.staging index 4c93f40871..4ac611a373 100644 --- a/tee-worker/ts-tests/worker/.env.staging +++ b/tee-worker/ts-tests/worker/.env.staging @@ -1,3 +1,4 @@ NODE_ENV=staging NODE_ENDPOINT="ws://litentry-node:9912" NODE_TLS_REJECT_UNAUTHORIZED=0 +BINARY_DIR=/usr/local/bin \ No newline at end of file From 08d8deb6db2b5078735a1f245dc0714d3a019b8d Mon Sep 17 00:00:00 2001 From: Kasper Ziemianek Date: Wed, 17 Jan 2024 09:38:59 +0100 Subject: [PATCH 13/19] add tracing --- tee-worker/ts-tests/worker/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tee-worker/ts-tests/worker/package.json b/tee-worker/ts-tests/worker/package.json index 24c02faeab..a3848f62a2 100644 --- a/tee-worker/ts-tests/worker/package.json +++ b/tee-worker/ts-tests/worker/package.json @@ -5,7 +5,7 @@ "scripts": { "format": "pnpm exec prettier --write .", "check-format": "pnpm exec prettier --check .", - "test-resuming-worker:staging": "pnpm exec eslint . && pnpm exec cross-env NODE_ENV=staging mocha --exit --sort -r ts-node/register --loader=ts-node/esm 'resuming_worker.test.ts'", + "test-resuming-worker:staging": "pnpm exec eslint . && pnpm exec cross-env NODE_ENV=staging mocha --trace-warnings --full-trace --exit --sort -r ts-node/register --loader=ts-node/esm 'resuming_worker.test.ts'", "test-resuming-worker:local": "pnpm exec eslint . && pnpm exec cross-env NODE_ENV=local mocha --exit --sort -r ts-node/register --loader=ts-node/esm 'resuming_worker.test.ts'" }, "dependencies": { From b43d89f4df7a059ef1b5236f4950e9f645e9b01c Mon Sep 17 00:00:00 2001 From: Kasper Ziemianek Date: Wed, 17 Jan 2024 09:53:41 +0100 Subject: [PATCH 14/19] add more dev deps --- tee-worker/ts-tests/pnpm-lock.yaml | 102 ++++++++++++++++++ tee-worker/ts-tests/worker/.env.local.example | 3 +- tee-worker/ts-tests/worker/package.json | 38 ++++++- 3 files changed, 140 insertions(+), 3 deletions(-) diff --git a/tee-worker/ts-tests/pnpm-lock.yaml b/tee-worker/ts-tests/pnpm-lock.yaml index e3d72b8652..da37548f54 100644 --- a/tee-worker/ts-tests/pnpm-lock.yaml +++ b/tee-worker/ts-tests/pnpm-lock.yaml @@ -246,12 +246,69 @@ importers: worker: dependencies: + '@noble/ed25519': + specifier: ^1.7.3 + version: 1.7.3 + '@polkadot/api': + specifier: ^10.9.1 + version: 10.9.1 + '@polkadot/api-augment': + specifier: ^10.9.1 + version: 10.9.1 + '@polkadot/api-base': + specifier: ^10.9.1 + version: 10.9.1 + '@polkadot/api-derive': + specifier: ^10.9.1 + version: 10.9.1 + '@polkadot/keyring': + specifier: ^12.2.1 + version: 12.5.1(@polkadot/util-crypto@12.5.1)(@polkadot/util@12.5.1) + '@polkadot/rpc-core': + specifier: ^10.9.1 + version: 10.9.1 + '@polkadot/types': + specifier: ^10.9.1 + version: 10.9.1 + '@polkadot/types-augment': + specifier: ^10.9.1 + version: 10.9.1 + '@polkadot/types-codec': + specifier: ^10.9.1 + version: 10.9.1 + '@polkadot/types-create': + specifier: ^10.9.1 + version: 10.9.1 + '@polkadot/types-known': + specifier: ^10.9.1 + version: 10.9.1 + '@polkadot/types-support': + specifier: ^10.9.1 + version: 10.9.1 '@polkadot/util': specifier: ^12.5.1 version: 12.5.1 + '@polkadot/util-crypto': + specifier: ^12.5.1 + version: 12.5.1(@polkadot/util@12.5.1) + add: + specifier: ^2.0.6 + version: 2.0.6 + ajv: + specifier: ^8.12.0 + version: 8.12.0 + bitcore-lib: + specifier: ^10.0.21 + version: 10.0.21 chai: specifier: ^4.3.6 version: 4.3.10 + colors: + specifier: ^1.4.0 + version: 1.4.0 + js-base64: + specifier: ^3.7.5 + version: 3.7.5 micro-base58: specifier: ^0.5.1 version: 0.5.1 @@ -261,16 +318,61 @@ importers: mocha-steps: specifier: ^1.3.0 version: 1.3.0 + scale-ts: + specifier: ^0.2.11 + version: 0.2.12 websocket-as-promised: specifier: ^2.0.1 version: 2.0.1 + ws: + specifier: ^8.8.1 + version: 8.14.2 devDependencies: + '@ethersproject/providers': + specifier: ^5.7.2 + version: 5.7.2 + '@types/bitcore-lib': + specifier: ^0.15.1 + version: 0.15.6 + '@types/chai': + specifier: ^4.3.3 + version: 4.3.6 + '@types/mocha': + specifier: ^10.0.0 + version: 10.0.2 + '@types/node': + specifier: ^20.4.4 + version: 20.7.1 + '@types/ws': + specifier: ^8.5.3 + version: 8.5.6 + '@typescript-eslint/eslint-plugin': + specifier: ^5.60.0 + version: 5.62.0(@typescript-eslint/parser@5.62.0)(eslint@8.50.0)(typescript@5.0.4) + '@typescript-eslint/parser': + specifier: ^5.60.0 + version: 5.62.0(eslint@8.50.0)(typescript@5.0.4) cross-env: specifier: ^7.0.3 version: 7.0.3 + dotenv: + specifier: ^16.0.3 + version: 16.3.1 + eslint: + specifier: ^8.43.0 + version: 8.50.0 + ethers: + specifier: ^5.7.2 + version: 5.7.2 + prettier: + specifier: 2.8.1 + version: 2.8.1 ts-node: specifier: ^10.9.1 version: 10.9.1(@types/node@20.7.1)(typescript@5.0.4) + typescript: + specifier: 5.0.4 + version: 5.0.4 packages: diff --git a/tee-worker/ts-tests/worker/.env.local.example b/tee-worker/ts-tests/worker/.env.local.example index 7c704c45fc..76543e7d79 100644 --- a/tee-worker/ts-tests/worker/.env.local.example +++ b/tee-worker/ts-tests/worker/.env.local.example @@ -1,2 +1,3 @@ NODE_ENV = local -NODE_ENDPOINT = "ws://localhost:9944" \ No newline at end of file +NODE_ENDPOINT = "ws://localhost:9944" +BINARY_DIR=/usr/local/bin \ No newline at end of file diff --git a/tee-worker/ts-tests/worker/package.json b/tee-worker/ts-tests/worker/package.json index a3848f62a2..2113d20611 100644 --- a/tee-worker/ts-tests/worker/package.json +++ b/tee-worker/ts-tests/worker/package.json @@ -14,11 +14,45 @@ "mocha": "^10.1.0", "mocha-steps": "^1.3.0", "@polkadot/util": "^12.5.1", - "websocket-as-promised": "^2.0.1" + "websocket-as-promised": "^2.0.1", + "@noble/ed25519": "^1.7.3", + "@polkadot/api": "^10.9.1", + "@polkadot/api-augment": "^10.9.1", + "@polkadot/api-base": "^10.9.1", + "@polkadot/api-derive": "^10.9.1", + "@polkadot/keyring": "^12.2.1", + "@polkadot/rpc-core": "^10.9.1", + "@polkadot/types": "^10.9.1", + "@polkadot/types-augment": "^10.9.1", + "@polkadot/types-codec": "^10.9.1", + "@polkadot/types-create": "^10.9.1", + "@polkadot/types-known": "^10.9.1", + "@polkadot/types-support": "^10.9.1", + "@polkadot/util-crypto": "^12.5.1", + "add": "^2.0.6", + "ajv": "^8.12.0", + "bitcore-lib": "^10.0.21", + "colors": "^1.4.0", + "js-base64": "^3.7.5", + "scale-ts": "^0.2.11", + "ws": "^8.8.1" }, "devDependencies": { + "@ethersproject/providers": "^5.7.2", + "@types/bitcore-lib": "^0.15.1", + "@types/chai": "^4.3.3", + "@types/mocha": "^10.0.0", + "@types/node": "^20.4.4", + "@types/ws": "^8.5.3", + "@typescript-eslint/eslint-plugin": "^5.60.0", + "@typescript-eslint/parser": "^5.60.0", "cross-env": "^7.0.3", - "ts-node": "^10.9.1" + "dotenv": "^16.0.3", + "eslint": "^8.43.0", + "ethers": "^5.7.2", + "prettier": "2.8.1", + "ts-node": "^10.9.1", + "typescript": "5.0.4" }, "packageManager": "pnpm@8.7.6" } From 33a9e8aaee459a3113cf3c8253139737ee8ce06a Mon Sep 17 00:00:00 2001 From: Kasper Ziemianek Date: Wed, 17 Jan 2024 11:37:01 +0100 Subject: [PATCH 15/19] debug --- .../ts-tests/worker/resuming_worker.test.ts | 348 +----------------- 1 file changed, 1 insertion(+), 347 deletions(-) diff --git a/tee-worker/ts-tests/worker/resuming_worker.test.ts b/tee-worker/ts-tests/worker/resuming_worker.test.ts index ecafccf6ad..9031697837 100644 --- a/tee-worker/ts-tests/worker/resuming_worker.test.ts +++ b/tee-worker/ts-tests/worker/resuming_worker.test.ts @@ -126,352 +126,6 @@ class SidechainDbLockUnavailable extends Error { } } -async function spawnWorkerJob( - command: Command, - { workingDir, nodeConfig, workerConfig }: JobConfig, - subprocessTracker: Set -): Promise<{ job: ChildProcess; shard: HexString | undefined }> { - const { name } = workerConfig; - const task = () => - new Promise<{ job: ChildProcess; shard: HexString | undefined }>((resolve, reject) => { - let shard: HexString | undefined = undefined; - - const job = spawn( - `./litentry-worker`, - [generateWorkerCommandArguments(command, nodeConfig, workerConfig)], - { - cwd: workingDir, - shell: '/bin/sh', - env: { - RUST_LOG: 'warn,sp_io::storage=error,substrate_api_client=warn', - ...process.env, - }, - detached: true, - } - ); - - job.on('error', (error) => reject(error)); - - const pid = job.pid; - if (!pid) { - return; - } - - subprocessTracker.add(pid); - job.on('exit', () => { - subprocessTracker.delete(pid); - }); - - job.on('close', (code) => { - console.log(`${name} close: ${code}`); - }); - - job.stderr.setEncoding('utf8'); - const errorStream = readline.createInterface(job.stderr); - errorStream.on('line', (line: string) => { - console.warn(name, line); - if (line.includes('sidechain_db/LOCK: Resource temporarily unavailable')) { - reject(new SidechainDbLockUnavailable()); - } - }); - - job.stdout.setEncoding('utf8'); - const outputStream = readline.createInterface(job.stdout); - outputStream.on('line', (line: string) => { - console.log(name, line); - - const match = line.match(/^Successfully initialized shard "(?[\w]{44}).*"/); - if (match !== null) { - /** - * Assertions needed because regex contents aren't reflected in function typing; - * see e.g. https://github.com/microsoft/TypeScript/issues/32098. - * - * If the regexp match succeeds, the `groups` property is guaranteed to be present, - * as well as the corresponding named capturing groups. See - * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/match - */ - const base58shard = match.groups!.shard; - shard = u8aToHex(base58.decode(base58shard)) as HexString; - return; - } - - if (line.includes('Untrusted RPC server is spawned on')) { - resolve({ job, shard }); - } - }); - }); - - return withRetry(task, { - isRetriable: (err) => err instanceof SidechainDbLockUnavailable, - maxRetries: 3, - initialDelaySeconds: 10, - backoffFactor: 1.5, - }); -} - -type WorkerState = { - shard: HexString; - job: ChildProcess; - connection: WebSocketAsPromised; - latestSeenBlock: number; -}; - -async function launchWorker( - binaryDir: string, - jobConfig: JobConfig, - subprocessTracker: Set -): Promise { - initializeFiles(jobConfig.workingDir, binaryDir); - - const { shard, job } = await spawnWorkerJob('launch', jobConfig, subprocessTracker); - - if (shard === undefined) { - throw new Error('RPC server spawned before shard initialization'); - } - - const connection = await initWorkerConnection(`ws://localhost:${jobConfig.workerConfig.untrustedWorkerPort}`); - const latestSeenBlock = await waitWorkerProducingBlock(connection, shard, 4); - - return { shard, job, connection, latestSeenBlock }; -} - -async function resumeWorker( - jobConfig: JobConfig, - connection: WebSocketAsPromised, - subprocessTracker: Set -): Promise { - const { shard, job } = await spawnWorkerJob('resume', jobConfig, subprocessTracker); - - if (shard !== undefined) { - throw new Error('Shard should have been reused from the previous run'); - } - - await connection.open(); - - return job; -} - -function killWorker(worker: ChildProcess): Promise { - const pid = worker.pid; - - if (pid === undefined) { - return Promise.reject(new Error('Attempted to kill an unspawned worker?')); - } - - return new Promise((resolve, reject) => { - /** - * Kill each process in the worker's group; - * see https://www.man7.org/linux/man-pages/man2/kill.2.html - */ - process.kill(-pid, 'SIGKILL'); - worker.on('exit', () => { - resolve(); - }); - worker.on('error', (error) => reject(error)); - }); -} - -async function latestBlock( - connection: WebSocketAsPromised, - shard: HexString -): Promise<{ result: undefined | { number: number; hash: string } }> { - return await connection.sendRequest( - { - jsonrpc: '2.0', - method: 'sidechain_latestBlock', - params: shard, - id: 1, - }, - { requestId: 1, timeout: 6000 } - ); -} - -async function waitForBlock(connection: WebSocketAsPromised, shard: HexString, lowerBound: number): Promise { - const task = async () => { - const resp = await latestBlock(connection, shard); - const blockNumber = resp.result?.number; - console.log(`${connection.ws.url} current block: ${blockNumber}`); - if (blockNumber != undefined && blockNumber >= lowerBound) { - return blockNumber; - } - throw new Error(`waiting for block ${lowerBound}; got ${blockNumber} instead`); - }; - - return withRetry(task, { isRetriable: () => true, maxRetries: 10, initialDelaySeconds: 5, backoffFactor: 1.5 }); -} - -async function waitWorkerProducingBlock( - connection: WebSocketAsPromised, - shard: HexString, - atLeast: number -): Promise { - const currentBlockNumber = await waitForBlock(connection, shard, 0); - return await waitForBlock(connection, shard, currentBlockNumber + atLeast); -} - -function getConfiguration(): { binaryDir: string; nodeUrl: string; nodePort: number } { - const binaryDir = process.env.BINARY_DIR; - if (binaryDir === undefined) { - throw new Error('Environment variable BINARY_DIR not defined'); - } - - const [, nodeHost, nodePortRaw] = process.env.NODE_ENDPOINT!.split(':'); - const nodePort = Number.parseInt(nodePortRaw); - if (nodeHost === undefined || Number.isNaN(nodePort)) { - throw new Error('Environment variable NODE_ENDPOINT undefined or malformed'); - } - - return { binaryDir, nodeUrl: `ws:${nodeHost}`, nodePort }; -} - describe('Resume worker', function () { - this.timeout(6000000); - - const { binaryDir, nodeUrl, nodePort } = getConfiguration(); - const nodeConfig = { nodeUrl, nodePort }; - - const tempDir = os.tmpdir(); - const worker0Dir = fs.mkdtempSync(path.join(tempDir, 'worker0-')); - const worker1Dir = fs.mkdtempSync(path.join(tempDir, 'worker1-')); - - const jobConfig: Record<'worker0' | 'worker1', JobConfig> = { - worker0: { - workingDir: worker0Dir, - nodeConfig, - workerConfig: workerConfig.worker0, - }, - worker1: { - workingDir: worker1Dir, - nodeConfig, - workerConfig: workerConfig.worker1, - }, - }; - - const subprocessTracker: Set = new Set(); - - after(() => { - fs.rmSync(worker0Dir, { recursive: true, force: true }); - fs.rmSync(worker1Dir, { recursive: true, force: true }); - subprocessTracker.forEach((pid) => { - if (pid !== undefined) { - try { - process.kill(-pid, 'SIGTERM'); - } catch (error) { - if ((error as { code: unknown }).code === 'ESRCH') { - return; // Process has already died; nothing to do - } - console.warn(error); - } - } - }); - }); - - let worker0State: WorkerState | undefined = undefined; - - step('One worker', async function () { - // first launch worker0 - worker0State = await launchWorker(binaryDir, jobConfig.worker0, subprocessTracker); - console.log('=========== worker0 launched and produced blocks =================='); - - // kill worker0 - await killWorker(worker0State.job); - console.log('=========== worker0 stopped =================='); - - // resume worker0 - worker0State.job = await resumeWorker(jobConfig.worker0, worker0State.connection, subprocessTracker); - console.log('=========== worker0 resumed =================='); - - // check block production - worker0State.latestSeenBlock = await waitForBlock( - worker0State.connection, - worker0State.shard, - worker0State.latestSeenBlock + 1 - ); - console.log('=========== worker0 produced blocks =================='); - }); - - let worker1State: WorkerState | undefined = undefined; - - step('Two workers & resume worker1', async function () { - assert(worker0State); - - // first launch worker1 - worker1State = await launchWorker(binaryDir, jobConfig.worker1, subprocessTracker); - console.log('=========== worker1 launched and produced blocks =================='); - - // kill worker1 - await killWorker(worker1State.job); - console.log('=========== worker1 stopped =================='); - - // let worker0 produce - worker0State.latestSeenBlock = await waitForBlock( - worker0State.connection, - worker0State.shard, - worker0State.latestSeenBlock + 1 - ); - console.log('=========== worker0 still produces blocks =================='); - - // resume worker1 - worker1State.job = await resumeWorker(jobConfig.worker1, worker1State.connection, subprocessTracker); - console.log('=========== worker1 resumed =================='); - - // check block production - worker1State.latestSeenBlock = await waitForBlock( - worker1State.connection, - worker1State.shard, - worker1State.latestSeenBlock + 1 - ); - console.log('=========== worker1 produced blocks =================='); - }); - - step('Kill and resume both workers', async function () { - assert(worker0State); - assert(worker1State); - - // kill both workers - await killWorker(worker0State.job); - console.log('=========== worker0 stopped =================='); - await killWorker(worker1State.job); - console.log('=========== worker1 stopped =================='); - - // resume and check worker1 - worker1State.job = await resumeWorker(jobConfig.worker1, worker1State.connection, subprocessTracker); - console.log('=========== worker1 resumed =================='); - worker1State.latestSeenBlock = await waitForBlock( - worker1State.connection, - worker1State.shard, - worker1State.latestSeenBlock + 1 - ); - console.log('=========== worker1 produced blocks =================='); - - // resume and check worker0 - worker0State.job = await resumeWorker(jobConfig.worker0, worker0State.connection, subprocessTracker); - console.log('=========== worker0 resumed =================='); - worker0State.latestSeenBlock = await waitForBlock( - worker0State.connection, - worker0State.shard, - worker0State.latestSeenBlock + 1 - ); - console.log('=========== worker0 produced blocks =================='); - - // check worker1 health - worker1State.latestSeenBlock = await waitForBlock( - worker1State.connection, - worker1State.shard, - worker1State.latestSeenBlock + 1 - ); - console.log('=========== worker1 still produces blocks =================='); - }); - - step('Tidy up', async function () { - // kill both workers - if (worker0State) { - await killWorker(worker0State.job); - console.log('=========== worker0 stopped =================='); - } - if (worker1State) { - await killWorker(worker1State.job); - console.log('=========== worker1 stopped =================='); - } - }); + console.log('works'); }); From 0c1befd00feede5af270861f3b873cdbbf75d5cb Mon Sep 17 00:00:00 2001 From: Kasper Ziemianek Date: Wed, 17 Jan 2024 14:56:07 +0100 Subject: [PATCH 16/19] remove problematic function --- .../ts-tests/worker/resuming_worker.test.ts | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/tee-worker/ts-tests/worker/resuming_worker.test.ts b/tee-worker/ts-tests/worker/resuming_worker.test.ts index 9031697837..3c59569346 100644 --- a/tee-worker/ts-tests/worker/resuming_worker.test.ts +++ b/tee-worker/ts-tests/worker/resuming_worker.test.ts @@ -93,27 +93,6 @@ type RetryConfig = { backoffFactor: number; }; -async function withRetry( - task: () => Promise, - { isRetriable, maxRetries, initialDelaySeconds, backoffFactor }: RetryConfig -): Promise { - let attempt = 0; - let delaySeconds = initialDelaySeconds; - // eslint-disable-next-line no-constant-condition - while (true) { - try { - return await task(); - } catch (err) { - if (attempt > maxRetries || !isRetriable(err)) { - throw err; - } - attempt += 1; - await sleep(delaySeconds); - delaySeconds *= backoffFactor; - } - } -} - type JobConfig = { workerConfig: WorkerConfig; nodeConfig: NodeConfig; From 44abd6c5c116f6ab057f4fab624f7d9e1bb444e3 Mon Sep 17 00:00:00 2001 From: Kasper Ziemianek Date: Wed, 17 Jan 2024 16:10:49 +0100 Subject: [PATCH 17/19] debugging --- tee-worker/ts-tests/worker/resuming_worker.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tee-worker/ts-tests/worker/resuming_worker.test.ts b/tee-worker/ts-tests/worker/resuming_worker.test.ts index 3c59569346..1b65dd962c 100644 --- a/tee-worker/ts-tests/worker/resuming_worker.test.ts +++ b/tee-worker/ts-tests/worker/resuming_worker.test.ts @@ -12,6 +12,10 @@ import type { HexString } from '@polkadot/util/types'; import * as base58 from 'micro-base58'; import { u8aToHex } from '@polkadot/util'; +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore +console.log(withRetry); + type WorkerConfig = { name: string; muRaPort: number; From 33cc7385a9ec9223903a578a97b1fb3fa4ece4ad Mon Sep 17 00:00:00 2001 From: Kasper Ziemianek Date: Wed, 17 Jan 2024 17:58:53 +0100 Subject: [PATCH 18/19] add withRetry --- tee-worker/ts-tests/pnpm-lock.yaml | 262 ++++++++++++++++++ tee-worker/ts-tests/worker/.mocharc.json | 4 + tee-worker/ts-tests/worker/package.json | 7 +- .../ts-tests/worker/resuming_worker.test.ts | 119 ++------ 4 files changed, 289 insertions(+), 103 deletions(-) create mode 100644 tee-worker/ts-tests/worker/.mocharc.json diff --git a/tee-worker/ts-tests/pnpm-lock.yaml b/tee-worker/ts-tests/pnpm-lock.yaml index da37548f54..e79699fb34 100644 --- a/tee-worker/ts-tests/pnpm-lock.yaml +++ b/tee-worker/ts-tests/pnpm-lock.yaml @@ -370,6 +370,9 @@ importers: ts-node: specifier: ^10.9.1 version: 10.9.1(@types/node@20.7.1)(typescript@5.0.4) + tsx: + specifier: 4.7.0 + version: 4.7.0 typescript: specifier: 5.0.4 version: 5.0.4 @@ -388,6 +391,213 @@ packages: '@jridgewell/trace-mapping': 0.3.9 dev: true + /@esbuild/aix-ppc64@0.19.11: + resolution: {integrity: sha512-FnzU0LyE3ySQk7UntJO4+qIiQgI7KoODnZg5xzXIrFJlKd2P2gwHsHY4927xj9y5PJmJSzULiUCWmv7iWnNa7g==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [aix] + requiresBuild: true + dev: true + optional: true + + /@esbuild/android-arm64@0.19.11: + resolution: {integrity: sha512-aiu7K/5JnLj//KOnOfEZ0D90obUkRzDMyqd/wNAUQ34m4YUPVhRZpnqKV9uqDGxT7cToSDnIHsGooyIczu9T+Q==} + engines: {node: '>=12'} + cpu: [arm64] + os: [android] + requiresBuild: true + dev: true + optional: true + + /@esbuild/android-arm@0.19.11: + resolution: {integrity: sha512-5OVapq0ClabvKvQ58Bws8+wkLCV+Rxg7tUVbo9xu034Nm536QTII4YzhaFriQ7rMrorfnFKUsArD2lqKbFY4vw==} + engines: {node: '>=12'} + cpu: [arm] + os: [android] + requiresBuild: true + dev: true + optional: true + + /@esbuild/android-x64@0.19.11: + resolution: {integrity: sha512-eccxjlfGw43WYoY9QgB82SgGgDbibcqyDTlk3l3C0jOVHKxrjdc9CTwDUQd0vkvYg5um0OH+GpxYvp39r+IPOg==} + engines: {node: '>=12'} + cpu: [x64] + os: [android] + requiresBuild: true + dev: true + optional: true + + /@esbuild/darwin-arm64@0.19.11: + resolution: {integrity: sha512-ETp87DRWuSt9KdDVkqSoKoLFHYTrkyz2+65fj9nfXsaV3bMhTCjtQfw3y+um88vGRKRiF7erPrh/ZuIdLUIVxQ==} + engines: {node: '>=12'} + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /@esbuild/darwin-x64@0.19.11: + resolution: {integrity: sha512-fkFUiS6IUK9WYUO/+22omwetaSNl5/A8giXvQlcinLIjVkxwTLSktbF5f/kJMftM2MJp9+fXqZ5ezS7+SALp4g==} + engines: {node: '>=12'} + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + + /@esbuild/freebsd-arm64@0.19.11: + resolution: {integrity: sha512-lhoSp5K6bxKRNdXUtHoNc5HhbXVCS8V0iZmDvyWvYq9S5WSfTIHU2UGjcGt7UeS6iEYp9eeymIl5mJBn0yiuxA==} + engines: {node: '>=12'} + cpu: [arm64] + os: [freebsd] + requiresBuild: true + dev: true + optional: true + + /@esbuild/freebsd-x64@0.19.11: + resolution: {integrity: sha512-JkUqn44AffGXitVI6/AbQdoYAq0TEullFdqcMY/PCUZ36xJ9ZJRtQabzMA+Vi7r78+25ZIBosLTOKnUXBSi1Kw==} + engines: {node: '>=12'} + cpu: [x64] + os: [freebsd] + requiresBuild: true + dev: true + optional: true + + /@esbuild/linux-arm64@0.19.11: + resolution: {integrity: sha512-LneLg3ypEeveBSMuoa0kwMpCGmpu8XQUh+mL8XXwoYZ6Be2qBnVtcDI5azSvh7vioMDhoJFZzp9GWp9IWpYoUg==} + engines: {node: '>=12'} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@esbuild/linux-arm@0.19.11: + resolution: {integrity: sha512-3CRkr9+vCV2XJbjwgzjPtO8T0SZUmRZla+UL1jw+XqHZPkPgZiyWvbDvl9rqAN8Zl7qJF0O/9ycMtjU67HN9/Q==} + engines: {node: '>=12'} + cpu: [arm] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@esbuild/linux-ia32@0.19.11: + resolution: {integrity: sha512-caHy++CsD8Bgq2V5CodbJjFPEiDPq8JJmBdeyZ8GWVQMjRD0sU548nNdwPNvKjVpamYYVL40AORekgfIubwHoA==} + engines: {node: '>=12'} + cpu: [ia32] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@esbuild/linux-loong64@0.19.11: + resolution: {integrity: sha512-ppZSSLVpPrwHccvC6nQVZaSHlFsvCQyjnvirnVjbKSHuE5N24Yl8F3UwYUUR1UEPaFObGD2tSvVKbvR+uT1Nrg==} + engines: {node: '>=12'} + cpu: [loong64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@esbuild/linux-mips64el@0.19.11: + resolution: {integrity: sha512-B5x9j0OgjG+v1dF2DkH34lr+7Gmv0kzX6/V0afF41FkPMMqaQ77pH7CrhWeR22aEeHKaeZVtZ6yFwlxOKPVFyg==} + engines: {node: '>=12'} + cpu: [mips64el] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@esbuild/linux-ppc64@0.19.11: + resolution: {integrity: sha512-MHrZYLeCG8vXblMetWyttkdVRjQlQUb/oMgBNurVEnhj4YWOr4G5lmBfZjHYQHHN0g6yDmCAQRR8MUHldvvRDA==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@esbuild/linux-riscv64@0.19.11: + resolution: {integrity: sha512-f3DY++t94uVg141dozDu4CCUkYW+09rWtaWfnb3bqe4w5NqmZd6nPVBm+qbz7WaHZCoqXqHz5p6CM6qv3qnSSQ==} + engines: {node: '>=12'} + cpu: [riscv64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@esbuild/linux-s390x@0.19.11: + resolution: {integrity: sha512-A5xdUoyWJHMMlcSMcPGVLzYzpcY8QP1RtYzX5/bS4dvjBGVxdhuiYyFwp7z74ocV7WDc0n1harxmpq2ePOjI0Q==} + engines: {node: '>=12'} + cpu: [s390x] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@esbuild/linux-x64@0.19.11: + resolution: {integrity: sha512-grbyMlVCvJSfxFQUndw5mCtWs5LO1gUlwP4CDi4iJBbVpZcqLVT29FxgGuBJGSzyOxotFG4LoO5X+M1350zmPA==} + engines: {node: '>=12'} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: true + optional: true + + /@esbuild/netbsd-x64@0.19.11: + resolution: {integrity: sha512-13jvrQZJc3P230OhU8xgwUnDeuC/9egsjTkXN49b3GcS5BKvJqZn86aGM8W9pd14Kd+u7HuFBMVtrNGhh6fHEQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [netbsd] + requiresBuild: true + dev: true + optional: true + + /@esbuild/openbsd-x64@0.19.11: + resolution: {integrity: sha512-ysyOGZuTp6SNKPE11INDUeFVVQFrhcNDVUgSQVDzqsqX38DjhPEPATpid04LCoUr2WXhQTEZ8ct/EgJCUDpyNw==} + engines: {node: '>=12'} + cpu: [x64] + os: [openbsd] + requiresBuild: true + dev: true + optional: true + + /@esbuild/sunos-x64@0.19.11: + resolution: {integrity: sha512-Hf+Sad9nVwvtxy4DXCZQqLpgmRTQqyFyhT3bZ4F2XlJCjxGmRFF0Shwn9rzhOYRB61w9VMXUkxlBy56dk9JJiQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [sunos] + requiresBuild: true + dev: true + optional: true + + /@esbuild/win32-arm64@0.19.11: + resolution: {integrity: sha512-0P58Sbi0LctOMOQbpEOvOL44Ne0sqbS0XWHMvvrg6NE5jQ1xguCSSw9jQeUk2lfrXYsKDdOe6K+oZiwKPilYPQ==} + engines: {node: '>=12'} + cpu: [arm64] + os: [win32] + requiresBuild: true + dev: true + optional: true + + /@esbuild/win32-ia32@0.19.11: + resolution: {integrity: sha512-6YOrWS+sDJDmshdBIQU+Uoyh7pQKrdykdefC1avn76ss5c+RN6gut3LZA4E2cH5xUEp5/cA0+YxRaVtRAb0xBg==} + engines: {node: '>=12'} + cpu: [ia32] + os: [win32] + requiresBuild: true + dev: true + optional: true + + /@esbuild/win32-x64@0.19.11: + resolution: {integrity: sha512-vfkhltrjCAb603XaFhqhAF4LGDi2M4OrCRrFusyQ+iTLQ/o60QQXxc9cZC/FFpihBI9N1Grn6SMKVJ4KP7Fuiw==} + engines: {node: '>=12'} + cpu: [x64] + os: [win32] + requiresBuild: true + dev: true + optional: true + /@eslint-community/eslint-utils@4.4.0(eslint@8.50.0): resolution: {integrity: sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -1909,6 +2119,37 @@ packages: is-symbol: 1.0.4 dev: false + /esbuild@0.19.11: + resolution: {integrity: sha512-HJ96Hev2hX/6i5cDVwcqiJBBtuo9+FeIJOtZ9W1kA5M6AMJRHUZlpYZ1/SbEwtO0ioNAW8rUooVpC/WehY2SfA==} + engines: {node: '>=12'} + hasBin: true + requiresBuild: true + optionalDependencies: + '@esbuild/aix-ppc64': 0.19.11 + '@esbuild/android-arm': 0.19.11 + '@esbuild/android-arm64': 0.19.11 + '@esbuild/android-x64': 0.19.11 + '@esbuild/darwin-arm64': 0.19.11 + '@esbuild/darwin-x64': 0.19.11 + '@esbuild/freebsd-arm64': 0.19.11 + '@esbuild/freebsd-x64': 0.19.11 + '@esbuild/linux-arm': 0.19.11 + '@esbuild/linux-arm64': 0.19.11 + '@esbuild/linux-ia32': 0.19.11 + '@esbuild/linux-loong64': 0.19.11 + '@esbuild/linux-mips64el': 0.19.11 + '@esbuild/linux-ppc64': 0.19.11 + '@esbuild/linux-riscv64': 0.19.11 + '@esbuild/linux-s390x': 0.19.11 + '@esbuild/linux-x64': 0.19.11 + '@esbuild/netbsd-x64': 0.19.11 + '@esbuild/openbsd-x64': 0.19.11 + '@esbuild/sunos-x64': 0.19.11 + '@esbuild/win32-arm64': 0.19.11 + '@esbuild/win32-ia32': 0.19.11 + '@esbuild/win32-x64': 0.19.11 + dev: true + /escalade@3.1.1: resolution: {integrity: sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==} engines: {node: '>=6'} @@ -2208,6 +2449,12 @@ packages: get-intrinsic: 1.2.1 dev: false + /get-tsconfig@4.7.2: + resolution: {integrity: sha512-wuMsz4leaj5hbGgg4IvDU0bqJagpftG5l5cXIAvo8uZrqn0NJqwtfupTN00VnkQJPcIRrxYrm1Ue24btpCha2A==} + dependencies: + resolve-pkg-maps: 1.0.0 + dev: true + /glob-parent@5.1.2: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} engines: {node: '>= 6'} @@ -2899,6 +3146,10 @@ packages: engines: {node: '>=4'} dev: true + /resolve-pkg-maps@1.0.0: + resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} + dev: true + /reusify@1.0.4: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} @@ -3130,6 +3381,17 @@ packages: typescript: 5.0.4 dev: true + /tsx@4.7.0: + resolution: {integrity: sha512-I+t79RYPlEYlHn9a+KzwrvEwhJg35h/1zHsLC2JXvhC2mdynMv6Zxzvhv5EMV6VF5qJlLlkSnMVvdZV3PSIGcg==} + engines: {node: '>=18.0.0'} + hasBin: true + dependencies: + esbuild: 0.19.11 + get-tsconfig: 4.7.2 + optionalDependencies: + fsevents: 2.3.3 + dev: true + /type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} diff --git a/tee-worker/ts-tests/worker/.mocharc.json b/tee-worker/ts-tests/worker/.mocharc.json new file mode 100644 index 0000000000..f1e313de38 --- /dev/null +++ b/tee-worker/ts-tests/worker/.mocharc.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://json.schemastore.org/mocharc.json", + "require": "tsx" +} diff --git a/tee-worker/ts-tests/worker/package.json b/tee-worker/ts-tests/worker/package.json index 2113d20611..a9b32ae97a 100644 --- a/tee-worker/ts-tests/worker/package.json +++ b/tee-worker/ts-tests/worker/package.json @@ -5,8 +5,8 @@ "scripts": { "format": "pnpm exec prettier --write .", "check-format": "pnpm exec prettier --check .", - "test-resuming-worker:staging": "pnpm exec eslint . && pnpm exec cross-env NODE_ENV=staging mocha --trace-warnings --full-trace --exit --sort -r ts-node/register --loader=ts-node/esm 'resuming_worker.test.ts'", - "test-resuming-worker:local": "pnpm exec eslint . && pnpm exec cross-env NODE_ENV=local mocha --exit --sort -r ts-node/register --loader=ts-node/esm 'resuming_worker.test.ts'" + "test-resuming-worker:staging": "pnpm exec eslint . && pnpm exec cross-env NODE_ENV=staging mocha resuming_worker.test.ts", + "test-resuming-worker:local": "pnpm exec eslint . && pnpm exec cross-env NODE_ENV=local mocha resuming_worker.test.ts" }, "dependencies": { "chai": "^4.3.6", @@ -52,7 +52,8 @@ "ethers": "^5.7.2", "prettier": "2.8.1", "ts-node": "^10.9.1", - "typescript": "5.0.4" + "typescript": "5.0.4", + "tsx": "4.7.0" }, "packageManager": "pnpm@8.7.6" } diff --git a/tee-worker/ts-tests/worker/resuming_worker.test.ts b/tee-worker/ts-tests/worker/resuming_worker.test.ts index 1b65dd962c..96e4ee03b7 100644 --- a/tee-worker/ts-tests/worker/resuming_worker.test.ts +++ b/tee-worker/ts-tests/worker/resuming_worker.test.ts @@ -1,94 +1,4 @@ -import { ChildProcess, spawn } from 'child_process'; -import * as readline from 'readline'; -import fs from 'fs'; -import * as path from 'path'; -import * as process from 'process'; -import { step } from 'mocha-steps'; -import WebSocketAsPromised from 'websocket-as-promised'; -import os from 'os'; -import { initWorkerConnection, sleep } from './../integration-tests/common/utils'; -import { assert } from 'chai'; -import type { HexString } from '@polkadot/util/types'; -import * as base58 from 'micro-base58'; -import { u8aToHex } from '@polkadot/util'; - -// eslint-disable-next-line @typescript-eslint/ban-ts-comment -// @ts-ignore -console.log(withRetry); - -type WorkerConfig = { - name: string; - muRaPort: number; - untrustedHttpPort: number; - trustedWorkerPort: number; - untrustedWorkerPort: number; - enableMockServer: boolean; - requestStateOnLaunch: boolean; -}; - -const workerConfig: Record<'worker0' | 'worker1', WorkerConfig> = { - worker0: { - name: 'worker0', - muRaPort: 3443, - untrustedHttpPort: 4545, - trustedWorkerPort: 2000, - untrustedWorkerPort: 3000, - enableMockServer: true, - requestStateOnLaunch: false, - }, - worker1: { - name: 'worker1', - muRaPort: 3444, - untrustedHttpPort: 4546, - trustedWorkerPort: 2001, - untrustedWorkerPort: 3001, - enableMockServer: false, - requestStateOnLaunch: true, - }, -} as const; - -type NodeConfig = { - nodeUrl: string; - nodePort: number; -}; - -type Command = 'launch' | 'resume'; - -function generateWorkerCommandArguments( - command: Command, - { nodeUrl, nodePort }: NodeConfig, - workerParams: WorkerConfig -): string { - const isLaunch = command === 'launch'; - - return [ - '--running-mode mock', - ...(workerParams.enableMockServer ? ['--enable-mock-server'] : []), - ...(isLaunch ? ['--clean-reset'] : []), - '--mu-ra-external-address localhost', - `--mu-ra-port ${workerParams.muRaPort}`, - `--untrusted-http-port ${workerParams.untrustedHttpPort}`, - '--ws-external', - '--trusted-external-address wss://localhost', - `--trusted-worker-port ${workerParams.trustedWorkerPort}`, - '--untrusted-external-address ws://localhost', - `--untrusted-worker-port ${workerParams.untrustedWorkerPort}`, - `--node-url ${nodeUrl}`, - `--node-port ${nodePort}`, - `run`, - `--skip-ra`, - `--dev`, - ...(isLaunch && workerParams.requestStateOnLaunch ? ['--request-state'] : []), - ].join(' '); -} - -function initializeFiles(workingDir: string, binaryDir: string) { - fs.mkdirSync(workingDir, { recursive: true }); - fs.copyFileSync(path.join(binaryDir, 'enclave.signed.so'), path.join(workingDir, 'enclave.signed.so')); - fs.copyFileSync(path.join(binaryDir, 'litentry-worker'), path.join(workingDir, 'litentry-worker')); - fs.closeSync(fs.openSync(path.join(workingDir, 'spid.txt'), 'w')); - fs.closeSync(fs.openSync(path.join(workingDir, 'key.txt'), 'w')); -} +import { sleep } from './../integration-tests/common/utils'; type RetryConfig = { isRetriable: (err: unknown) => boolean; @@ -97,15 +7,24 @@ type RetryConfig = { backoffFactor: number; }; -type JobConfig = { - workerConfig: WorkerConfig; - nodeConfig: NodeConfig; - workingDir: string; -}; - -class SidechainDbLockUnavailable extends Error { - constructor() { - super('sidechain_db lock unavailable'); +async function withRetry( + task: () => Promise, + { isRetriable, maxRetries, initialDelaySeconds, backoffFactor }: RetryConfig +): Promise { + let attempt = 0; + let delaySeconds = initialDelaySeconds; + // eslint-disable-next-line no-constant-condition + while (true) { + try { + return await task(); + } catch (err) { + if (attempt > maxRetries || !isRetriable(err)) { + throw err; + } + attempt += 1; + await sleep(delaySeconds); + delaySeconds *= backoffFactor; + } } } From d005dc59ffd31f84747d160b64806e1c591218fa Mon Sep 17 00:00:00 2001 From: Kasper Ziemianek Date: Thu, 18 Jan 2024 14:48:16 +0100 Subject: [PATCH 19/19] remove sleep --- tee-worker/ts-tests/pnpm-lock.yaml | 262 ---------- tee-worker/ts-tests/worker/.mocharc.json | 4 - tee-worker/ts-tests/worker/package.json | 7 +- .../ts-tests/worker/resuming_worker.test.ts | 472 +++++++++++++++++- 4 files changed, 473 insertions(+), 272 deletions(-) delete mode 100644 tee-worker/ts-tests/worker/.mocharc.json diff --git a/tee-worker/ts-tests/pnpm-lock.yaml b/tee-worker/ts-tests/pnpm-lock.yaml index e79699fb34..da37548f54 100644 --- a/tee-worker/ts-tests/pnpm-lock.yaml +++ b/tee-worker/ts-tests/pnpm-lock.yaml @@ -370,9 +370,6 @@ importers: ts-node: specifier: ^10.9.1 version: 10.9.1(@types/node@20.7.1)(typescript@5.0.4) - tsx: - specifier: 4.7.0 - version: 4.7.0 typescript: specifier: 5.0.4 version: 5.0.4 @@ -391,213 +388,6 @@ packages: '@jridgewell/trace-mapping': 0.3.9 dev: true - /@esbuild/aix-ppc64@0.19.11: - resolution: {integrity: sha512-FnzU0LyE3ySQk7UntJO4+qIiQgI7KoODnZg5xzXIrFJlKd2P2gwHsHY4927xj9y5PJmJSzULiUCWmv7iWnNa7g==} - engines: {node: '>=12'} - cpu: [ppc64] - os: [aix] - requiresBuild: true - dev: true - optional: true - - /@esbuild/android-arm64@0.19.11: - resolution: {integrity: sha512-aiu7K/5JnLj//KOnOfEZ0D90obUkRzDMyqd/wNAUQ34m4YUPVhRZpnqKV9uqDGxT7cToSDnIHsGooyIczu9T+Q==} - engines: {node: '>=12'} - cpu: [arm64] - os: [android] - requiresBuild: true - dev: true - optional: true - - /@esbuild/android-arm@0.19.11: - resolution: {integrity: sha512-5OVapq0ClabvKvQ58Bws8+wkLCV+Rxg7tUVbo9xu034Nm536QTII4YzhaFriQ7rMrorfnFKUsArD2lqKbFY4vw==} - engines: {node: '>=12'} - cpu: [arm] - os: [android] - requiresBuild: true - dev: true - optional: true - - /@esbuild/android-x64@0.19.11: - resolution: {integrity: sha512-eccxjlfGw43WYoY9QgB82SgGgDbibcqyDTlk3l3C0jOVHKxrjdc9CTwDUQd0vkvYg5um0OH+GpxYvp39r+IPOg==} - engines: {node: '>=12'} - cpu: [x64] - os: [android] - requiresBuild: true - dev: true - optional: true - - /@esbuild/darwin-arm64@0.19.11: - resolution: {integrity: sha512-ETp87DRWuSt9KdDVkqSoKoLFHYTrkyz2+65fj9nfXsaV3bMhTCjtQfw3y+um88vGRKRiF7erPrh/ZuIdLUIVxQ==} - engines: {node: '>=12'} - cpu: [arm64] - os: [darwin] - requiresBuild: true - dev: true - optional: true - - /@esbuild/darwin-x64@0.19.11: - resolution: {integrity: sha512-fkFUiS6IUK9WYUO/+22omwetaSNl5/A8giXvQlcinLIjVkxwTLSktbF5f/kJMftM2MJp9+fXqZ5ezS7+SALp4g==} - engines: {node: '>=12'} - cpu: [x64] - os: [darwin] - requiresBuild: true - dev: true - optional: true - - /@esbuild/freebsd-arm64@0.19.11: - resolution: {integrity: sha512-lhoSp5K6bxKRNdXUtHoNc5HhbXVCS8V0iZmDvyWvYq9S5WSfTIHU2UGjcGt7UeS6iEYp9eeymIl5mJBn0yiuxA==} - engines: {node: '>=12'} - cpu: [arm64] - os: [freebsd] - requiresBuild: true - dev: true - optional: true - - /@esbuild/freebsd-x64@0.19.11: - resolution: {integrity: sha512-JkUqn44AffGXitVI6/AbQdoYAq0TEullFdqcMY/PCUZ36xJ9ZJRtQabzMA+Vi7r78+25ZIBosLTOKnUXBSi1Kw==} - engines: {node: '>=12'} - cpu: [x64] - os: [freebsd] - requiresBuild: true - dev: true - optional: true - - /@esbuild/linux-arm64@0.19.11: - resolution: {integrity: sha512-LneLg3ypEeveBSMuoa0kwMpCGmpu8XQUh+mL8XXwoYZ6Be2qBnVtcDI5azSvh7vioMDhoJFZzp9GWp9IWpYoUg==} - engines: {node: '>=12'} - cpu: [arm64] - os: [linux] - requiresBuild: true - dev: true - optional: true - - /@esbuild/linux-arm@0.19.11: - resolution: {integrity: sha512-3CRkr9+vCV2XJbjwgzjPtO8T0SZUmRZla+UL1jw+XqHZPkPgZiyWvbDvl9rqAN8Zl7qJF0O/9ycMtjU67HN9/Q==} - engines: {node: '>=12'} - cpu: [arm] - os: [linux] - requiresBuild: true - dev: true - optional: true - - /@esbuild/linux-ia32@0.19.11: - resolution: {integrity: sha512-caHy++CsD8Bgq2V5CodbJjFPEiDPq8JJmBdeyZ8GWVQMjRD0sU548nNdwPNvKjVpamYYVL40AORekgfIubwHoA==} - engines: {node: '>=12'} - cpu: [ia32] - os: [linux] - requiresBuild: true - dev: true - optional: true - - /@esbuild/linux-loong64@0.19.11: - resolution: {integrity: sha512-ppZSSLVpPrwHccvC6nQVZaSHlFsvCQyjnvirnVjbKSHuE5N24Yl8F3UwYUUR1UEPaFObGD2tSvVKbvR+uT1Nrg==} - engines: {node: '>=12'} - cpu: [loong64] - os: [linux] - requiresBuild: true - dev: true - optional: true - - /@esbuild/linux-mips64el@0.19.11: - resolution: {integrity: sha512-B5x9j0OgjG+v1dF2DkH34lr+7Gmv0kzX6/V0afF41FkPMMqaQ77pH7CrhWeR22aEeHKaeZVtZ6yFwlxOKPVFyg==} - engines: {node: '>=12'} - cpu: [mips64el] - os: [linux] - requiresBuild: true - dev: true - optional: true - - /@esbuild/linux-ppc64@0.19.11: - resolution: {integrity: sha512-MHrZYLeCG8vXblMetWyttkdVRjQlQUb/oMgBNurVEnhj4YWOr4G5lmBfZjHYQHHN0g6yDmCAQRR8MUHldvvRDA==} - engines: {node: '>=12'} - cpu: [ppc64] - os: [linux] - requiresBuild: true - dev: true - optional: true - - /@esbuild/linux-riscv64@0.19.11: - resolution: {integrity: sha512-f3DY++t94uVg141dozDu4CCUkYW+09rWtaWfnb3bqe4w5NqmZd6nPVBm+qbz7WaHZCoqXqHz5p6CM6qv3qnSSQ==} - engines: {node: '>=12'} - cpu: [riscv64] - os: [linux] - requiresBuild: true - dev: true - optional: true - - /@esbuild/linux-s390x@0.19.11: - resolution: {integrity: sha512-A5xdUoyWJHMMlcSMcPGVLzYzpcY8QP1RtYzX5/bS4dvjBGVxdhuiYyFwp7z74ocV7WDc0n1harxmpq2ePOjI0Q==} - engines: {node: '>=12'} - cpu: [s390x] - os: [linux] - requiresBuild: true - dev: true - optional: true - - /@esbuild/linux-x64@0.19.11: - resolution: {integrity: sha512-grbyMlVCvJSfxFQUndw5mCtWs5LO1gUlwP4CDi4iJBbVpZcqLVT29FxgGuBJGSzyOxotFG4LoO5X+M1350zmPA==} - engines: {node: '>=12'} - cpu: [x64] - os: [linux] - requiresBuild: true - dev: true - optional: true - - /@esbuild/netbsd-x64@0.19.11: - resolution: {integrity: sha512-13jvrQZJc3P230OhU8xgwUnDeuC/9egsjTkXN49b3GcS5BKvJqZn86aGM8W9pd14Kd+u7HuFBMVtrNGhh6fHEQ==} - engines: {node: '>=12'} - cpu: [x64] - os: [netbsd] - requiresBuild: true - dev: true - optional: true - - /@esbuild/openbsd-x64@0.19.11: - resolution: {integrity: sha512-ysyOGZuTp6SNKPE11INDUeFVVQFrhcNDVUgSQVDzqsqX38DjhPEPATpid04LCoUr2WXhQTEZ8ct/EgJCUDpyNw==} - engines: {node: '>=12'} - cpu: [x64] - os: [openbsd] - requiresBuild: true - dev: true - optional: true - - /@esbuild/sunos-x64@0.19.11: - resolution: {integrity: sha512-Hf+Sad9nVwvtxy4DXCZQqLpgmRTQqyFyhT3bZ4F2XlJCjxGmRFF0Shwn9rzhOYRB61w9VMXUkxlBy56dk9JJiQ==} - engines: {node: '>=12'} - cpu: [x64] - os: [sunos] - requiresBuild: true - dev: true - optional: true - - /@esbuild/win32-arm64@0.19.11: - resolution: {integrity: sha512-0P58Sbi0LctOMOQbpEOvOL44Ne0sqbS0XWHMvvrg6NE5jQ1xguCSSw9jQeUk2lfrXYsKDdOe6K+oZiwKPilYPQ==} - engines: {node: '>=12'} - cpu: [arm64] - os: [win32] - requiresBuild: true - dev: true - optional: true - - /@esbuild/win32-ia32@0.19.11: - resolution: {integrity: sha512-6YOrWS+sDJDmshdBIQU+Uoyh7pQKrdykdefC1avn76ss5c+RN6gut3LZA4E2cH5xUEp5/cA0+YxRaVtRAb0xBg==} - engines: {node: '>=12'} - cpu: [ia32] - os: [win32] - requiresBuild: true - dev: true - optional: true - - /@esbuild/win32-x64@0.19.11: - resolution: {integrity: sha512-vfkhltrjCAb603XaFhqhAF4LGDi2M4OrCRrFusyQ+iTLQ/o60QQXxc9cZC/FFpihBI9N1Grn6SMKVJ4KP7Fuiw==} - engines: {node: '>=12'} - cpu: [x64] - os: [win32] - requiresBuild: true - dev: true - optional: true - /@eslint-community/eslint-utils@4.4.0(eslint@8.50.0): resolution: {integrity: sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -2119,37 +1909,6 @@ packages: is-symbol: 1.0.4 dev: false - /esbuild@0.19.11: - resolution: {integrity: sha512-HJ96Hev2hX/6i5cDVwcqiJBBtuo9+FeIJOtZ9W1kA5M6AMJRHUZlpYZ1/SbEwtO0ioNAW8rUooVpC/WehY2SfA==} - engines: {node: '>=12'} - hasBin: true - requiresBuild: true - optionalDependencies: - '@esbuild/aix-ppc64': 0.19.11 - '@esbuild/android-arm': 0.19.11 - '@esbuild/android-arm64': 0.19.11 - '@esbuild/android-x64': 0.19.11 - '@esbuild/darwin-arm64': 0.19.11 - '@esbuild/darwin-x64': 0.19.11 - '@esbuild/freebsd-arm64': 0.19.11 - '@esbuild/freebsd-x64': 0.19.11 - '@esbuild/linux-arm': 0.19.11 - '@esbuild/linux-arm64': 0.19.11 - '@esbuild/linux-ia32': 0.19.11 - '@esbuild/linux-loong64': 0.19.11 - '@esbuild/linux-mips64el': 0.19.11 - '@esbuild/linux-ppc64': 0.19.11 - '@esbuild/linux-riscv64': 0.19.11 - '@esbuild/linux-s390x': 0.19.11 - '@esbuild/linux-x64': 0.19.11 - '@esbuild/netbsd-x64': 0.19.11 - '@esbuild/openbsd-x64': 0.19.11 - '@esbuild/sunos-x64': 0.19.11 - '@esbuild/win32-arm64': 0.19.11 - '@esbuild/win32-ia32': 0.19.11 - '@esbuild/win32-x64': 0.19.11 - dev: true - /escalade@3.1.1: resolution: {integrity: sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==} engines: {node: '>=6'} @@ -2449,12 +2208,6 @@ packages: get-intrinsic: 1.2.1 dev: false - /get-tsconfig@4.7.2: - resolution: {integrity: sha512-wuMsz4leaj5hbGgg4IvDU0bqJagpftG5l5cXIAvo8uZrqn0NJqwtfupTN00VnkQJPcIRrxYrm1Ue24btpCha2A==} - dependencies: - resolve-pkg-maps: 1.0.0 - dev: true - /glob-parent@5.1.2: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} engines: {node: '>= 6'} @@ -3146,10 +2899,6 @@ packages: engines: {node: '>=4'} dev: true - /resolve-pkg-maps@1.0.0: - resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} - dev: true - /reusify@1.0.4: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} @@ -3381,17 +3130,6 @@ packages: typescript: 5.0.4 dev: true - /tsx@4.7.0: - resolution: {integrity: sha512-I+t79RYPlEYlHn9a+KzwrvEwhJg35h/1zHsLC2JXvhC2mdynMv6Zxzvhv5EMV6VF5qJlLlkSnMVvdZV3PSIGcg==} - engines: {node: '>=18.0.0'} - hasBin: true - dependencies: - esbuild: 0.19.11 - get-tsconfig: 4.7.2 - optionalDependencies: - fsevents: 2.3.3 - dev: true - /type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} diff --git a/tee-worker/ts-tests/worker/.mocharc.json b/tee-worker/ts-tests/worker/.mocharc.json deleted file mode 100644 index f1e313de38..0000000000 --- a/tee-worker/ts-tests/worker/.mocharc.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "$schema": "https://json.schemastore.org/mocharc.json", - "require": "tsx" -} diff --git a/tee-worker/ts-tests/worker/package.json b/tee-worker/ts-tests/worker/package.json index a9b32ae97a..2113d20611 100644 --- a/tee-worker/ts-tests/worker/package.json +++ b/tee-worker/ts-tests/worker/package.json @@ -5,8 +5,8 @@ "scripts": { "format": "pnpm exec prettier --write .", "check-format": "pnpm exec prettier --check .", - "test-resuming-worker:staging": "pnpm exec eslint . && pnpm exec cross-env NODE_ENV=staging mocha resuming_worker.test.ts", - "test-resuming-worker:local": "pnpm exec eslint . && pnpm exec cross-env NODE_ENV=local mocha resuming_worker.test.ts" + "test-resuming-worker:staging": "pnpm exec eslint . && pnpm exec cross-env NODE_ENV=staging mocha --trace-warnings --full-trace --exit --sort -r ts-node/register --loader=ts-node/esm 'resuming_worker.test.ts'", + "test-resuming-worker:local": "pnpm exec eslint . && pnpm exec cross-env NODE_ENV=local mocha --exit --sort -r ts-node/register --loader=ts-node/esm 'resuming_worker.test.ts'" }, "dependencies": { "chai": "^4.3.6", @@ -52,8 +52,7 @@ "ethers": "^5.7.2", "prettier": "2.8.1", "ts-node": "^10.9.1", - "typescript": "5.0.4", - "tsx": "4.7.0" + "typescript": "5.0.4" }, "packageManager": "pnpm@8.7.6" } diff --git a/tee-worker/ts-tests/worker/resuming_worker.test.ts b/tee-worker/ts-tests/worker/resuming_worker.test.ts index 96e4ee03b7..62a12a7c9d 100644 --- a/tee-worker/ts-tests/worker/resuming_worker.test.ts +++ b/tee-worker/ts-tests/worker/resuming_worker.test.ts @@ -1,4 +1,95 @@ -import { sleep } from './../integration-tests/common/utils'; +import dotenv from 'dotenv'; +import { ChildProcess, spawn } from 'child_process'; +import * as readline from 'readline'; +import fs from 'fs'; +import * as path from 'path'; +import * as process from 'process'; +import { step } from 'mocha-steps'; +import WebSocketAsPromised from 'websocket-as-promised'; +import os from 'os'; +import { assert } from 'chai'; +import type { HexString } from '@polkadot/util/types'; +import * as base58 from 'micro-base58'; +import { u8aToHex } from '@polkadot/util'; +import Options from 'websocket-as-promised/types/options'; +import WebSocket from 'ws'; + +// eslint-disable-next-line @typescript-eslint/no-var-requires, no-undef +dotenv.config({ path: `.env.${process.env.NODE_ENV}` }); + +type WorkerConfig = { + name: string; + muRaPort: number; + untrustedHttpPort: number; + trustedWorkerPort: number; + untrustedWorkerPort: number; + enableMockServer: boolean; + requestStateOnLaunch: boolean; +}; + +const workerConfig: Record<'worker0' | 'worker1', WorkerConfig> = { + worker0: { + name: 'worker0', + muRaPort: 3443, + untrustedHttpPort: 4545, + trustedWorkerPort: 2000, + untrustedWorkerPort: 3000, + enableMockServer: true, + requestStateOnLaunch: false, + }, + worker1: { + name: 'worker1', + muRaPort: 3444, + untrustedHttpPort: 4546, + trustedWorkerPort: 2001, + untrustedWorkerPort: 3001, + enableMockServer: false, + requestStateOnLaunch: true, + }, +} as const; + +type NodeConfig = { + nodeUrl: string; + nodePort: number; +}; + +type Command = 'launch' | 'resume'; + +function generateWorkerCommandArguments( + command: Command, + { nodeUrl, nodePort }: NodeConfig, + workerParams: WorkerConfig +): string { + const isLaunch = command === 'launch'; + + return [ + '--running-mode mock', + ...(workerParams.enableMockServer ? ['--enable-mock-server'] : []), + ...(isLaunch ? ['--clean-reset'] : []), + '--mu-ra-external-address localhost', + `--mu-ra-port ${workerParams.muRaPort}`, + `--untrusted-http-port ${workerParams.untrustedHttpPort}`, + '--ws-external', + '--trusted-external-address wss://localhost', + `--trusted-worker-port ${workerParams.trustedWorkerPort}`, + '--untrusted-external-address ws://localhost', + `--untrusted-worker-port ${workerParams.untrustedWorkerPort}`, + `--node-url ${nodeUrl}`, + `--node-port ${nodePort}`, + `run`, + `--skip-ra`, + `--dev`, + ...(isLaunch && workerParams.requestStateOnLaunch ? ['--request-state'] : []), + ].join(' '); +} + +function initializeFiles(workingDir: string, binaryDir: string) { + fs.mkdirSync(workingDir, { recursive: true }); + fs.copyFileSync(path.join(binaryDir, 'enclave.signed.so'), path.join(workingDir, 'enclave.signed.so')); + fs.copyFileSync(path.join(binaryDir, 'litentry-worker'), path.join(workingDir, 'litentry-worker')); + fs.closeSync(fs.openSync(path.join(workingDir, 'spid.txt'), 'w')); + fs.closeSync(fs.openSync(path.join(workingDir, 'key.txt'), 'w')); +} type RetryConfig = { isRetriable: (err: unknown) => boolean; @@ -28,6 +119,383 @@ async function withRetry( } } +type JobConfig = { + workerConfig: WorkerConfig; + nodeConfig: NodeConfig; + workingDir: string; +}; + +class SidechainDbLockUnavailable extends Error { + constructor() { + super('sidechain_db lock unavailable'); + } +} + +async function spawnWorkerJob( + command: Command, + { workingDir, nodeConfig, workerConfig }: JobConfig, + subprocessTracker: Set +): Promise<{ job: ChildProcess; shard: HexString | undefined }> { + const { name } = workerConfig; + const task = () => + new Promise<{ job: ChildProcess; shard: HexString | undefined }>((resolve, reject) => { + let shard: HexString | undefined = undefined; + + const job = spawn( + `./litentry-worker`, + [generateWorkerCommandArguments(command, nodeConfig, workerConfig)], + { + cwd: workingDir, + shell: '/bin/sh', + env: { + RUST_LOG: 'warn,sp_io::storage=error,substrate_api_client=warn', + ...process.env, + }, + detached: true, + } + ); + + job.on('error', (error) => reject(error)); + + const pid = job.pid; + if (!pid) { + return; + } + + subprocessTracker.add(pid); + job.on('exit', () => { + subprocessTracker.delete(pid); + }); + + job.on('close', (code) => { + console.log(`${name} close: ${code}`); + }); + + job.stderr.setEncoding('utf8'); + const errorStream = readline.createInterface(job.stderr); + errorStream.on('line', (line: string) => { + console.warn(name, line); + if (line.includes('sidechain_db/LOCK: Resource temporarily unavailable')) { + reject(new SidechainDbLockUnavailable()); + } + }); + + job.stdout.setEncoding('utf8'); + const outputStream = readline.createInterface(job.stdout); + outputStream.on('line', (line: string) => { + console.log(name, line); + + const match = line.match(/^Successfully initialized shard "(?[\w].*).*"/); + if (match !== null) { + /** + * Assertions needed because regex contents aren't reflected in function typing; + * see e.g. https://github.com/microsoft/TypeScript/issues/32098. + * + * If the regexp match succeeds, the `groups` property is guaranteed to be present, + * as well as the corresponding named capturing groups. See + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/match + */ + const base58shard = match.groups!.shard; + shard = u8aToHex(base58.decode(base58shard)) as HexString; + return; + } + + if (line.includes('Untrusted RPC server is spawned on')) { + resolve({ job, shard }); + } + }); + }); + + return withRetry(task, { + isRetriable: (err) => err instanceof SidechainDbLockUnavailable, + maxRetries: 3, + initialDelaySeconds: 10, + backoffFactor: 1.5, + }); +} + +type WorkerState = { + shard: HexString; + job: ChildProcess; + connection: WebSocketAsPromised; + latestSeenBlock: number; +}; + +async function launchWorker( + binaryDir: string, + jobConfig: JobConfig, + subprocessTracker: Set +): Promise { + initializeFiles(jobConfig.workingDir, binaryDir); + + const { shard, job } = await spawnWorkerJob('launch', jobConfig, subprocessTracker); + + if (shard === undefined) { + throw new Error('RPC server spawned before shard initialization'); + } + + const connection = await initWorkerConnection(`ws://localhost:${jobConfig.workerConfig.untrustedWorkerPort}`); + const latestSeenBlock = await waitWorkerProducingBlock(connection, shard, 4); + + return { shard, job, connection, latestSeenBlock }; +} + +async function resumeWorker( + jobConfig: JobConfig, + connection: WebSocketAsPromised, + subprocessTracker: Set +): Promise { + const { shard, job } = await spawnWorkerJob('resume', jobConfig, subprocessTracker); + + if (shard !== undefined) { + throw new Error('Shard should have been reused from the previous run'); + } + + await connection.open(); + + return job; +} + +function killWorker(worker: ChildProcess): Promise { + const pid = worker.pid; + + if (pid === undefined) { + return Promise.reject(new Error('Attempted to kill an unspawned worker?')); + } + + return new Promise((resolve, reject) => { + /** + * Kill each process in the worker's group; + * see https://www.man7.org/linux/man-pages/man2/kill.2.html + */ + process.kill(-pid, 'SIGKILL'); + worker.on('exit', () => { + resolve(); + }); + worker.on('error', (error) => reject(error)); + }); +} + +async function latestBlock( + connection: WebSocketAsPromised, + shard: HexString +): Promise<{ result: undefined | { number: number; hash: string } }> { + return await connection.sendRequest( + { + jsonrpc: '2.0', + method: 'sidechain_latestBlock', + params: shard, + id: 1, + }, + { requestId: 1, timeout: 6000 } + ); +} + +async function waitForBlock(connection: WebSocketAsPromised, shard: HexString, lowerBound: number): Promise { + const task = async () => { + const resp = await latestBlock(connection, shard); + const blockNumber = resp.result?.number; + console.log(`${connection.ws.url} current block: ${blockNumber}`); + if (blockNumber != undefined && blockNumber >= lowerBound) { + return blockNumber; + } + throw new Error(`waiting for block ${lowerBound}; got ${blockNumber} instead`); + }; + + return withRetry(task, { isRetriable: () => true, maxRetries: 10, initialDelaySeconds: 5, backoffFactor: 1.5 }); +} + +async function waitWorkerProducingBlock( + connection: WebSocketAsPromised, + shard: HexString, + atLeast: number +): Promise { + const currentBlockNumber = await waitForBlock(connection, shard, 0); + return await waitForBlock(connection, shard, currentBlockNumber + atLeast); +} + +function getConfiguration(): { binaryDir: string; nodeUrl: string; nodePort: number } { + const binaryDir = process.env.BINARY_DIR; + if (binaryDir === undefined) { + throw new Error('Environment variable BINARY_DIR not defined'); + } + + const [, nodeHost, nodePortRaw] = process.env.NODE_ENDPOINT!.split(':'); + const nodePort = Number.parseInt(nodePortRaw); + if (nodeHost === undefined || Number.isNaN(nodePort)) { + throw new Error('Environment variable NODE_ENDPOINT undefined or malformed'); + } + + return { binaryDir, nodeUrl: `ws:${nodeHost}`, nodePort }; +} + describe('Resume worker', function () { - console.log('works'); + this.timeout(6000000); + + const { binaryDir, nodeUrl, nodePort } = getConfiguration(); + const nodeConfig = { nodeUrl, nodePort }; + + const tempDir = os.tmpdir(); + const worker0Dir = fs.mkdtempSync(path.join(tempDir, 'worker0-')); + const worker1Dir = fs.mkdtempSync(path.join(tempDir, 'worker1-')); + + const jobConfig: Record<'worker0' | 'worker1', JobConfig> = { + worker0: { + workingDir: worker0Dir, + nodeConfig, + workerConfig: workerConfig.worker0, + }, + worker1: { + workingDir: worker1Dir, + nodeConfig, + workerConfig: workerConfig.worker1, + }, + }; + + const subprocessTracker: Set = new Set(); + + after(() => { + fs.rmSync(worker0Dir, { recursive: true, force: true }); + fs.rmSync(worker1Dir, { recursive: true, force: true }); + subprocessTracker.forEach((pid) => { + if (pid !== undefined) { + try { + process.kill(-pid, 'SIGTERM'); + } catch (error) { + if ((error as { code: unknown }).code === 'ESRCH') { + return; // Process has already died; nothing to do + } + console.warn(error); + } + } + }); + }); + + let worker0State: WorkerState | undefined = undefined; + + step('One worker', async function () { + // first launch worker0 + worker0State = await launchWorker(binaryDir, jobConfig.worker0, subprocessTracker); + console.log('=========== worker0 launched and produced blocks =================='); + + // kill worker0 + await killWorker(worker0State.job); + console.log('=========== worker0 stopped =================='); + + // resume worker0 + worker0State.job = await resumeWorker(jobConfig.worker0, worker0State.connection, subprocessTracker); + console.log('=========== worker0 resumed =================='); + + // check block production + worker0State.latestSeenBlock = await waitForBlock( + worker0State.connection, + worker0State.shard, + worker0State.latestSeenBlock + 1 + ); + console.log('=========== worker0 produced blocks =================='); + }); + + let worker1State: WorkerState | undefined = undefined; + + step('Two workers & resume worker1', async function () { + assert(worker0State); + + // first launch worker1 + worker1State = await launchWorker(binaryDir, jobConfig.worker1, subprocessTracker); + console.log('=========== worker1 launched and produced blocks =================='); + + // kill worker1 + await killWorker(worker1State.job); + console.log('=========== worker1 stopped =================='); + + // let worker0 produce + worker0State.latestSeenBlock = await waitForBlock( + worker0State.connection, + worker0State.shard, + worker0State.latestSeenBlock + 1 + ); + console.log('=========== worker0 still produces blocks =================='); + + // resume worker1 + worker1State.job = await resumeWorker(jobConfig.worker1, worker1State.connection, subprocessTracker); + console.log('=========== worker1 resumed =================='); + + // check block production + worker1State.latestSeenBlock = await waitForBlock( + worker1State.connection, + worker1State.shard, + worker1State.latestSeenBlock + 1 + ); + console.log('=========== worker1 produced blocks =================='); + }); + + step('Kill and resume both workers', async function () { + assert(worker0State); + assert(worker1State); + + // kill both workers + await killWorker(worker0State.job); + console.log('=========== worker0 stopped =================='); + await killWorker(worker1State.job); + console.log('=========== worker1 stopped =================='); + + // resume and check worker1 + worker1State.job = await resumeWorker(jobConfig.worker1, worker1State.connection, subprocessTracker); + console.log('=========== worker1 resumed =================='); + worker1State.latestSeenBlock = await waitForBlock( + worker1State.connection, + worker1State.shard, + worker1State.latestSeenBlock + 1 + ); + console.log('=========== worker1 produced blocks =================='); + + // resume and check worker0 + worker0State.job = await resumeWorker(jobConfig.worker0, worker0State.connection, subprocessTracker); + console.log('=========== worker0 resumed =================='); + worker0State.latestSeenBlock = await waitForBlock( + worker0State.connection, + worker0State.shard, + worker0State.latestSeenBlock + 1 + ); + console.log('=========== worker0 produced blocks =================='); + + // check worker1 health + worker1State.latestSeenBlock = await waitForBlock( + worker1State.connection, + worker1State.shard, + worker1State.latestSeenBlock + 1 + ); + console.log('=========== worker1 still produces blocks =================='); + }); + + step('Tidy up', async function () { + // kill both workers + if (worker0State) { + await killWorker(worker0State.job); + console.log('=========== worker0 stopped =================='); + } + if (worker1State) { + await killWorker(worker1State.job); + console.log('=========== worker1 stopped =================='); + } + }); }); + +export async function initWorkerConnection(endpoint: string): Promise { + const wsp = new WebSocketAsPromised(endpoint, ({ + createWebSocket: (url: any) => new WebSocket(url), + extractMessageData: (event: any) => event, + packMessage: (data: any) => JSON.stringify(data), + unpackMessage: (data: string | ArrayBuffer | Blob) => JSON.parse(data.toString()), + attachRequestId: (data: any, requestId: string | number) => Object.assign({ id: requestId }, data), + extractRequestId: (data: any) => data && data.id, // read requestId from message `id` field + })); + await wsp.open(); + return wsp; +} + +export function sleep(secs: number) { + return new Promise((resolve) => { + setTimeout(resolve, secs * 1000); + }); +}