feat(test): Add filebased config support for integration tests (#2043)

## What ❔

<!-- What are the changes this PR brings about? -->
<!-- Example: This PR adds a PR template to the repo. -->
<!-- (For bigger PRs adding more context is appreciated) -->

## Why ❔

<!-- Why are these changes done? What goal do they contribute to? What
are the principles behind them? -->
<!-- Example: PR templates ensure PR reviewers, observers, and future
iterators are in context about the evolution of repos. -->

## Checklist

<!-- Check your PR fulfills the following items. -->
<!-- For draft PRs check the boxes as you complete them. -->

- [ ] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [ ] Tests for the changes have been added / updated.
- [ ] Documentation comments have been added / updated.
- [ ] Code has been formatted via `zk fmt` and `zk lint`.
- [ ] Spellcheck has been run via `zk spellcheck`.

---------

Signed-off-by: Danil <deniallugo@gmail.com>

.gitignore

@@ -97,3 +97,7 @@ hyperchain-*.yml
 
 # Prover keys that should not be commited
 prover/vk_setup_data_generator_server_fri/data/setup_*
+
+# Zk Toolbox
+chains/era/configs/*
+configs/*

ZkStack.yaml

@@ -0,0 +1,10 @@
+name: zk
+l1_network: Localhost
+link_to_code: .
+chains: ./chains
+config: ./configs/
+default_chain: era
+l1_rpc_url: http://localhost:8545
+era_chain_id: 270
+prover_version: NoProofs
+wallet_creation: Localhost

chains/era/ZkStack.yaml

@@ -0,0 +1,12 @@
+id: 1
+name: era
+chain_id: 271
+prover_version: NoProofs
+configs: ./chains/era/configs/
+rocks_db_path: ./chains/era/db/
+l1_batch_commit_data_generator_mode: Rollup
+base_token:
+  address: '0x0000000000000000000000000000000000000001'
+  nominator: 1
+  denominator: 1
+wallet_creation: Localhost

core/tests/ts-integration/package.json

@@ -31,6 +31,7 @@
     "ts-node": "^10.1.0",
     "typescript": "^4.3.5",
     "zksync-ethers": "5.8.0-beta.5",
-    "elliptic": "^6.5.5"
+    "elliptic": "^6.5.5",
+    "yaml": "^2.4.2"
   }
 }

core/tests/ts-integration/src/env.ts

@@ -4,6 +4,7 @@ import * as ethers from 'ethers';
 import * as zksync from 'zksync-ethers';
 import { DataAvailabityMode, NodeMode, TestEnvironment } from './types';
 import { Reporter } from './reporter';
+import * as yaml from 'yaml';
 import { L2_BASE_TOKEN_ADDRESS } from 'zksync-ethers/build/utils';
 
 /**
@@ -14,16 +15,12 @@ import { L2_BASE_TOKEN_ADDRESS } from 'zksync-ethers/build/utils';
  * This function is expected to be called *before* loading an environment via `loadTestEnvironment`,
  * because the latter expects server to be running and may throw otherwise.
  */
-export async function waitForServer() {
+export async function waitForServer(l2NodeUrl: string) {
     const reporter = new Reporter();
     // Server startup may take a lot of time on the staging.
     const attemptIntervalMs = 1000;
     const maxAttempts = 20 * 60; // 20 minutes
 
-    const l2NodeUrl = ensureVariable(
-        process.env.ZKSYNC_WEB3_API_URL || process.env.API_WEB3_JSON_RPC_HTTP_URL,
-        'L2 node URL'
-    );
     const l2Provider = new zksync.Provider(l2NodeUrl);
 
     reporter.startAction('Connecting to server');
@@ -45,25 +42,146 @@ export async function waitForServer() {
     throw new Error('Failed to wait for the server to start');
 }
 
+function getMainWalletPk(pathToHome: string, network: string): string {
+    if (network.toLowerCase() == 'localhost') {
+        const testConfigPath = path.join(pathToHome, `etc/test_config/constant`);
+        const ethTestConfig = JSON.parse(fs.readFileSync(`${testConfigPath}/eth.json`, { encoding: 'utf-8' }));
+        return ethers.Wallet.fromMnemonic(ethTestConfig.test_mnemonic as string, "m/44'/60'/0'/0/0").privateKey;
+    } else {
+        return ensureVariable(process.env.MASTER_WALLET_PK, 'Main wallet private key');
+    }
+}
+
+/*
+    Loads the environment for file based configs.
+ */
+async function loadTestEnvironmentFromFile(chain: string): Promise<TestEnvironment> {
+    const pathToHome = path.join(__dirname, '../../../..');
+    let ecosystem = loadEcosystem(pathToHome);
+
+    let generalConfig = loadConfig(pathToHome, chain, 'general.yaml');
+    let genesisConfig = loadConfig(pathToHome, chain, 'genesis.yaml');
+
+    const network = ecosystem.l1_network;
+    let mainWalletPK = getMainWalletPk(pathToHome, network);
+    const l2NodeUrl = generalConfig.api.web3_json_rpc.http_url;
+
+    await waitForServer(l2NodeUrl);
+
+    const l2Provider = new zksync.Provider(l2NodeUrl);
+    const baseTokenAddress = await l2Provider.getBaseTokenContractAddress();
+
+    const l1NodeUrl = ecosystem.l1_rpc_url;
+    const wsL2NodeUrl = generalConfig.api.web3_json_rpc.ws_url;
+
+    const contractVerificationUrl = generalConfig.contract_verifier.url;
+
+    const tokens = getTokensNew(pathToHome);
+    // wBTC is chosen because it has decimals different from ETH (8 instead of 18).
+    // Using this token will help us to detect decimals-related errors.
+    // but if it's not available, we'll use the first token from the list.
+    let token = tokens.tokens['wBTC'];
+    if (token === undefined) {
+        token = Object.values(tokens.tokens)[0];
+    }
+    const weth = tokens.tokens['WETH'];
+    let baseToken;
+
+    for (const key in tokens.tokens) {
+        const token = tokens.tokens[key];
+        if (zksync.utils.isAddressEq(token.address, baseTokenAddress)) {
+            baseToken = token;
+        }
+    }
+    // `waitForServer` is expected to be executed. Otherwise this call may throw.
+
+    const l2TokenAddress = await new zksync.Wallet(
+        mainWalletPK,
+        l2Provider,
+        ethers.getDefaultProvider(l1NodeUrl)
+    ).l2TokenAddress(token.address);
+
+    const l2WethAddress = await new zksync.Wallet(
+        mainWalletPK,
+        l2Provider,
+        ethers.getDefaultProvider(l1NodeUrl)
+    ).l2TokenAddress(weth.address);
+
+    const baseTokenAddressL2 = L2_BASE_TOKEN_ADDRESS;
+    const l2ChainId = parseInt(genesisConfig.l2_chain_id);
+    const l1BatchCommitDataGeneratorMode = genesisConfig.l1_batch_commit_data_generator_mode as DataAvailabityMode;
+    let minimalL2GasPrice = generalConfig.state_keeper.minimal_l2_gas_price;
+    // TODO add support for en
+    let nodeMode = NodeMode.Main;
+
+    const validationComputationalGasLimit = parseInt(generalConfig.state_keeper.validation_computational_gas_limit);
+    // TODO set it properly
+    const priorityTxMaxGasLimit = 72000000;
+    const maxLogsLimit = parseInt(generalConfig.api.web3_json_rpc.req_entities_limit);
+
+    return {
+        maxLogsLimit,
+        pathToHome,
+        priorityTxMaxGasLimit,
+        validationComputationalGasLimit,
+        nodeMode,
+        minimalL2GasPrice,
+        l1BatchCommitDataGeneratorMode,
+        l2ChainId,
+        network,
+        mainWalletPK,
+        l2NodeUrl,
+        l1NodeUrl,
+        wsL2NodeUrl,
+        contractVerificationUrl,
+        erc20Token: {
+            name: token.name,
+            symbol: token.symbol,
+            decimals: token.decimals,
+            l1Address: token.address,
+            l2Address: l2TokenAddress
+        },
+        wethToken: {
+            name: weth.name,
+            symbol: weth.symbol,
+            decimals: weth.decimals,
+            l1Address: weth.address,
+            l2Address: l2WethAddress
+        },
+        baseToken: {
+            name: baseToken?.name || token.name,
+            symbol: baseToken?.symbol || token.symbol,
+            decimals: baseToken?.decimals || token.decimals,
+            l1Address: baseToken?.address || token.address,
+            l2Address: baseTokenAddressL2
+        }
+    };
+}
+
+export async function loadTestEnvironment(): Promise<TestEnvironment> {
+    let chain = process.env.CHAIN_NAME;
+
+    if (chain) {
+        return await loadTestEnvironmentFromFile(chain);
+    }
+    return await loadTestEnvironmentFromEnv();
+}
+
 /**
  * Loads the test environment from the env variables.
  */
-export async function loadTestEnvironment(): Promise<TestEnvironment> {
+export async function loadTestEnvironmentFromEnv(): Promise<TestEnvironment> {
     const network = process.env.CHAIN_ETH_NETWORK || 'localhost';
+    const pathToHome = path.join(__dirname, '../../../../');
 
-    let mainWalletPK;
-    if (network == 'localhost') {
-        const testConfigPath = path.join(process.env.ZKSYNC_HOME!, `etc/test_config/constant`);
-        const ethTestConfig = JSON.parse(fs.readFileSync(`${testConfigPath}/eth.json`, { encoding: 'utf-8' }));
-        mainWalletPK = ethers.Wallet.fromMnemonic(ethTestConfig.test_mnemonic as string, "m/44'/60'/0'/0/0").privateKey;
-    } else {
-        mainWalletPK = ensureVariable(process.env.MASTER_WALLET_PK, 'Main wallet private key');
-    }
+    let mainWalletPK = getMainWalletPk(pathToHome, network);
 
     const l2NodeUrl = ensureVariable(
         process.env.ZKSYNC_WEB3_API_URL || process.env.API_WEB3_JSON_RPC_HTTP_URL,
         'L2 node URL'
     );
+
+    await waitForServer(l2NodeUrl);
     const l2Provider = new zksync.Provider(l2NodeUrl);
     const baseTokenAddress = await l2Provider.getBaseTokenContractAddress();
 
@@ -76,7 +194,6 @@ export async function loadTestEnvironment(): Promise<TestEnvironment> {
         ? process.env.CONTRACT_VERIFIER_URL!
         : ensureVariable(process.env.CONTRACT_VERIFIER_URL, 'Contract verification API');
 
-    const pathToHome = path.join(__dirname, '../../../../');
     const tokens = getTokens(pathToHome, process.env.CHAIN_ETH_NETWORK || 'localhost');
     // wBTC is chosen because it has decimals different from ETH (8 instead of 18).
     // Using this token will help us to detect decimals-related errors.
@@ -177,6 +294,14 @@ function ensureVariable(value: string | undefined, variableName: string): string
     return value;
 }
 
+interface TokensDict {
+    [key: string]: L1Token;
+}
+
+type Tokens = {
+    tokens: TokensDict;
+};
+
 type L1Token = {
     name: string;
     symbol: string;
@@ -195,3 +320,56 @@ function getTokens(pathToHome: string, network: string): L1Token[] {
         })
     );
 }
+
+function getTokensNew(pathToHome: string): Tokens {
+    const configPath = path.join(pathToHome, '/configs/erc20.yaml');
+    if (!fs.existsSync(configPath)) {
+        throw Error('Tokens config not found');
+    }
+
+    return yaml.parse(
+        fs.readFileSync(configPath, {
+            encoding: 'utf-8'
+        }),
+        {
+            customTags
+        }
+    );
+}
+
+function loadEcosystem(pathToHome: string): any {
+    const configPath = path.join(pathToHome, '/ZkStack.yaml');
+    if (!fs.existsSync(configPath)) {
+        return [];
+    }
+    return yaml.parse(
+        fs.readFileSync(configPath, {
+            encoding: 'utf-8'
+        })
+    );
+}
+
+function loadConfig(pathToHome: string, chainName: string, config: string): any {
+    const configPath = path.join(pathToHome, `/chains/${chainName}/configs/${config}`);
+    if (!fs.existsSync(configPath)) {
+        return [];
+    }
+    return yaml.parse(
+        fs.readFileSync(configPath, {
+            encoding: 'utf-8'
+        })
+    );
+}
+
+function customTags(tags: yaml.Tags): yaml.Tags {
+    for (const tag of tags) {
+        // @ts-ignore
+        if (tag.format === 'HEX') {
+            // @ts-ignore
+            tag.resolve = (str, _onError, _opt) => {
+                return str;
+            };
+        }
+    }
+    return tags;
+}

core/tests/ts-integration/src/jest-setup/global-setup.ts

@@ -1,4 +1,4 @@
-import { TestContextOwner, loadTestEnvironment, waitForServer } from '../index';
+import { TestContextOwner, loadTestEnvironment } from '../index';
 
 declare global {
     var __ZKSYNC_TEST_CONTEXT_OWNER__: TestContextOwner;
@@ -18,7 +18,6 @@ async function performSetup(_globalConfig: any, _projectConfig: any) {
 
     // Before starting any actual logic, we need to ensure that the server is running (it may not
     // be the case, for example, right after deployment on stage).
-    await waitForServer();
 
     const testEnvironment = await loadTestEnvironment();
     const testContextOwner = new TestContextOwner(testEnvironment);

yarn.lock

@@ -2805,6 +2805,11 @@
     expect "^29.0.0"
     pretty-format "^29.0.0"
 
+"@types/js-yaml@^4.0.9":
+  version "4.0.9"
+  resolved "https://registry.yarnpkg.com/@types/js-yaml/-/js-yaml-4.0.9.tgz#cd82382c4f902fed9691a2ed79ec68c5898af4c2"
+  integrity sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==
+
 "@types/json-schema@^7.0.12":
   version "7.0.15"
   resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.15.tgz#596a1747233694d50f6ad8a7869fcb6f56cf5841"