From 590892ee1874cd35705dc30d2e9261cf6a328d8e Mon Sep 17 00:00:00 2001 From: p Date: Thu, 1 Dec 2022 17:57:44 -0800 Subject: [PATCH] Move basic.py to where ci-cd will pick it up from (#610) * Move basic.py to where ci-cd will pick it up from * Update README * Update test to pass in CI --- .github/workflows/dev-cd.yaml | 4 +- .gitignore | 2 +- .internal-ci/test/fs-integration/README.md | 26 ++- .internal-ci/test/fs-integration/basic.py | 171 +++++++++++++++++- .../test/fs-integration/test_config.json | 10 + integ-tests/basic.py | 134 -------------- python-library/fullservice.py | 5 +- tools/build-fs.sh | 4 +- tools/run-fs.sh | 4 +- 9 files changed, 209 insertions(+), 151 deletions(-) create mode 100644 .internal-ci/test/fs-integration/test_config.json delete mode 100644 integ-tests/basic.py diff --git a/.github/workflows/dev-cd.yaml b/.github/workflows/dev-cd.yaml index 633a0d7d6..a7f74d486 100644 --- a/.github/workflows/dev-cd.yaml +++ b/.github/workflows/dev-cd.yaml @@ -173,8 +173,6 @@ jobs: rancher_url: ${{ secrets.DEV_RANCHER_URL }} rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} - # TODO: Add integration tests. - # This will need to run on our self-hosted so it can connect to the privately deployed full-service. test: runs-on: [self-hosted, Linux, small] @@ -213,7 +211,7 @@ jobs: "${POETRY_HOME}/bin/poetry" install --without=dev # Run test - "${POETRY_HOME}/bin/poetry" run python3 ./basic.py + "${POETRY_HOME}/bin/poetry" run python3 ./basic.py "${CONFIG_JSON}" popd || exit 0 # remove the testing environment after all tests are run successfully when this diff --git a/.gitignore b/.gitignore index 3cdc1bcfb..5d34922b5 100644 --- a/.gitignore +++ b/.gitignore @@ -80,7 +80,7 @@ __pycache__ *sgx_linux_x64_sdk_2.9.101.2.bin -integ-tests/entropy_config +.internal-ci/test/fs-integration/test_config.json # mob prompt and build tmp .mob/ diff --git a/.internal-ci/test/fs-integration/README.md b/.internal-ci/test/fs-integration/README.md index 7aec6effa..c72fc09f0 100644 --- a/.internal-ci/test/fs-integration/README.md +++ b/.internal-ci/test/fs-integration/README.md @@ -1,6 +1,24 @@ # fs-integration -These scrips use Python Poetry Env/Package manager. +These scrips use Python Poetry Env/Package manager + +## To run locally + +1. Setup poetry (see sections below for more details) + +``` +curl -sSL https://install.python-poetry.org | python3 - +poetry install +``` + +2. Add mnemonics to `test_config.json` (or create another configuration file and pass as another parameter when running the test) + - Don't worry, `.gitignore` will prevent you from accidentally checking in your secrets + +3. Run the test + +``` +poetry run python3 basic.py +``` ## Install Dependencies @@ -47,6 +65,12 @@ Remove the default directory project directory in the `pyproject.toml`, we're no {include = "fs_integration"} ``` +## Use existing poetry configuration +To read and install dependencies from current project's `pyproject.toml`: +``` +poetry install +``` + ## VSCode hints. **Activate Poetry** diff --git a/.internal-ci/test/fs-integration/basic.py b/.internal-ci/test/fs-integration/basic.py index ea3e5a50b..a035f5b7e 100644 --- a/.internal-ci/test/fs-integration/basic.py +++ b/.internal-ci/test/fs-integration/basic.py @@ -1,17 +1,176 @@ - -import sys +import argparse +import asyncio +import json import subprocess +import sys -# Path to "python-library" modules. repo_root_dir = subprocess.check_output("git rev-parse --show-toplevel", shell=True).decode("utf8").strip() sys.path.append("{}/python-library".format(repo_root_dir)) from fullservice import FullServiceAPIv2 as v2 -from dataobjects import Response, Account # TODO rename as FSDataObjects +from FSDataObjects import Response, Account + +sleepy_time = 15 +default_config_path = "./test_config.json" +config = [] +account_ids = [] + +fs = v2() + +# TODO: this is very uggly and needs to be refactored +async def wait_for_account_to_sync(id): + account_status = await fs.get_account_status(id) + while ((account_status.get("result").get("account").get("next_block_index") + != account_status.get("result").get("local_block_height")) or (account_status.get("result").get("balance_per_token").get("0").get("pending") != "0")): + await asyncio.sleep(sleepy_time) + account_status = await fs.get_account_status(id) + + +async def test_cleanup(): + global account_ids + for id in account_ids: + await wait_for_account_to_sync(id) + await fs.remove_account(id) + accounts = await fs.get_accounts() + for id in account_ids: + assert id not in accounts.get('result').get('account_ids'),"Failed to clear out accounts" + account_ids = [] + + +# If this test fails before reaching the last cleanup step, we have leftover +# artifacts in the FS instance. We clean up those residual artifacts here. +# Note: Using a testing framework like pytest would allow us to bundle this in +# a test fixture +async def preclean_this_test(): + await get_account(0, "alice", True) + await get_account(1, "bob", True) + await test_cleanup() + + +def get_mnemonics(n=2): + if n > len(config["Account Mnemonics"]): + raise ValueError("Not enough account available in config") + return config["Account Mnemonics"][:n] + + +async def get_account(i, name="", okay_if_already_imported=False): + global account_ids + + mnemonic = config["Account Mnemonics"][i]["mnemonic"] + account = await fs.import_account( + mnemonic, + "2", # This parameter indicates that we are using the 2nd key derivations method (mnemonics) + name=name + ) + + if not okay_if_already_imported: + assert "error" not in account.keys(), "Failed to import account" + + # Newly imported + if "error" not in account.keys(): + result = Account(account["result"]["account"]) + account_ids.append(result.id) + # Previously imported + else: + error_msg = account.get("error").get("data").get("details") + assert error_msg.startswith("Error interacting& with the database: Account already exists:"), "Unknown import failure" + id = error_msg.split()[-1] + result = Response(await fs.get_account_status(id)).account + account_ids.append(result.id) + return result + + +async def wait_for_network_sync(): + network_status = await fs.get_network_status() + while network_status.get('result').get('network_block_height') != network_status.get('result').get('local_block_height'): + print("Sleep") + await asyncio.sleep(sleepy_time) + network_status = await fs.get_network_status() async def main(): - print("Placeholder script") + while (await fs.get_wallet_status())['result']['wallet_status']['is_synced_all'] != True: + await asyncio.sleep(sleepy_time) + await does_it_go() + + +async def does_it_go(amount_pmob: int = 600000000) -> bool: + network_status = await fs.get_network_status() + assert "error" not in network_status.keys(), "Failed to get network status" + fee = int(network_status.get("result") + .get("network_status") + .get("fees") + .get("0") # zero is the fee key for mob + ) + + """Test Setup """ + pmob_to_send = amount_pmob + + # await preclean_this_test() + + alice = await get_account(0, "alice", True) + bob = await get_account(1, "bob", True) + + await wait_for_account_to_sync(alice.id) + await wait_for_account_to_sync(bob.id) + + alice_balance_0 = int( + (await fs.get_account_status(alice.id)) + .get("result") + .get("balance_per_token") + .get("0") + .get("unspent") + ) + + assert alice_balance_0 >= pmob_to_send + fee, "Insufficient funds in first account." + + bob_balance_0 = int( + (await fs.get_account_status(bob.id)) + .get("result") + .get("balance_per_token") + .get("0") + .get("unspent") + ) + + first_transaction = await fs.build_and_submit_transaction( + alice.id, + recipient_public_address=bob.main_address, + amount={"value": str(pmob_to_send), "token_id": str(0)}, + ) + + # TODO: replace this with a poll loop that waits a block or two + await wait_for_network_sync() + await wait_for_account_to_sync(alice.id) + await wait_for_account_to_sync(bob.id) + + """ Check Results """ + alice_balance_1 = int( + (await fs.get_account_status(alice.id)) + .get("result") + .get("balance_per_token") + .get("0") + .get("unspent") + ) + + bob_balance_1 = int( + (await fs.get_account_status(bob.id)) + .get("result") + .get("balance_per_token") + .get("0") + .get("unspent") + ) + + assert alice_balance_0 == alice_balance_1 + fee + pmob_to_send, "Alice doesn't end with the expected amount" + assert bob_balance_1 == bob_balance_0 + pmob_to_send, "Bob doesn't end with the expected amount" + + # await test_cleanup() if __name__ == "__main__": - main() + parser = argparse.ArgumentParser(description="Basic test") + parser.add_argument("config_path", nargs='?', type=str, default=default_config_path) + args = parser.parse_args() + + with open(args.config_path) as json_file: + config = json.load(json_file) + + asyncio.run(main()) diff --git a/.internal-ci/test/fs-integration/test_config.json b/.internal-ci/test/fs-integration/test_config.json new file mode 100644 index 000000000..437131075 --- /dev/null +++ b/.internal-ci/test/fs-integration/test_config.json @@ -0,0 +1,10 @@ +{ + "Account Mnemonics" : [ + { + "mnemonic":"" + }, + { + "mnemonic":"" + } + ] +} \ No newline at end of file diff --git a/integ-tests/basic.py b/integ-tests/basic.py deleted file mode 100644 index a67b4d5bf..000000000 --- a/integ-tests/basic.py +++ /dev/null @@ -1,134 +0,0 @@ -# verify that the transaction went through -# the mob went through -# the transaction log updatedx -# Ideally all of the endpoints (v2) that actually hit the mobilecoin network -# -# get_network_status -# get_wallet_status -# build, build_and_submit, build_split_txo .. etc - -import argparse -import os -import sys -import asyncio -import json - -sys.path.append(os.path.abspath("../cli")) - -from fullservice import FullServiceAPIv2 as v2 -from FSDataObjects import Response, Account - -default_config_path = "./config" -config = [] -account_ids = [] - -fs = v2() - - -def get_mnemonics(n=2): - if n > len(config["Account Mnemonics"]): - raise ValueError("Not enough account available in config") - return config["Account Mnemonics"][:n] - - -async def get_account(i, okay_if_already_imported=False): - global account_ids - - mnemonic = config["Account Mnemonics"][i]["mnemonic"] - account = await fs.import_account( - mnemonic, "2" # This parameter indicates that we are using the 2nd key derivations method (mnemonics) - ) - - if not okay_if_already_imported: - assert "error" not in account.keys(), "Failed to import account" - - if "error" not in account.keys(): - return Account(account["result"]["account"]) - else: - if len(account_ids) <= i: - accounts_response = Response(await fs.get_accounts()) - account_ids = accounts_response.account_ids - return accounts_response.accounts[account_ids[i]] - else: - return Response(await fs.get_account_status(account_ids[i])).account - - -async def main(): - while (await fs.get_wallet_status())['result']['wallet_status']['is_synced_all'] != True: - await asyncio.sleep(1) - print(await does_it_go()) - - -async def does_it_go(amount_pmob: int = 600000000) -> bool: - network_status = await fs.get_network_status() - assert "error" not in network_status.keys(), "Failed to get network status" - fee = int(network_status.get("result") - .get("network_status") - .get("fees") - .get("0") # zero is the fee key for mob - ) - - """Test Setup """ - - alice = await get_account(0) - bob = await get_account(1) - - pmob_to_send = amount_pmob - alice_status_0 = int( - (await fs.get_account_status(alice.id)) - .get("result") - .get("balance_per_token") - .get("0") - .get("unspent") - ) - - assert alice_status_0 >= pmob_to_send + fee, "Insufficient funds in first account." - - bob_status_0 = int( - (await fs.get_account_status(bob.id)) - .get("result") - .get("balance_per_token") - .get("0") - .get("unspent") - ) - - """ Test action """ - - first_transaction = await fs.build_and_submit_transaction( - alice.id, - recipient_public_address=bob.main_address, - amount={"value": str(pmob_to_send), "token_id": str(0)}, - ) - - """ Check Results """ - - # TODO: replace this with a poll loop that waits a block or two - await asyncio.sleep(15) - alice_status_1 = int( - (await fs.get_account_status(alice.id)) - .get("result") - .get("balance_per_token") - .get("0") - .get("unspent") - ) - bob_status_1 = int( - (await fs.get_account_status(bob.id)) - .get("result") - .get("balance_per_token") - .get("0") - .get("unspent") - ) - - # TODO check that the transaction actually went through/ wait long enough for it to go through - assert alice_status_0 == alice_status_1 + fee + pmob_to_send, "Alice doesn't end with the expected amount" - assert bob_status_1 == bob_status_0 + pmob_to_send, "Bob doesn't end with the expected amount" - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Basic test") - parser.add_argument("config_path", nargs='?', type=str, default=default_config_path) - args = parser.parse_args() - - with open(args.config_path) as json_file: - config = json.load(json_file) - - asyncio.run(main()) diff --git a/python-library/fullservice.py b/python-library/fullservice.py index edd93f6ea..c1cea8e4c 100644 --- a/python-library/fullservice.py +++ b/python-library/fullservice.py @@ -32,7 +32,7 @@ def __init__(self, logLevel = logging.ERROR): self.logger = utils.logger url = utils.get_secret('URL') - async def req(self, request_data: dict) -> dict: + async def req(self, request_data: dict, ) -> dict: logging.info("request: %s", request_data.get("method")) if len(request_data["params"]) > 0: request_data["params"] = { @@ -49,7 +49,8 @@ async def req(self, request_data: dict) -> dict: async def request(self, request_data: dict): request_data = {"jsonrpc": "2.0", "id": "1", **request_data} - self.logger.debug(f"request data: {request_data}") + # TODO: come back and sanitize + # self.logger.debug(f"request data: {request_data}") async with aiohttp.TCPConnector(ssl=ssl_context) as conn: async with aiohttp.ClientSession(connector=conn) as sess: # this can hang (forever?) if there's no full-service at that url diff --git a/tools/build-fs.sh b/tools/build-fs.sh index 014efd778..7947666c2 100755 --- a/tools/build-fs.sh +++ b/tools/build-fs.sh @@ -85,8 +85,8 @@ case ${net} in echo "Setting '${net}' SGX, IAS and enclave values" SGX_MODE=SW IAS_MODE=DEV - CONSENSUS_ENCLAVE_CSS="" - INGEST_ENCLAVE_CSS="" + CONSENSUS_ENCLAVE_CSS="${WORK_DIR}/consensus-enclave.css" + INGEST_ENCLAVE_CSS="${WORK_DIR}/ingest-enclave.css" ;; *) echo "Using current environment's SGX, IAS and enclave values" diff --git a/tools/run-fs.sh b/tools/run-fs.sh index 27cbfc78f..bcab13705 100755 --- a/tools/run-fs.sh +++ b/tools/run-fs.sh @@ -114,9 +114,9 @@ case "${net}" in local) # Set chain id, peer and tx_sources for 2 nodes. MC_CHAIN_ID="${net}" - MC_PEER="mc-insecure://localhost:3200/,mc-insecure://localhost:3201/" + MC_PEER="insecure-mc://localhost:3200/,insecure-mc://localhost:3201/" MC_TX_SOURCE_URL="http://localhost:4566/node-0-ledger/,http://localhost:4566/node-1-ledger/" - MC_FOG_INGEST_ENCLAVE_CSS="${INGEST_ENCLAVE_CSS}" + MC_FOG_INGEST_ENCLAVE_CSS="${WORK_DIR}/ingest-enclave.css" ;; *) echo "Using current environment's SGX, IAS and enclave values"