From bc0dddd0f6474373bc52141067b136b4bd3a6637 Mon Sep 17 00:00:00 2001 From: Pouneh Aghababazadeh Date: Wed, 9 Nov 2022 13:55:50 -0800 Subject: [PATCH 1/9] initial kind of outline for test --- integ-tests/basic.py | 51 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 integ-tests/basic.py diff --git a/integ-tests/basic.py b/integ-tests/basic.py new file mode 100644 index 000000000..738d3a595 --- /dev/null +++ b/integ-tests/basic.py @@ -0,0 +1,51 @@ +# verify that the transaction went through +# the mob went through +# the transaction log updatedx +# Ideally all of the endpoints (v2) that actually hit the mobilecoin network +# +# get_network_status +# get_wallet_status +# build, build_and_submit, build_split_txo .. etc + +import sys +import os +sys.path.append(os.path.abspath("../cli")) + +from fullservice import FullServiceAPIv2 as v2 +import asyncio + +async def main(): + fs = v2() + + network_status = fs.get_network_status().result.network_status + + assert(network_status.network_block_height == network_status.local_block_height) + + alice = fs.import_account().result.account + alice = fs.get_account_status(alice.account_id).result + bob = fs.import_account().result.account + bob = fs.get_account_status(bob.account_id).result + + + first_transaction = fs.build_and_submit_transaction( + alice.account.account_id, + recipient_public_address = bob.account.main_address, + amount = { "value" : str(1), "token_id": str(0)} + ).result + + + total_spent = first_transaction.transactionlog.fee_value + first_transaction.payload_txos[0].value + alice2 = fs.get_account_status(alice.account.account_id).result + assert(alice2.balance_per_token["0"].unspent == + alice.balance_per_token["0"].unspent - total_spent) + + + log = fs.get_transaction_log(first_transaction.transaction_log.id).result + assert(log.status == "tx_status_succeeded") + + + existing_accounts = await fs.get_accounts() + print(existing_accounts) + +if __name__ == '__main__': + asyncio.run(main()) \ No newline at end of file From 2603563877c46e55a09f489c73e63739be0e033d Mon Sep 17 00:00:00 2001 From: Pouneh Aghababazadeh Date: Wed, 9 Nov 2022 15:31:06 -0800 Subject: [PATCH 2/9] Enable run-fs to support local networks running peers on ports 3200 and 3201 --- tools/run-fs.sh | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/tools/run-fs.sh b/tools/run-fs.sh index c7c471593..27f931820 100755 --- a/tools/run-fs.sh +++ b/tools/run-fs.sh @@ -18,6 +18,10 @@ elif [ "$NET" == "alpha" ]; then PEER_DOMAIN="alpha.development.mobilecoin.com/" TX_SOURCE_URL="https://s3-eu-central-1.amazonaws.com/mobilecoin.eu.development.chain" INGEST_SIGSTRUCT_URI="" +elif [ "$NET" == "local" ]; then + NAMESPACE=$NET + TX_SOURCE_URL="https://s3-eu-central-1.amazonaws.com/mobilecoin.eu.development.chain" + INGEST_SIGSTRUCT_URI="" else # TODO: add support for local network echo "Unknown network" @@ -31,11 +35,12 @@ LEDGER_DB_DIR="${WORK_DIR}/ledger-db" INGEST_DOWNLOAD_LOCATION="$WORK_DIR/ingest-enclave.css" mkdir -p ${WORK_DIR} - +# If there is no file at the download location and we know where to download it from if ! test -f "$INGEST_DOWNLOAD_LOCATION" && [ "$INGEST_SIGSTRUCT_URI" != "" ]; then (cd ${WORK_DIR} && curl -O https://enclave-distribution.${NAMESPACE}.mobilecoin.com/${INGEST_SIGSTRUCT_URI}) fi +# If the environment variable isn't set if [ -z "$INGEST_ENCLAVE_CSS" ]; then export INGEST_ENCLAVE_CSS=$INGEST_DOWNLOAD_LOCATION fi @@ -54,13 +59,26 @@ if [ "$2" != "--no-build" ]; then cp $SCRIPT_DIR/../target/release/full-service $WORK_DIR fi + mkdir -p ${WALLET_DB_DIR} -$WORK_DIR/full-service \ - --wallet-db ${WALLET_DB_DIR}/wallet.db \ - --ledger-db ${LEDGER_DB_DIR} \ - --peer mc://node1.${PEER_DOMAIN} \ - --peer mc://node2.${PEER_DOMAIN} \ - --tx-source-url ${TX_SOURCE_URL}/node1.${PEER_DOMAIN} \ - --tx-source-url ${TX_SOURCE_URL}/node2.${PEER_DOMAIN} \ - --fog-ingest-enclave-css $INGEST_ENCLAVE_CSS \ - --chain-id $NET +if [ "$NET" == "local" ]; then + $WORK_DIR/full-service \ + --wallet-db ${WALLET_DB_DIR}/wallet.db \ + --ledger-db ${LEDGER_DB_DIR} \ + --peer insecure-mc://localhost:3200 \ + --peer insecure-mc://localhost:3201 \ + --tx-source-url http://localhost:4566/node-0-ledger \ + --tx-source-url http://localhost:4566/node-1-ledger \ + --fog-ingest-enclave-css $INGEST_ENCLAVE_CSS \ + --chain-id $NET +else + $WORK_DIR/full-service \ + --wallet-db ${WALLET_DB_DIR}/wallet.db \ + --ledger-db ${LEDGER_DB_DIR} \ + --peer mc://node1.${PEER_DOMAIN} \ + --peer mc://node2.${PEER_DOMAIN} \ + --tx-source-url ${TX_SOURCE_URL}/node1.${PEER_DOMAIN} \ + --tx-source-url ${TX_SOURCE_URL}/node2.${PEER_DOMAIN} \ + --fog-ingest-enclave-css $INGEST_ENCLAVE_CSS \ + --chain-id $NET +fi From a3aba40e0dae17efa91c09b0deeef125162a410a Mon Sep 17 00:00:00 2001 From: Pouneh Aghababazadeh Date: Mon, 14 Nov 2022 15:30:25 -0800 Subject: [PATCH 3/9] Make forest_utils compatible with python 3.8 --- cli/config | 4 +++- cli/forest_utils.py | 24 ++++++++++++------------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/cli/config b/cli/config index 73a4dd675..874d82d62 100644 --- a/cli/config +++ b/cli/config @@ -1 +1,3 @@ -URL=http://localhost:9090/wallet/v2 +{ + "URL" : "http://localhost:9090/wallet/v2" +} diff --git a/cli/forest_utils.py b/cli/forest_utils.py index 27f66b0f4..d0cb181a9 100644 --- a/cli/forest_utils.py +++ b/cli/forest_utils.py @@ -1,7 +1,8 @@ #!/usr/bin/python3.9 # Copyright (c) 2021 MobileCoin Inc. # Copyright (c) 2021 The Forest Team -import functools + +import json import logging import os from typing import Optional, cast, Dict @@ -34,25 +35,24 @@ def MuteAiohttp(record: logging.LogRecord) -> bool: # edge cases: # accessing an unset secret loads other variables and potentially overwrites existing ones -def parse_secrets(secrets: str) -> dict[str, str]: - pairs = [ - line.strip().split("=", 1) - for line in secrets.split("\n") - if line and not line.startswith("#") - ] - can_be_a_dict = cast(list[tuple[str, str]], pairs) - return dict(can_be_a_dict) +def parse_secrets(file: str) -> Dict[str, str]: + with open('file') as json_file: + config = json.load(json_file) + return config -# to dump: "\n".join(f"{k}={v}" for k, v in secrets.items()) +# to dump: "\n".join(f"{k}={v}" for k, v in secrets.items()) +env_cache = set() -@functools.cache # don't load the same env more than once def load_secrets(env: Optional[str] = None, overwrite: bool = False) -> None: + if env in env_cache: + return + env_cache.add(env) if not env: env = os.environ.get("ENV", "dev") try: - secrets = parse_secrets(open(f"config").read()) + secrets = parse_secrets("config") if overwrite: new_env = secrets else: From 1108026cc64c9aa027ee9c189d71fe0be7780f2f Mon Sep 17 00:00:00 2001 From: Pouneh Aghababazadeh Date: Mon, 14 Nov 2022 15:31:03 -0800 Subject: [PATCH 4/9] Make sure we can import CLI FS properly --- integ-tests/basic.py | 23 ++++++++++++++++++++++- integ-tests/config | 3 +++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 integ-tests/config diff --git a/integ-tests/basic.py b/integ-tests/basic.py index 738d3a595..2804cc8c9 100644 --- a/integ-tests/basic.py +++ b/integ-tests/basic.py @@ -12,9 +12,30 @@ sys.path.append(os.path.abspath("../cli")) from fullservice import FullServiceAPIv2 as v2 -import asyncio +import asyncio +import json +import subprocess + +with open('config') as json_file: + config = json.load(json_file) + + +def get_account_setup(n = 2): + # copy over files from the running docker instance and import them + containerId = config["local_network_containerid"] + print(containerId) + return + for i in range(n): + filePath = f'/mc-network/sample_data/keys/account_keys_{i}.json' + containerId = "1234" + cmd = f"docker cp {containerId}:{filePath} ." + subprocess.Popen(cmd, shell=True) + + async def main(): + get_account_setup() + return fs = v2() network_status = fs.get_network_status().result.network_status diff --git a/integ-tests/config b/integ-tests/config new file mode 100644 index 000000000..f4bc5c886 --- /dev/null +++ b/integ-tests/config @@ -0,0 +1,3 @@ +{ + "local_network_containerid" : "0125c53b5e55" +} \ No newline at end of file From f0990fe6925d6a43cbe2e667cee81dc5efd6d44d Mon Sep 17 00:00:00 2001 From: Pouneh Aghababazadeh Date: Mon, 14 Nov 2022 16:36:17 -0800 Subject: [PATCH 5/9] Update forest_utils to pull config from same dir, and also make the env part 3.8 compatible --- cli/forest_utils.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/cli/forest_utils.py b/cli/forest_utils.py index d0cb181a9..21dab90b5 100644 --- a/cli/forest_utils.py +++ b/cli/forest_utils.py @@ -7,6 +7,10 @@ import os from typing import Optional, cast, Dict +if __name__ == '__main__': + module_root = os.getcwd() +else: + module_root = os.path.dirname(__file__) def MuteAiohttp(record: logging.LogRecord) -> bool: str_msg = str(getattr(record, "msg", "")) @@ -36,7 +40,7 @@ def MuteAiohttp(record: logging.LogRecord) -> bool: # edge cases: # accessing an unset secret loads other variables and potentially overwrites existing ones def parse_secrets(file: str) -> Dict[str, str]: - with open('file') as json_file: + with open(file) as json_file: config = json.load(json_file) return config @@ -46,18 +50,18 @@ def parse_secrets(file: str) -> Dict[str, str]: env_cache = set() def load_secrets(env: Optional[str] = None, overwrite: bool = False) -> None: - if env in env_cache: + if str(env) in env_cache: return - env_cache.add(env) + env_cache.add(str(env)) if not env: env = os.environ.get("ENV", "dev") try: - secrets = parse_secrets("config") + secrets = parse_secrets(f"{module_root}/config") if overwrite: new_env = secrets else: # mask loaded secrets with existing env - new_env = secrets | os.environ + new_env = {**secrets, **dict(os.environ)} os.environ.update(new_env) except FileNotFoundError: pass From 6d5dd1f1a34c56874bf0aac02dee9991bc333c08 Mon Sep 17 00:00:00 2001 From: Pouneh Aghababazadeh Date: Mon, 14 Nov 2022 16:51:03 -0800 Subject: [PATCH 6/9] Fix request format to not include params if not are supplied --- cli/fullservice.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cli/fullservice.py b/cli/fullservice.py index a55f51df7..057ae96e8 100644 --- a/cli/fullservice.py +++ b/cli/fullservice.py @@ -29,9 +29,12 @@ class Request: url = utils.get_secret('URL') async def req(self, request_data: dict) -> dict: logging.info("request: %s", request_data.get("method")) - request_data["params"] = { - k: v for k, v in request_data["params"].items() if v - } # handle optional params + if len(request_data["params"]) > 0: + request_data["params"] = { + k: v for k, v in request_data["params"].items() if v + } # handle optional params + else: + del request_data["params"] response_data = await self.request(request_data) if "error" in str(response_data): logging.error(response_data) From 0112a1d3963689e3a4b48135bc403991b0e38c30 Mon Sep 17 00:00:00 2001 From: Pouneh Aghababazadeh Date: Mon, 14 Nov 2022 17:07:11 -0800 Subject: [PATCH 7/9] Basic test --- cli/fullservice.py | 10 ++++++++-- integ-tests/basic.py | 46 +++----------------------------------------- integ-tests/config | 3 ++- 3 files changed, 13 insertions(+), 46 deletions(-) diff --git a/cli/fullservice.py b/cli/fullservice.py index 057ae96e8..8ee2395a8 100644 --- a/cli/fullservice.py +++ b/cli/fullservice.py @@ -26,7 +26,10 @@ class Request: + def __init__(self, logLevel = logging.ERROR): + logging.basicConfig( level=logLevel) url = utils.get_secret('URL') + logging.info("Woohoo error") async def req(self, request_data: dict) -> dict: logging.info("request: %s", request_data.get("method")) if len(request_data["params"]) > 0: @@ -36,10 +39,10 @@ async def req(self, request_data: dict) -> dict: else: del request_data["params"] response_data = await self.request(request_data) - if "error" in str(response_data): + if "error" in str(response_data) or "InvalidRequest" in str(response_data): logging.error(response_data) else: - print(response_data) + logging.info(response_data) return response_data async def request(self, request_data: dict): @@ -58,6 +61,9 @@ async def request(self, request_data: dict): class FullServiceAPIv2(Request): + def __init__(self, logLevel=logging.ERROR): + logging.basicConfig( level=logLevel) + async def assign_address_for_account(self, account_id, metadata=""): return await self.req( { diff --git a/integ-tests/basic.py b/integ-tests/basic.py index 2804cc8c9..125513cea 100644 --- a/integ-tests/basic.py +++ b/integ-tests/basic.py @@ -15,58 +15,18 @@ import asyncio import json import subprocess +from pathlib import Path with open('config') as json_file: config = json.load(json_file) -def get_account_setup(n = 2): - # copy over files from the running docker instance and import them - containerId = config["local_network_containerid"] - print(containerId) - return - for i in range(n): - filePath = f'/mc-network/sample_data/keys/account_keys_{i}.json' - containerId = "1234" - cmd = f"docker cp {containerId}:{filePath} ." - subprocess.Popen(cmd, shell=True) - - - async def main(): - get_account_setup() - return fs = v2() - network_status = fs.get_network_status().result.network_status - - assert(network_status.network_block_height == network_status.local_block_height) - - alice = fs.import_account().result.account - alice = fs.get_account_status(alice.account_id).result - bob = fs.import_account().result.account - bob = fs.get_account_status(bob.account_id).result - - - first_transaction = fs.build_and_submit_transaction( - alice.account.account_id, - recipient_public_address = bob.account.main_address, - amount = { "value" : str(1), "token_id": str(0)} - ).result - - - total_spent = first_transaction.transactionlog.fee_value + first_transaction.payload_txos[0].value - alice2 = fs.get_account_status(alice.account.account_id).result - assert(alice2.balance_per_token["0"].unspent == - alice.balance_per_token["0"].unspent - total_spent) - - - log = fs.get_transaction_log(first_transaction.transaction_log.id).result - assert(log.status == "tx_status_succeeded") + network_status = await fs.get_network_status() + print(network_status) - existing_accounts = await fs.get_accounts() - print(existing_accounts) - if __name__ == '__main__': asyncio.run(main()) \ No newline at end of file diff --git a/integ-tests/config b/integ-tests/config index f4bc5c886..eef0f0839 100644 --- a/integ-tests/config +++ b/integ-tests/config @@ -1,3 +1,4 @@ { - "local_network_containerid" : "0125c53b5e55" + "local_network_containerid" : "0125c53b5e55", + "test_dump" : "./test_dump" } \ No newline at end of file From 78eb5041d395e69d9bbe83930284e078b77a1bd6 Mon Sep 17 00:00:00 2001 From: Pouneh Aghababazadeh Date: Mon, 14 Nov 2022 17:29:53 -0800 Subject: [PATCH 8/9] Remove config file contents since we don't need them at this pt --- integ-tests/config | 2 -- 1 file changed, 2 deletions(-) diff --git a/integ-tests/config b/integ-tests/config index eef0f0839..7a73a41bf 100644 --- a/integ-tests/config +++ b/integ-tests/config @@ -1,4 +1,2 @@ { - "local_network_containerid" : "0125c53b5e55", - "test_dump" : "./test_dump" } \ No newline at end of file From 9fe377c3b53206c8cb921e04aa75d54c639ad405 Mon Sep 17 00:00:00 2001 From: Pouneh Aghababazadeh Date: Mon, 14 Nov 2022 17:41:11 -0800 Subject: [PATCH 9/9] minor update to run script --- tools/run-fs.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/run-fs.sh b/tools/run-fs.sh index 27f931820..f0f70b245 100755 --- a/tools/run-fs.sh +++ b/tools/run-fs.sh @@ -20,12 +20,10 @@ elif [ "$NET" == "alpha" ]; then INGEST_SIGSTRUCT_URI="" elif [ "$NET" == "local" ]; then NAMESPACE=$NET - TX_SOURCE_URL="https://s3-eu-central-1.amazonaws.com/mobilecoin.eu.development.chain" INGEST_SIGSTRUCT_URI="" else - # TODO: add support for local network echo "Unknown network" - echo "Usage: run-fs.sh {main|test|alpha} [--no-build]" + echo "Usage: run-fs.sh {main|test|alpha|local} [--no-build]" exit 1 fi