From 9e06b07b2540fec3294fa86e5ddc34a4f0aa6472 Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Thu, 6 May 2021 12:55:50 -0400 Subject: [PATCH 01/29] gm first stab --- scripts/gm/bin/gm | 105 +++++++++++ scripts/gm/bin/lib-gm | 357 +++++++++++++++++++++++++++++++++++ scripts/gm/bin/shell-support | 71 +++++++ scripts/gm/gm.toml | 14 ++ 4 files changed, 547 insertions(+) create mode 100755 scripts/gm/bin/gm create mode 100644 scripts/gm/bin/lib-gm create mode 100644 scripts/gm/bin/shell-support create mode 100644 scripts/gm/gm.toml diff --git a/scripts/gm/bin/gm b/scripts/gm/bin/gm new file mode 100755 index 0000000000..b36e3591e8 --- /dev/null +++ b/scripts/gm/bin/gm @@ -0,0 +1,105 @@ +#!/usr/bin/env sh + +set -eu + +# Load lib-gm either from the local folder or from the global $HOME/.gm/bin/lib-gm +export SCRIPT_DIR="${0%%gm}" +export LOCAL_LIB_GM="${SCRIPT_DIR}lib-gm" +if [ -f "$LOCAL_LIB_GM" ]; then + source "$LOCAL_LIB_GM" +elif [ -f "$HOME/.gm/bin/lib-gm" ]; then + source $HOME/.gm/bin/lib-gm +else + echo "ERROR: could not find lib-gm, exiting..." + exit 1 +fi + +enforce_requirements + +usage() { + echo "Gaiad Manager $(version)" + echo + echo "Usage:" + echo " gm [[]...]" + echo + echo "COMMANDS DESCRIPTION" + echo + echo "help print this help and exit" + echo "install install the script for the local user" + echo "keys print the keys of validator nodes" + echo "log print the log of a node" + echo "ports print the ports of a (running) node" + echo "start start one or more nodes (starts all nodes if no parameter is given)" + echo "status print the status of nodes" + echo "stop stop one or more nodes (stops all nodes if no parameter is given)" + echo "version print the application version" + echo +} + +case "${1:-help}" in + help) + usage + exit 0 + ;; + install) + install + ;; + keys) + load_config + echo "not implemented" + ;; + log) + load_config + shift + echo "not implemented" + ;; + ports) + load_config + echo "not implemented" + ;; + start) + load_config + if [ $# -eq 1 ]; then + NODES_TO_START="$ALL_NODES" + else + shift + NODES_TO_START="$@" + fi + for i in $NODES_TO_START + do + if [ ! -d "$(get_home_dir "$i")" ]; then + create "$i" || continue + fi + if get_auto_maintain_config "$i"; then + configure "$i" + fi + start "$i" + done + ;; + status) + load_config + status + ;; + stop) + load_config + if [ $# -eq 1 ]; then + NODES_TO_STOP="$ALL_NODES" + else + shift + NODES_TO_STOP="$@" + fi + for i in $NODES_TO_STOP + do + stop "$i" + done + ;; + version) + version + exit 0 + ;; + *) + usage + exit_with_error "could not understand the command \"$1\"" + ;; +esac + diff --git a/scripts/gm/bin/lib-gm b/scripts/gm/bin/lib-gm new file mode 100644 index 0000000000..c01b2ff0fc --- /dev/null +++ b/scripts/gm/bin/lib-gm @@ -0,0 +1,357 @@ +#!/usr/bin/env sh +#set -x +set -eu + +version() { + echo "v0.0.1" +} + +install() { + mkdir -p "${HOME}/.gm/bin" + cp "$0" "${HOME}/.gm/bin/gm" + chmod 755 "${HOME}/.gm/bin/gm" + cp "${0%%gm}lib-gm" "${HOME}/.gm/bin/lib-gm" + chmod 644 "${HOME}/.gm/bin/lib-gm" + CONFIG_FILE="${HOME}/.gm/gm.toml" + write_default_config + if [ -z "$(which stoml)" ]; then + warn "missing mandatory stoml, install it from https://github.com/freshautomations/stoml/releases" + fi + if [ -z "$(which sconfig)" ]; then + warn "missing mandatory sconfig, install it from https://github.com/freshautomations/sconfig/releases" + fi + if [ ! -d /usr/local/etc/bash_completion.d ]; then + warn "run \"brew install bash-completion\" to install optional bash completion" + fi + echo "Please add \"source $HOME/.gm/bin/shell-support\" to your .bash_profile to access the tool and restart your shell." +} + +enforce_requirements() { + if [ -z "$(which stoml)" ]; then + exit_with_error "missing stoml, install it from https://github.com/freshautomations/stoml/releases" + fi + if [ -z "$(which sconfig)" ]; then + exit_with_error "missing sconfig, install it from https://github.com/freshautomations/sconfig/releases" + fi +} + +debug() { + if [ -n "${DEBUG:-}" ]; then + echo "DEBUG: $@" + fi +} + +warn() { + echo "WARNING: $@" +} + +exit_with_error() { + echo "ERROR: $@, exiting..." + return 1 +} + +usage() { + echo "Gaiad Manager $(version)" + echo + echo "Usage:" + echo " gm [[]...]" + echo + echo "COMMANDS DESCRIPTION" + echo + echo "help print this help and exit" + echo "install install the script for the local user" + echo "keys print the keys of validator nodes" + echo "log print the log of a node" + echo "ports print the ports of a (running) node" + echo "start start one or more nodes (starts all nodes if no parameter is given)" + echo "status print the status of nodes" + echo "stop stop one or more nodes (stops all nodes if no parameter is given)" + echo +} + +# Is string A in space-separated list B? +a_in_b() { + test -n "$(echo "$2" | grep '\(^\| \+\)'"${1}"'\( \+\|$\)')" +} + +# Todo: (high-prio) fix sconfig so it reads empty sections too +load_config() { + FRESH_CONFIG=no + if [ -f "${SCRIPT_DIR}gm.toml" ]; then + export CONFIG_FILE="${SCRIPT_DIR}gm.toml" + elif [ -f "${HOME}/.gm/gm.toml" ]; then + export CONFIG_FILE="${HOME}/.gm/gm.toml" + else + FRESH_CONFIG=yes + if [ -d "${HOME}/.gm" ]; then + export CONFIG_FILE="${HOME}/.gm/gm.toml" + else + export CONFIG_FILE="${SCRIPT_DIR}gm.toml" + fi + fi + export GLOBAL_GAIAD_BINARY="$(eval echo "$(stoml -q "$CONFIG_FILE" global.gaiad_binary || echo "$(which gaiad)")")" + export GLOBAL_PORTS_START_AT="$(stoml -q "$CONFIG_FILE" global.ports_start_at || echo 27000)" + export GLOBAL_HOME_DIR="$(eval echo "$(stoml -q "$CONFIG_FILE" global.home_dir || echo "${HOME}/.gm")")" + export GLOBAL_AUTO_MAINTAIN_CONFIG="$(stoml -q "$CONFIG_FILE" global.auto_maintain_config || echo "true")" + if [ "$FRESH_CONFIG" = "yes" ]; then + write_default_config + export VALIDATORS="" + export FULL_NODES="" + else + stoml "$CONFIG_FILE" global 1> /dev/null || exit_with_error invalid config file. Make sure all strings are quoted + test -x "$GLOBAL_GAIAD_BINARY" || exit_with_error "gaiad binary cannot be executed: $GLOBAL_GAIAD_BINARY" + RAW_SECTIONS="$(stoml -q "$CONFIG_FILE" . || echo "")" + VALIDATORS="" + RAW_NODES="" + for i in $RAW_SECTIONS + do + if [ "$i" = "global" ]; then + continue + fi + if [ -z "$(stoml "$CONFIG_FILE" ${i}.network)" ]; then + VALIDATORS="$VALIDATORS $i" + else + RAW_NODES="$RAW_NODES $i" + fi + done + FULL_NODES="" + for i in $RAW_NODES + do + NODE_NETWORK="$(stoml "$CONFIG_FILE" ${i}.network)" + if ! a_in_b "${NODE_NETWORK}" "$VALIDATORS"; then + warn "invalid full node: $i, invalid network entry: ${NODE_NETWORK}, skipping..." + continue + fi + FULL_NODES="${FULL_NODES} $i" + done + export VALIDATORS + export FULL_NODES + fi + export ALL_NODES="$VALIDATORS $FULL_NODES" +} + +write_default_config() { + set +e + cat < "$CONFIG_FILE" +[global] +gaiad_binary="$GLOBAL_GAIAD_BINARY" +ports_start_at=$GLOBAL_PORTS_START_AT +home_dir="$GLOBAL_HOME_DIR" +auto_maintain_config=$GLOBAL_AUTO_MAINTAIN_CONFIG +EOF + set -e +} + +get_gaiad_binary() { + RESULT="$(stoml -q "$CONFIG_FILE" ${1}.gaiad_binary)" + if [ -z "$RESULT" ]; then + echo "$GLOBAL_GAIAD_BINARY" + else + echo "$(eval echo "$RESULT")" + fi +} + +get_ports_start_at() { + RESULT="$(stoml -q "$CONFIG_FILE" ${1}.ports_start_at)" + if [ -z "$RESULT" ]; then + i=0 + for j in $ALL_NODES + do + if [ "$j" = "$1" ]; then + break + fi + i=$((i+1)) + done + echo "$((GLOBAL_PORTS_START_AT+10*i))" + else + echo "$RESULT" + fi +} + +get_rpc_port() { + echo "$(get_ports_start_at "$1")" +} + +get_app_port() { + echo "$(($(get_ports_start_at "$1")+1))" +} + +get_grpc_port() { + echo "$(($(get_ports_start_at "$1")+2))" +} + +get_p2p_port() { + echo "$(($(get_ports_start_at "$1")+3))" +} + +get_pprof_port() { + echo "$(($(get_ports_start_at "$1")+4))" +} + +get_home_dir() { + RESULT="$(stoml -q "$CONFIG_FILE" ${1}.home_dir)" + if [ -z "$RESULT" ]; then + echo "$GLOBAL_HOME_DIR/$1" + else + echo "$(eval echo "$RESULT")" + fi +} + +get_auto_maintain_config() { + RESULT="$(stoml -q "$CONFIG_FILE" ${1}.auto_maintain_config)" + if [ -z "$RESULT" ]; then + test "$GLOBAL_AUTO_MAINTAIN_CONFIG" = "true" + else + test "$RESULT" = "true" + fi +} + +get_network() { + RESULT="$(stoml -q "$CONFIG_FILE" ${1}.network)" + if [ -z "$RESULT" ]; then + exit_on_error "Network not found for node ${1}" + else + if ! a_in_b "$RESULT" "$VALIDATORS"; then + return 1 + fi + fi + echo "$RESULT" +} + +get_node_id() { + GAIAD="$(get_gaiad_binary "$1")" + HOME_DIR="$(get_home_dir "$1")" + $GAIAD tendermint show-node-id --home "$HOME_DIR" +} + +create() { + i="$1" + echo "Creating $i config..." + GAIAD="$(get_gaiad_binary "$i")" + HOME_DIR="$(get_home_dir "$i")" + HOME_ROOT="${HOME_DIR%%${i}}" + if a_in_b "$i" "$VALIDATORS"; then + EXEC_RESULT="$("$GAIAD" testnet --chain-id "$i" --keyring-backend test --node-dir-prefix "${i}v" -o "$HOME_ROOT" --node-daemon-home . --v 1 2>&1)" + if [ "$EXEC_RESULT" != "Successfully initialized 1 node directories" ]; then + warn "could not create config for ${i}: \"$EXEC_RESULT\", skipping..." + return 1 + else + mv "${HOME_ROOT}/${i}v0" "${HOME_DIR}" + mv "${HOME_ROOT}/gentxs" "${HOME_DIR}" + $GAIAD keys add "wallet" --keyring-backend test --keyring-dir "${HOME_DIR}" --output json > "${HOME_DIR}/wallet_seed.json" + $GAIAD add-genesis-account "wallet" "10000000stake,10000${i}coin" --keyring-backend test --home "${HOME_DIR}" + fi + else + NETWORK="$(get_network "$i")" + EXEC_RESULT="$("$GAIAD" testnet --chain-id "$NETWORK" --keyring-backend test --node-dir-prefix "${i}n" -o "$HOME_ROOT" --node-daemon-home . --v 1 2>&1)" + if [ "$EXEC_RESULT" != "Successfully initialized 1 node directories" ]; then + warn "could not create config for ${i}: \"$EXEC_RESULT\", skipping..." + return 1 + fi + mv "${HOME_ROOT}/${i}n0" "${HOME_DIR}" + rm -rf "${HOME_ROOT}/gentxs" + # Todo: Maybe replace the below with calling the configure function. + NETWORK_HOME_DIR="$(get_home_dir "$NETWORK")" + cp "$NETWORK_HOME_DIR/config/genesis.json" "$HOME_DIR/config/genesis.json" + NETWORK_NODE="$(get_node_id "$NETWORK")@localhost:$(get_p2p_port "$NETWORK")" + sconfig "$HOME_DIR/config/config.toml" "p2p.persistent_peers=$NETWORK_NODE" 1> /dev/null + if get_auto_maintain_config "$NETWORK"; then + NODE_ID="$(get_node_id "$i")" + sconfig "$NETWORK_HOME_DIR/config/config.toml" "p2p.unconditional_peer_ids=$NODE_ID" 1> /dev/null + # Todo: Make this more robust: if two full nodes connect to the same validator, this override each other + fi + fi + sconfig "$HOME_DIR/config/config.toml" p2p.addr_book_strict=false 1> /dev/null + sconfig "$HOME_DIR/config/config.toml" p2p.allow_duplicate_ip=true 1> /dev/null +} + +configure() { + HOME_DIR="$(get_home_dir "$1")" + # Todo: rpc.pprof_laddr=6060, maybe needs moving? + # Todo: rpc.grpc also? + P2P="$(get_p2p_port "$1")" + RPC="$(get_rpc_port "$1")" + APP="$(get_app_port "$1")" + GRPC="$(get_grpc_port "$1")" + PPROF="$(get_pprof_port "$1")" + sconfig "$HOME_DIR/config/config.toml" "p2p.laddr=tcp://0.0.0.0:${P2P}" 1> /dev/null + sconfig "$HOME_DIR/config/config.toml" "rpc.laddr=tcp://0.0.0.0:${RPC}" 1> /dev/null + sconfig "$HOME_DIR/config/config.toml" "rpc.pprof_laddr=localhost:${PPROF}" 1> /dev/null + # Todo: sconfig doesn't support it yet + #sconfig "$HOME_DIR/config/app.toml" "api.address=tcp://0.0.0.0:${APP}" 1> /dev/null + #sconfig "$HOME_DIR/config/app.toml" "grpc.address=0.0.0.0:${GRPC}" 1> /dev/null + if ! a_in_b "$i" "$VALIDATORS"; then + NETWORK="$(get_network "$1")" + NETWORK_HOME_DIR="$(get_home_dir "$NETWORK")" + cp "$NETWORK_HOME_DIR/config/genesis.json" "$HOME_DIR/config/genesis.json" + NETWORK_NODE="$(get_node_id "$NETWORK")@localhost:$(get_p2p_port "$NETWORK")" + sconfig "$HOME_DIR/config/config.toml" "p2p.persistent_peers=$NETWORK_NODE" 1> /dev/null + if get_auto_maintain_config "$NETWORK"; then + NODE_ID="$(get_node_id "$1")" + sconfig "$NETWORK_HOME_DIR/config/config.toml" "p2p.unconditional_peer_ids=$NODE_ID" 1> /dev/null + # Todo: Make this more robust: if two full nodes connect to the same validator, this override each other + fi + fi +} + +# Todo: (high-prio) fix sconfig to be able to map in app.toml +start() { + GAIAD="$(get_gaiad_binary "$1")" + HOME_DIR="$(get_home_dir "$1")" + GAIAD_LOG="${HOME_DIR}/log" + nohup $GAIAD start --x-crisis-skip-assert-invariants --home "$HOME_DIR" > "$GAIAD_LOG" 2>&1 & + GAIAD_PID=$! + echo "$GAIAD_PID" > "$HOME_DIR/pid" + echo "$i started, PID: $GAIAD_PID, LOG: $GAIAD_LOG" +} + +stop() { + HOME_DIR="$(get_home_dir "$1")" + GAIAD_BINARY="$(get_gaiad_binary "$1")" + test -f "${HOME_DIR}/pid" + GAIAD_PID="$(cat "${HOME_DIR}/pid")" + #COMM="$(ps -p "$GAIAD_PID" -o comm | tail -1)" + echo "Stopping $1 with PID $GAIAD_PID..." + kill -TERM "$GAIAD_PID" 2> /dev/null || echo " No process with PID $GAIAD_PID. Cleaning up run config." + sleep 2 + # Todo: make this more robust by checking if the process was killed properly + rm -f "$HOME_DIR/pid" +} + +print_header_line() { + echo "NODE PID RPC APP GRPC HOME_DIR" +} + +print_status_line() { + I="${2:-}" + HOME_DIR="$(get_home_dir "$1")" + GAIAD_PID_FILE="${HOME_DIR}/pid" + if [ -f "$GAIAD_PID_FILE" ]; then + GAIAD_PID="$(cat "$GAIAD_PID_FILE")" + if [ $(ps -p "$GAIAD_PID" -o pid | wc -l) -eq 2 ]; then + echo "$I$1 $GAIAD_PID $(get_rpc_port "$1") $(get_app_port "$1") $(get_grpc_port "$1") $HOME_DIR" + else + echo "$I$1 ($GAIAD_PID) - - - $HOME_DIR" + fi + else + echo "$I$1 - - - - $HOME_DIR" + fi +} + +status() { + print_header_line + for i in $VALIDATORS + do + print_status_line "$i" + for j in $FULL_NODES + do + NETWORK="$(get_network "$j")" + if [ "$i" = "$NETWORK" ]; then + print_status_line "$j" "-" + else + continue + fi + done + done +} + diff --git a/scripts/gm/bin/shell-support b/scripts/gm/bin/shell-support new file mode 100644 index 0000000000..f7498315e9 --- /dev/null +++ b/scripts/gm/bin/shell-support @@ -0,0 +1,71 @@ +# shell support for gm + +alias gm="$HOME/.gm/bin/gm" + +__gm_get_nodes() { + if [ -f "$HOME/.gm/gm.toml" ]; then + #Simplified grepping of sections + local NODES="" + for i in $(grep '^\[.\+\]$' "$HOME/.gm/gm.toml") + do + if [ "$i" = "[global]" ]; then + continue + fi + o="${i##[}" + NODES="$NODES ${o%%]}" + done + echo "$NODES" + fi +} + +_gm_nodes() { + local cur="${COMP_WORDS[COMP_CWORD]}" + local cmds="$(__gm_get_nodes)" + while read -r line; do COMPREPLY+=("$line"); done < <(compgen -W "$cmds" -- "$cur") +} + +_gm_complete_commands() { + local cur="${COMP_WORDS[COMP_CWORD]}" + local cmds="help install keys log ports start status stop version" + while read -r line; do COMPREPLY+=("$line"); done < <(compgen -W "$cmds" -- "$cur") +} + +_gm() { + local i=1 cmd + + # find the subcommand + while [[ "$i" -lt "$COMP_CWORD" ]] + do + local s="${COMP_WORDS[i]}" + case "$s" in + --*) + cmd="$s" + break + ;; + -*) + ;; + *) + cmd="$s" + break + ;; + esac + (( i++ )) + done + + if [[ "$i" -eq "$COMP_CWORD" ]] + then + _gm_complete_commands + return + fi + + # subcommands have their own completion functions + case "$cmd" in + log) _gm_nodes ;; + start) _gm_nodes ;; + stop) _gm_nodes ;; + *) ;; + esac +} + +complete -o bashdefault -o default -F _gm gm + diff --git a/scripts/gm/gm.toml b/scripts/gm/gm.toml new file mode 100644 index 0000000000..0445d7b52c --- /dev/null +++ b/scripts/gm/gm.toml @@ -0,0 +1,14 @@ +# Example configuration with one validator and one full node +# Todo: document this fully +[global] +gaiad_binary="$GOPATH/bin/gaiad" +ports_start_at=27000 +home_dir="$HOME/.gm" +auto_maintain_config=true + +[validator1] +#auto_maintain_config=false +fake_entry="" + +[node1] +network="validator1" From 5ffaa73e54a9ab6fa808878453bd916f102cd5bc Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Thu, 6 May 2021 22:16:07 -0400 Subject: [PATCH 02/29] Shellism fixes --- scripts/gm/bin/gm | 6 ++--- scripts/gm/bin/lib-gm | 63 +++++++++++++++++++++++-------------------- scripts/gm/gm.toml | 5 ++-- 3 files changed, 40 insertions(+), 34 deletions(-) diff --git a/scripts/gm/bin/gm b/scripts/gm/bin/gm index b36e3591e8..c4b1550012 100755 --- a/scripts/gm/bin/gm +++ b/scripts/gm/bin/gm @@ -46,16 +46,16 @@ case "${1:-help}" in ;; keys) load_config - echo "not implemented" + echo "Todo: not implemented" ;; log) load_config shift - echo "not implemented" + echo "Todo: not implemented" ;; ports) load_config - echo "not implemented" + echo "Todo: not implemented" ;; start) load_config diff --git a/scripts/gm/bin/lib-gm b/scripts/gm/bin/lib-gm index c01b2ff0fc..1345046b99 100644 --- a/scripts/gm/bin/lib-gm +++ b/scripts/gm/bin/lib-gm @@ -37,16 +37,16 @@ enforce_requirements() { debug() { if [ -n "${DEBUG:-}" ]; then - echo "DEBUG: $@" + echo "DEBUG: $*" fi } warn() { - echo "WARNING: $@" + echo "WARNING: $*" } exit_with_error() { - echo "ERROR: $@, exiting..." + echo "ERROR: $*, exiting..." return 1 } @@ -74,7 +74,7 @@ a_in_b() { test -n "$(echo "$2" | grep '\(^\| \+\)'"${1}"'\( \+\|$\)')" } -# Todo: (high-prio) fix sconfig so it reads empty sections too +# Todo: fix "stoml" so it reads empty sections too - currently we work around this with the mandatory comment. load_config() { FRESH_CONFIG=no if [ -f "${SCRIPT_DIR}gm.toml" ]; then @@ -89,14 +89,18 @@ load_config() { export CONFIG_FILE="${SCRIPT_DIR}gm.toml" fi fi - export GLOBAL_GAIAD_BINARY="$(eval echo "$(stoml -q "$CONFIG_FILE" global.gaiad_binary || echo "$(which gaiad)")")" - export GLOBAL_PORTS_START_AT="$(stoml -q "$CONFIG_FILE" global.ports_start_at || echo 27000)" - export GLOBAL_HOME_DIR="$(eval echo "$(stoml -q "$CONFIG_FILE" global.home_dir || echo "${HOME}/.gm")")" - export GLOBAL_AUTO_MAINTAIN_CONFIG="$(stoml -q "$CONFIG_FILE" global.auto_maintain_config || echo "true")" + GLOBAL_GAIAD_BINARY="$(eval echo "$(stoml -q "$CONFIG_FILE" global.gaiad_binary || which gaiad)")" + GLOBAL_PORTS_START_AT="$(stoml -q "$CONFIG_FILE" global.ports_start_at || echo 27000)" + GLOBAL_HOME_DIR="$(eval echo "$(stoml -q "$CONFIG_FILE" global.home_dir || echo "${HOME}/.gm")")" + GLOBAL_AUTO_MAINTAIN_CONFIG="$(stoml -q "$CONFIG_FILE" global.auto_maintain_config || echo "true")" + export GLOBAL_GAIAD_BINARY + export GLOBAL_PORTS_START_AT + export GLOBAL_HOME_DIR + export GLOBAL_AUTO_MAINTAIN_CONFIG if [ "$FRESH_CONFIG" = "yes" ]; then write_default_config - export VALIDATORS="" - export FULL_NODES="" + VALIDATORS="" + FULL_NODES="" else stoml "$CONFIG_FILE" global 1> /dev/null || exit_with_error invalid config file. Make sure all strings are quoted test -x "$GLOBAL_GAIAD_BINARY" || exit_with_error "gaiad binary cannot be executed: $GLOBAL_GAIAD_BINARY" @@ -108,7 +112,7 @@ load_config() { if [ "$i" = "global" ]; then continue fi - if [ -z "$(stoml "$CONFIG_FILE" ${i}.network)" ]; then + if [ -z "$(stoml "$CONFIG_FILE" "${i}.network")" ]; then VALIDATORS="$VALIDATORS $i" else RAW_NODES="$RAW_NODES $i" @@ -117,16 +121,16 @@ load_config() { FULL_NODES="" for i in $RAW_NODES do - NODE_NETWORK="$(stoml "$CONFIG_FILE" ${i}.network)" + NODE_NETWORK="$(stoml "$CONFIG_FILE" "${i}.network")" if ! a_in_b "${NODE_NETWORK}" "$VALIDATORS"; then warn "invalid full node: $i, invalid network entry: ${NODE_NETWORK}, skipping..." continue fi FULL_NODES="${FULL_NODES} $i" done - export VALIDATORS - export FULL_NODES fi + export VALIDATORS + export FULL_NODES export ALL_NODES="$VALIDATORS $FULL_NODES" } @@ -138,21 +142,22 @@ gaiad_binary="$GLOBAL_GAIAD_BINARY" ports_start_at=$GLOBAL_PORTS_START_AT home_dir="$GLOBAL_HOME_DIR" auto_maintain_config=$GLOBAL_AUTO_MAINTAIN_CONFIG +comment="Please add comments to all node definitions. (Mandatory.)" EOF set -e } get_gaiad_binary() { - RESULT="$(stoml -q "$CONFIG_FILE" ${1}.gaiad_binary)" + RESULT="$(stoml -q "$CONFIG_FILE" "${1}.gaiad_binary")" if [ -z "$RESULT" ]; then echo "$GLOBAL_GAIAD_BINARY" else - echo "$(eval echo "$RESULT")" + eval echo "$RESULT" fi } get_ports_start_at() { - RESULT="$(stoml -q "$CONFIG_FILE" ${1}.ports_start_at)" + RESULT="$(stoml -q "$CONFIG_FILE" "${1}.ports_start_at")" if [ -z "$RESULT" ]; then i=0 for j in $ALL_NODES @@ -169,7 +174,7 @@ get_ports_start_at() { } get_rpc_port() { - echo "$(get_ports_start_at "$1")" + get_ports_start_at "$1" } get_app_port() { @@ -189,16 +194,16 @@ get_pprof_port() { } get_home_dir() { - RESULT="$(stoml -q "$CONFIG_FILE" ${1}.home_dir)" + RESULT="$(stoml -q "$CONFIG_FILE" "${1}.home_dir")" if [ -z "$RESULT" ]; then echo "$GLOBAL_HOME_DIR/$1" else - echo "$(eval echo "$RESULT")" + eval echo "$RESULT" fi } get_auto_maintain_config() { - RESULT="$(stoml -q "$CONFIG_FILE" ${1}.auto_maintain_config)" + RESULT="$(stoml -q "$CONFIG_FILE" "${1}.auto_maintain_config")" if [ -z "$RESULT" ]; then test "$GLOBAL_AUTO_MAINTAIN_CONFIG" = "true" else @@ -207,7 +212,7 @@ get_auto_maintain_config() { } get_network() { - RESULT="$(stoml -q "$CONFIG_FILE" ${1}.network)" + RESULT="$(stoml -q "$CONFIG_FILE" "${1}.network")" if [ -z "$RESULT" ]; then exit_on_error "Network not found for node ${1}" else @@ -271,8 +276,8 @@ configure() { # Todo: rpc.grpc also? P2P="$(get_p2p_port "$1")" RPC="$(get_rpc_port "$1")" - APP="$(get_app_port "$1")" - GRPC="$(get_grpc_port "$1")" + #APP="$(get_app_port "$1")" + #GRPC="$(get_grpc_port "$1")" PPROF="$(get_pprof_port "$1")" sconfig "$HOME_DIR/config/config.toml" "p2p.laddr=tcp://0.0.0.0:${P2P}" 1> /dev/null sconfig "$HOME_DIR/config/config.toml" "rpc.laddr=tcp://0.0.0.0:${RPC}" 1> /dev/null @@ -289,17 +294,17 @@ configure() { if get_auto_maintain_config "$NETWORK"; then NODE_ID="$(get_node_id "$1")" sconfig "$NETWORK_HOME_DIR/config/config.toml" "p2p.unconditional_peer_ids=$NODE_ID" 1> /dev/null - # Todo: Make this more robust: if two full nodes connect to the same validator, this override each other + # Todo: Make this more robust: if two full nodes connect to the same validator, don't override each other fi fi } -# Todo: (high-prio) fix sconfig to be able to map in app.toml +# Todo: (high-priority) fix sconfig to be able to map in app.toml start() { GAIAD="$(get_gaiad_binary "$1")" HOME_DIR="$(get_home_dir "$1")" GAIAD_LOG="${HOME_DIR}/log" - nohup $GAIAD start --x-crisis-skip-assert-invariants --home "$HOME_DIR" > "$GAIAD_LOG" 2>&1 & + nohup "$GAIAD" start --x-crisis-skip-assert-invariants --home "$HOME_DIR" > "$GAIAD_LOG" 2>&1 & GAIAD_PID=$! echo "$GAIAD_PID" > "$HOME_DIR/pid" echo "$i started, PID: $GAIAD_PID, LOG: $GAIAD_LOG" @@ -307,7 +312,7 @@ start() { stop() { HOME_DIR="$(get_home_dir "$1")" - GAIAD_BINARY="$(get_gaiad_binary "$1")" + #GAIAD_BINARY="$(get_gaiad_binary "$1")" test -f "${HOME_DIR}/pid" GAIAD_PID="$(cat "${HOME_DIR}/pid")" #COMM="$(ps -p "$GAIAD_PID" -o comm | tail -1)" @@ -328,7 +333,7 @@ print_status_line() { GAIAD_PID_FILE="${HOME_DIR}/pid" if [ -f "$GAIAD_PID_FILE" ]; then GAIAD_PID="$(cat "$GAIAD_PID_FILE")" - if [ $(ps -p "$GAIAD_PID" -o pid | wc -l) -eq 2 ]; then + if [ "$(ps -p "$GAIAD_PID" -o pid | wc -l)" -eq 2 ]; then echo "$I$1 $GAIAD_PID $(get_rpc_port "$1") $(get_app_port "$1") $(get_grpc_port "$1") $HOME_DIR" else echo "$I$1 ($GAIAD_PID) - - - $HOME_DIR" diff --git a/scripts/gm/gm.toml b/scripts/gm/gm.toml index 0445d7b52c..f614ecead0 100644 --- a/scripts/gm/gm.toml +++ b/scripts/gm/gm.toml @@ -5,10 +5,11 @@ gaiad_binary="$GOPATH/bin/gaiad" ports_start_at=27000 home_dir="$HOME/.gm" auto_maintain_config=true +comment="Please add comments to all node definitions. (Mandatory.)" [validator1] -#auto_maintain_config=false -fake_entry="" +comment="This is a validator with global defaults." [node1] +comment="This is a full node that attaches to validator1." network="validator1" From 6245e04d9f6f2ab37e8d3d1adc6c9aa7bfda66ea Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Thu, 6 May 2021 23:27:35 -0400 Subject: [PATCH 03/29] new features implemented --- scripts/gm/bin/gm | 51 +++++++++++++++++---- scripts/gm/bin/lib-gm | 89 +++++++++++++++++++++++++++++++++--- scripts/gm/bin/shell-support | 2 + 3 files changed, 126 insertions(+), 16 deletions(-) diff --git a/scripts/gm/bin/gm b/scripts/gm/bin/gm index c4b1550012..843fb639f3 100755 --- a/scripts/gm/bin/gm +++ b/scripts/gm/bin/gm @@ -6,9 +6,9 @@ set -eu export SCRIPT_DIR="${0%%gm}" export LOCAL_LIB_GM="${SCRIPT_DIR}lib-gm" if [ -f "$LOCAL_LIB_GM" ]; then - source "$LOCAL_LIB_GM" + . "$LOCAL_LIB_GM" elif [ -f "$HOME/.gm/bin/lib-gm" ]; then - source $HOME/.gm/bin/lib-gm + . "$HOME/.gm/bin/lib-gm" else echo "ERROR: could not find lib-gm, exiting..." exit 1 @@ -39,23 +39,55 @@ usage() { case "${1:-help}" in help) usage - exit 0 ;; install) install ;; keys) load_config - echo "Todo: not implemented" + if [ $# -eq 1 ]; then + NODES="$ALL_NODES" + else + shift + NODES="$*" + fi + for i in $NODES + do + warn_unknown_node "$i" || continue + list_keys "$i" + done ;; log) load_config shift - echo "Todo: not implemented" + F_FLAG="" + while [ "$#" -gt 0 ]; + do + case "$1" in + "-f") + F_FLAG="-f" ;; + *) + NODE="$1" + esac + shift + done + HOME_DIR="$(get_home_dir "$NODE")" + echo "tail $F_FLAG \"${HOME_DIR}/log\"" + tail "$F_FLAG" "${HOME_DIR}/log" ;; ports) load_config - echo "Todo: not implemented" + if [ $# -eq 1 ]; then + NODES="$ALL_NODES" + else + shift + NODES="$*" + fi + for i in $NODES + do + warn_unknown_node "$i" || continue + ports "$i" + done ;; start) load_config @@ -63,10 +95,11 @@ case "${1:-help}" in NODES_TO_START="$ALL_NODES" else shift - NODES_TO_START="$@" + NODES_TO_START="$*" fi for i in $NODES_TO_START do + warn_unknown_node "$i" || continue if [ ! -d "$(get_home_dir "$i")" ]; then create "$i" || continue fi @@ -86,10 +119,11 @@ case "${1:-help}" in NODES_TO_STOP="$ALL_NODES" else shift - NODES_TO_STOP="$@" + NODES_TO_STOP="$*" fi for i in $NODES_TO_STOP do + warn_unknown_node "$i" || continue stop "$i" done ;; @@ -102,4 +136,3 @@ case "${1:-help}" in exit_with_error "could not understand the command \"$1\"" ;; esac - diff --git a/scripts/gm/bin/lib-gm b/scripts/gm/bin/lib-gm index 1345046b99..e09414a420 100644 --- a/scripts/gm/bin/lib-gm +++ b/scripts/gm/bin/lib-gm @@ -12,8 +12,12 @@ install() { chmod 755 "${HOME}/.gm/bin/gm" cp "${0%%gm}lib-gm" "${HOME}/.gm/bin/lib-gm" chmod 644 "${HOME}/.gm/bin/lib-gm" + cp "${0%%gm}shell-support" "${HOME}/.gm/bin/shell-support" + chmod 644 "${HOME}/.gm/bin/shell-support" CONFIG_FILE="${HOME}/.gm/gm.toml" - write_default_config + if [ ! -f "$CONFIG_FILE" ]; then + write_default_config + fi if [ -z "$(which stoml)" ]; then warn "missing mandatory stoml, install it from https://github.com/freshautomations/stoml/releases" fi @@ -23,7 +27,7 @@ install() { if [ ! -d /usr/local/etc/bash_completion.d ]; then warn "run \"brew install bash-completion\" to install optional bash completion" fi - echo "Please add \"source $HOME/.gm/bin/shell-support\" to your .bash_profile to access the tool and restart your shell." + echo "Please add \"source $HOME/.gm/bin/shell-support\" to your .profile, .bash_profile or other startup script and restart your shell." } enforce_requirements() { @@ -45,6 +49,13 @@ warn() { echo "WARNING: $*" } +warn_unknown_node() { + if ! a_in_b "$1" "$ALL_NODES"; then + warn "unknown node $1, skipping..." + return 1 + fi +} + exit_with_error() { echo "ERROR: $*, exiting..." return 1 @@ -324,22 +335,30 @@ stop() { } print_header_line() { - echo "NODE PID RPC APP GRPC HOME_DIR" + echo "NODE PID RPC APP GRPC HOME_DIR" } print_status_line() { - I="${2:-}" + NAME="${2:-}$1" + NAME_LENGTH="${#NAME}" + NAME_PAD="" + if [ "$NAME_LENGTH" -lt 15 ]; then + for p in $(seq "$NAME_LENGTH" 15); + do + NAME_PAD="$NAME_PAD " + done + fi HOME_DIR="$(get_home_dir "$1")" GAIAD_PID_FILE="${HOME_DIR}/pid" if [ -f "$GAIAD_PID_FILE" ]; then GAIAD_PID="$(cat "$GAIAD_PID_FILE")" if [ "$(ps -p "$GAIAD_PID" -o pid | wc -l)" -eq 2 ]; then - echo "$I$1 $GAIAD_PID $(get_rpc_port "$1") $(get_app_port "$1") $(get_grpc_port "$1") $HOME_DIR" + echo "${NAME}${NAME_PAD} $GAIAD_PID $(get_rpc_port "$1") $(get_app_port "$1") $(get_grpc_port "$1") $HOME_DIR" else - echo "$I$1 ($GAIAD_PID) - - - $HOME_DIR" + echo "${NAME}${NAME_PAD}($GAIAD_PID) - - - $HOME_DIR" fi else - echo "$I$1 - - - - $HOME_DIR" + echo "${NAME}${NAME_PAD} - - - - $HOME_DIR" fi } @@ -360,3 +379,59 @@ status() { done } +ports() { + P2P="$(get_p2p_port "$1")" + RPC="$(get_rpc_port "$1")" + APP="$(get_app_port "$1")" + GRPC="$(get_grpc_port "$1")" + PPROF="$(get_pprof_port "$1")" + echo "${1} RPC : http://localhost:${RPC}" + echo "${1} APP : http://localhost:${APP}" + echo "${1} GRPC : http://localhost:${GRPC}" + echo "${1} P2P : http://localhost:${P2P}" + echo "${1} PPROF: http://localhost:${PPROF}" +} + +list_keys() { + HOME_DIR="$(get_home_dir "$1")" + GAIAD_BINARY="$(get_gaiad_binary "$1")" + echo "\"$GAIAD_BINARY\" keys list --keyring-backend test --keyring-dir \"$HOME_DIR\"" + KEY_NAME="" + "$GAIAD_BINARY" keys list --keyring-backend test --keyring-dir "$HOME_DIR" | while read -r line + do + NAME="${line##'- name: '}" + TYPE="${line##'type: '}" + MNEMONIC="${line##'mnemonic:'}" + THRESHOLD="${line##'threshold: '}" + PUBKEYS="${line##'pubkeys: '}" + if [ "$NAME" != "$line" ]; then + KEY_NAME="$NAME" + echo + echo "$line" + elif [ "$TYPE" != "$line" ]; then + if [ "$line" != "type: local" ]; then + echo "$line" + fi + elif [ "$MNEMONIC" != "$line" ]; then + if a_in_b "${KEY_NAME%%v0}" "$VALIDATORS"; then + echo "mnemonic: \"$(stoml "${HOME_DIR}/key_seed.json" secret)\"" + elif a_in_b "${KEY_NAME%%n0}" "$FULL_NODES"; then + echo "mnemonic: \"$(stoml "${HOME_DIR}/key_seed.json" secret)\"" + elif [ "${KEY_NAME}" = "wallet" ]; then + echo "mnemonic: \"$(stoml "${HOME_DIR}/wallet_seed.json" mnemonic)\"" + else + echo "mnemonic: \"\"" + fi + elif [ "$THRESHOLD" != "$line" ]; then + if [ "$line" != "threshold: 0" ]; then + echo "$line" + fi + elif [ "$PUBKEYS" != "$line" ]; then + if [ "$line" != "pubkeys: []" ]; then + echo "$line" + fi + else + echo "$line" + fi + done +} \ No newline at end of file diff --git a/scripts/gm/bin/shell-support b/scripts/gm/bin/shell-support index f7498315e9..b977767955 100644 --- a/scripts/gm/bin/shell-support +++ b/scripts/gm/bin/shell-support @@ -60,7 +60,9 @@ _gm() { # subcommands have their own completion functions case "$cmd" in + keys) _gm_nodes ;; log) _gm_nodes ;; + ports) _gm_nodes ;; start) _gm_nodes ;; stop) _gm_nodes ;; *) ;; From ebf1e1ff5e853ce99de2ed8897ad3f733c59b8c3 Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Sat, 8 May 2021 10:59:41 -0400 Subject: [PATCH 04/29] Compatibility with the full-mesh tool's different naming convention --- scripts/gm/bin/lib-gm | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/gm/bin/lib-gm b/scripts/gm/bin/lib-gm index e09414a420..2e30c65923 100644 --- a/scripts/gm/bin/lib-gm +++ b/scripts/gm/bin/lib-gm @@ -417,8 +417,11 @@ list_keys() { echo "mnemonic: \"$(stoml "${HOME_DIR}/key_seed.json" secret)\"" elif a_in_b "${KEY_NAME%%n0}" "$FULL_NODES"; then echo "mnemonic: \"$(stoml "${HOME_DIR}/key_seed.json" secret)\"" - elif [ "${KEY_NAME}" = "wallet" ]; then - echo "mnemonic: \"$(stoml "${HOME_DIR}/wallet_seed.json" mnemonic)\"" + elif [ -f "${HOME_DIR}/${KEY_NAME}_seed.json" ]; then + echo "mnemonic: \"$(stoml "${HOME_DIR}/${KEY_NAME}_seed.json" mnemonic)\"" + # Compatibility with the full-mesh tool + elif a_in_b "${KEY_NAME%%c0}" "$VALIDATORS" && [ -f "${HOME_DIR}/client_wallet_seed.json" ]; then + echo "mnemonic: \"$(stoml "${HOME_DIR}/client_wallet_seed.json" mnemonic)\"" else echo "mnemonic: \"\"" fi @@ -434,4 +437,4 @@ list_keys() { echo "$line" fi done -} \ No newline at end of file +} From 18743dd5705247c917003c6a454f8ad306dec47d Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Tue, 11 May 2021 18:15:11 -0400 Subject: [PATCH 05/29] Final feature set, bugfixes --- scripts/gm/bin/gm | 26 ++++-- scripts/gm/bin/lib-gm | 157 +++++++++++++++++++++++------------ scripts/gm/bin/shell-support | 8 +- 3 files changed, 132 insertions(+), 59 deletions(-) diff --git a/scripts/gm/bin/gm b/scripts/gm/bin/gm index 843fb639f3..9f0eef2a58 100755 --- a/scripts/gm/bin/gm +++ b/scripts/gm/bin/gm @@ -3,7 +3,8 @@ set -eu # Load lib-gm either from the local folder or from the global $HOME/.gm/bin/lib-gm -export SCRIPT_DIR="${0%%gm}" +SCRIPT_0="${0:-$HOME/.gm/bin/gm}" +export SCRIPT_DIR="${SCRIPT_0%%gm}" export LOCAL_LIB_GM="${SCRIPT_DIR}lib-gm" if [ -f "$LOCAL_LIB_GM" ]; then . "$LOCAL_LIB_GM" @@ -61,19 +62,29 @@ case "${1:-help}" in load_config shift F_FLAG="" + LIST="" while [ "$#" -gt 0 ]; do case "$1" in "-f") F_FLAG="-f" ;; *) - NODE="$1" + LIST="$LIST $(get_home_dir "$1")/log" esac shift done - HOME_DIR="$(get_home_dir "$NODE")" - echo "tail $F_FLAG \"${HOME_DIR}/log\"" - tail "$F_FLAG" "${HOME_DIR}/log" + if [ -z "$F_FLAG" ]; then + echo "less -R $LIST" + # Todo: spaces in folder names and file names are not supported yet. + # shellcheck disable=SC2086 + less -R $LIST + else + echo "tail -f $LIST" + # Todo: spaces in folder names and file names are not supported yet. + # shellcheck disable=SC2086 + tail -f $LIST + fi + ;; ports) load_config @@ -89,6 +100,11 @@ case "${1:-help}" in ports "$i" done ;; + ss) + # For debug purposes + # Todo: either make this a supported command or remove it + lsof -i4:27000-28000 -P + ;; start) load_config if [ $# -eq 1 ]; then diff --git a/scripts/gm/bin/lib-gm b/scripts/gm/bin/lib-gm index 2e30c65923..d8df17079e 100644 --- a/scripts/gm/bin/lib-gm +++ b/scripts/gm/bin/lib-gm @@ -37,6 +37,9 @@ enforce_requirements() { if [ -z "$(which sconfig)" ]; then exit_with_error "missing sconfig, install it from https://github.com/freshautomations/sconfig/releases" fi + if [ -z "$(which sed)" ]; then + exit_with_error "missing sed, please install it (this requirement will be removed in the future)" + fi } debug() { @@ -85,7 +88,15 @@ a_in_b() { test -n "$(echo "$2" | grep '\(^\| \+\)'"${1}"'\( \+\|$\)')" } -# Todo: fix "stoml" so it reads empty sections too - currently we work around this with the mandatory comment. +# Is string A in comma-separated list B? +a_in_b_comma_separated() { + test -n "$(echo "$2" | grep '\(^\|,\) *'"${1}"' *\(,\|$\)')" +} + +load_all_sections() { + grep '^ *\[.\+\] *$' "$CONFIG_FILE" | sed 's/^ *\[\([^]]*\)\] *$/\1/' +} + load_config() { FRESH_CONFIG=no if [ -f "${SCRIPT_DIR}gm.toml" ]; then @@ -101,11 +112,10 @@ load_config() { fi fi GLOBAL_GAIAD_BINARY="$(eval echo "$(stoml -q "$CONFIG_FILE" global.gaiad_binary || which gaiad)")" - GLOBAL_PORTS_START_AT="$(stoml -q "$CONFIG_FILE" global.ports_start_at || echo 27000)" + #GLOBAL_PORTS_START_AT is deliberately not exported because it is updated sometimes during execution GLOBAL_HOME_DIR="$(eval echo "$(stoml -q "$CONFIG_FILE" global.home_dir || echo "${HOME}/.gm")")" GLOBAL_AUTO_MAINTAIN_CONFIG="$(stoml -q "$CONFIG_FILE" global.auto_maintain_config || echo "true")" export GLOBAL_GAIAD_BINARY - export GLOBAL_PORTS_START_AT export GLOBAL_HOME_DIR export GLOBAL_AUTO_MAINTAIN_CONFIG if [ "$FRESH_CONFIG" = "yes" ]; then @@ -115,7 +125,11 @@ load_config() { else stoml "$CONFIG_FILE" global 1> /dev/null || exit_with_error invalid config file. Make sure all strings are quoted test -x "$GLOBAL_GAIAD_BINARY" || exit_with_error "gaiad binary cannot be executed: $GLOBAL_GAIAD_BINARY" - RAW_SECTIONS="$(stoml -q "$CONFIG_FILE" . || echo "")" + # Workaround for issue: https://github.com/spf13/viper/issues/1131 + # Fix "stoml" so it reads empty sections too. + #RAW_SECTIONS="$(stoml -q "$CONFIG_FILE" . || echo "")" + RAW_SECTIONS="$(load_all_sections || echo "")" + # End of workaround VALIDATORS="" RAW_NODES="" for i in $RAW_SECTIONS @@ -150,7 +164,7 @@ write_default_config() { cat < "$CONFIG_FILE" [global] gaiad_binary="$GLOBAL_GAIAD_BINARY" -ports_start_at=$GLOBAL_PORTS_START_AT +ports_start_at=27000 home_dir="$GLOBAL_HOME_DIR" auto_maintain_config=$GLOBAL_AUTO_MAINTAIN_CONFIG comment="Please add comments to all node definitions. (Mandatory.)" @@ -170,15 +184,10 @@ get_gaiad_binary() { get_ports_start_at() { RESULT="$(stoml -q "$CONFIG_FILE" "${1}.ports_start_at")" if [ -z "$RESULT" ]; then - i=0 - for j in $ALL_NODES - do - if [ "$j" = "$1" ]; then - break - fi - i=$((i+1)) - done - echo "$((GLOBAL_PORTS_START_AT+10*i))" + THIS_PORTS_START_AT="$(stoml -q "$CONFIG_FILE" global.ports_start_at || echo 27000)" + sconfig "$CONFIG_FILE" -t int "global.ports_start_at=$((THIS_PORTS_START_AT+10))" 1> /dev/null + sconfig "$CONFIG_FILE" -t int "${1}.ports_start_at=$THIS_PORTS_START_AT" 1> /dev/null + echo "$THIS_PORTS_START_AT" else echo "$RESULT" fi @@ -266,36 +275,34 @@ create() { fi mv "${HOME_ROOT}/${i}n0" "${HOME_DIR}" rm -rf "${HOME_ROOT}/gentxs" - # Todo: Maybe replace the below with calling the configure function. - NETWORK_HOME_DIR="$(get_home_dir "$NETWORK")" - cp "$NETWORK_HOME_DIR/config/genesis.json" "$HOME_DIR/config/genesis.json" - NETWORK_NODE="$(get_node_id "$NETWORK")@localhost:$(get_p2p_port "$NETWORK")" - sconfig "$HOME_DIR/config/config.toml" "p2p.persistent_peers=$NETWORK_NODE" 1> /dev/null - if get_auto_maintain_config "$NETWORK"; then - NODE_ID="$(get_node_id "$i")" - sconfig "$NETWORK_HOME_DIR/config/config.toml" "p2p.unconditional_peer_ids=$NODE_ID" 1> /dev/null - # Todo: Make this more robust: if two full nodes connect to the same validator, this override each other - fi fi + configure "$i" sconfig "$HOME_DIR/config/config.toml" p2p.addr_book_strict=false 1> /dev/null sconfig "$HOME_DIR/config/config.toml" p2p.allow_duplicate_ip=true 1> /dev/null } configure() { HOME_DIR="$(get_home_dir "$1")" - # Todo: rpc.pprof_laddr=6060, maybe needs moving? - # Todo: rpc.grpc also? P2P="$(get_p2p_port "$1")" RPC="$(get_rpc_port "$1")" - #APP="$(get_app_port "$1")" - #GRPC="$(get_grpc_port "$1")" + APP="$(get_app_port "$1")" + GRPC="$(get_grpc_port "$1")" PPROF="$(get_pprof_port "$1")" sconfig "$HOME_DIR/config/config.toml" "p2p.laddr=tcp://0.0.0.0:${P2P}" 1> /dev/null sconfig "$HOME_DIR/config/config.toml" "rpc.laddr=tcp://0.0.0.0:${RPC}" 1> /dev/null - sconfig "$HOME_DIR/config/config.toml" "rpc.pprof_laddr=localhost:${PPROF}" 1> /dev/null - # Todo: sconfig doesn't support it yet - #sconfig "$HOME_DIR/config/app.toml" "api.address=tcp://0.0.0.0:${APP}" 1> /dev/null - #sconfig "$HOME_DIR/config/app.toml" "grpc.address=0.0.0.0:${GRPC}" 1> /dev/null + sconfig "$HOME_DIR/config/config.toml" "rpc.pprof_laddr=0.0.0.0:${PPROF}" 1> /dev/null + # Workaround for https://github.com/spf13/viper/issues/1132 + # Viper does not support writing array of arrays so we clean out the variable + sconfig "$HOME_DIR/config/app.toml" -t stringSlice "telemetry.global-labels=panic gaiad" 1> /dev/null + OS="$(uname -s)" + if [ "$OS" = "Darwin" ]; then + sed -i '' 's/ global-labels = \["panic","gaiad"\]/ global-labels = []/' "$HOME_DIR/config/app.toml" + else + sed -i'' 's/ global-labels = \["panic","gaiad"\]/ global-labels = []/' "$HOME_DIR/config/app.toml" + fi + # End of workaround + sconfig "$HOME_DIR/config/app.toml" "api.address=tcp://0.0.0.0:${APP}" 1> /dev/null + sconfig "$HOME_DIR/config/app.toml" "grpc.address=0.0.0.0:${GRPC}" 1> /dev/null if ! a_in_b "$i" "$VALIDATORS"; then NETWORK="$(get_network "$1")" NETWORK_HOME_DIR="$(get_home_dir "$NETWORK")" @@ -303,35 +310,76 @@ configure() { NETWORK_NODE="$(get_node_id "$NETWORK")@localhost:$(get_p2p_port "$NETWORK")" sconfig "$HOME_DIR/config/config.toml" "p2p.persistent_peers=$NETWORK_NODE" 1> /dev/null if get_auto_maintain_config "$NETWORK"; then + EXISTING_PEERS="$(stoml -q "$NETWORK_HOME_DIR/config/config.toml" "p2p.unconditional_peer_ids")" NODE_ID="$(get_node_id "$1")" - sconfig "$NETWORK_HOME_DIR/config/config.toml" "p2p.unconditional_peer_ids=$NODE_ID" 1> /dev/null - # Todo: Make this more robust: if two full nodes connect to the same validator, don't override each other + if ! a_in_b_comma_separated "$NODE_ID" "$EXISTING_PEERS"; then + if [ -z "$EXISTING_PEERS" ]; then + EXISTING_PEERS="$NODE_ID" + else + EXISTING_PEERS="$EXISTING_PEERS,$NODE_ID" + fi + sconfig "$NETWORK_HOME_DIR/config/config.toml" "p2p.unconditional_peer_ids=$EXISTING_PEERS" 1> /dev/null + fi fi fi } -# Todo: (high-priority) fix sconfig to be able to map in app.toml -start() { - GAIAD="$(get_gaiad_binary "$1")" +is_running() { HOME_DIR="$(get_home_dir "$1")" - GAIAD_LOG="${HOME_DIR}/log" - nohup "$GAIAD" start --x-crisis-skip-assert-invariants --home "$HOME_DIR" > "$GAIAD_LOG" 2>&1 & - GAIAD_PID=$! - echo "$GAIAD_PID" > "$HOME_DIR/pid" - echo "$i started, PID: $GAIAD_PID, LOG: $GAIAD_LOG" + GAIAD_PID_FILE="${HOME_DIR}/pid" + if [ -f "$GAIAD_PID_FILE" ]; then + GAIAD_PID="$(cat "$GAIAD_PID_FILE")" + if [ -n "$GAIAD_PID" ]; then + test "$(ps -p "$GAIAD_PID" -o pid | wc -l)" -eq 2 + else + return 1 + fi + else + return 1 + fi +} + +start() { + if is_running "$1"; then + warn "$1 is already running, skipping" + else + GAIAD="$(get_gaiad_binary "$1")" + HOME_DIR="$(get_home_dir "$1")" + GAIAD_LOG="${HOME_DIR}/log" + nohup "$GAIAD" start --x-crisis-skip-assert-invariants --home "$HOME_DIR" > "$GAIAD_LOG" 2>&1 & + GAIAD_PID=$! + echo "$GAIAD_PID" > "$HOME_DIR/pid" + echo "$i started, PID: $GAIAD_PID, LOG: $GAIAD_LOG" + fi } stop() { - HOME_DIR="$(get_home_dir "$1")" - #GAIAD_BINARY="$(get_gaiad_binary "$1")" - test -f "${HOME_DIR}/pid" - GAIAD_PID="$(cat "${HOME_DIR}/pid")" - #COMM="$(ps -p "$GAIAD_PID" -o comm | tail -1)" - echo "Stopping $1 with PID $GAIAD_PID..." - kill -TERM "$GAIAD_PID" 2> /dev/null || echo " No process with PID $GAIAD_PID. Cleaning up run config." - sleep 2 - # Todo: make this more robust by checking if the process was killed properly - rm -f "$HOME_DIR/pid" + if is_running "$1"; then + HOME_DIR="$(get_home_dir "$1")" + GAIAD_PID="$(cat "${HOME_DIR}/pid")" + echo "Stopping $1 with PID $GAIAD_PID..." + kill -TERM "$GAIAD_PID" 2> /dev/null && rm -f "$HOME_DIR/pid" + RESULT="$?" + if [ "$RESULT" != "0" ]; then + warn "SIGTERM failed on PID ${GAIAD_PID}. Trying KILL..." + kill -KILL "$GAIAD_PID" && rm -f "$HOME_DIR/pid" + RESULT="$?" + if [ "$RESULT" != "0" ]; then + warn "SIGKILL failed on PID ${GAIAD_PID}. Giving up..." + fi + fi + else + HOME_DIR="$(get_home_dir "$1")" + if [ -f "${HOME_DIR}/pid" ]; then + GAIAD_PID="$(cat "${HOME_DIR}/pid")" + if [ -n "$GAIAD_PID" ]; then + warn "no process with PID $GAIAD_PID. Removing stale data." + else + warn "no process ID found. Removing stale data." + fi + rm -f "$HOME_DIR/pid" + fi + fi } print_header_line() { @@ -352,6 +400,9 @@ print_status_line() { GAIAD_PID_FILE="${HOME_DIR}/pid" if [ -f "$GAIAD_PID_FILE" ]; then GAIAD_PID="$(cat "$GAIAD_PID_FILE")" + if [ -z "$GAIAD_PID" ]; then + GAIAD_PID=" N/A " + fi if [ "$(ps -p "$GAIAD_PID" -o pid | wc -l)" -eq 2 ]; then echo "${NAME}${NAME_PAD} $GAIAD_PID $(get_rpc_port "$1") $(get_app_port "$1") $(get_grpc_port "$1") $HOME_DIR" else @@ -371,7 +422,7 @@ status() { do NETWORK="$(get_network "$j")" if [ "$i" = "$NETWORK" ]; then - print_status_line "$j" "-" + print_status_line "$j" " " else continue fi diff --git a/scripts/gm/bin/shell-support b/scripts/gm/bin/shell-support index b977767955..292ae58b89 100644 --- a/scripts/gm/bin/shell-support +++ b/scripts/gm/bin/shell-support @@ -24,6 +24,12 @@ _gm_nodes() { while read -r line; do COMPREPLY+=("$line"); done < <(compgen -W "$cmds" -- "$cur") } +_gm_nodes_f() { + local cur="${COMP_WORDS[COMP_CWORD]}" + local cmds="$(__gm_get_nodes) -f" + while read -r line; do COMPREPLY+=("$line"); done < <(compgen -W "$cmds" -- "$cur") +} + _gm_complete_commands() { local cur="${COMP_WORDS[COMP_CWORD]}" local cmds="help install keys log ports start status stop version" @@ -61,7 +67,7 @@ _gm() { # subcommands have their own completion functions case "$cmd" in keys) _gm_nodes ;; - log) _gm_nodes ;; + log) _gm_nodes_f ;; ports) _gm_nodes ;; start) _gm_nodes ;; stop) _gm_nodes ;; From 7f7a7308308525c8cdc0e4ab210ccf1410ff96b5 Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Tue, 11 May 2021 18:17:04 -0400 Subject: [PATCH 06/29] config fix --- scripts/gm/gm.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/gm/gm.toml b/scripts/gm/gm.toml index f614ecead0..ef81f6bed5 100644 --- a/scripts/gm/gm.toml +++ b/scripts/gm/gm.toml @@ -5,11 +5,8 @@ gaiad_binary="$GOPATH/bin/gaiad" ports_start_at=27000 home_dir="$HOME/.gm" auto_maintain_config=true -comment="Please add comments to all node definitions. (Mandatory.)" [validator1] -comment="This is a validator with global defaults." [node1] -comment="This is a full node that attaches to validator1." network="validator1" From 60e0c37dcb1f7ab8f96d9351c824267250bc697e Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Wed, 12 May 2021 01:06:38 -0400 Subject: [PATCH 07/29] [global] strategy = 'naive' log_level = 'info' [[chains]] id='node1' rpc_addr='http://localhost:27010' grpc_addr='https://localhost:27012' websocket_addr='ws://localhost:27010/websocket' rpc_timeout='1s' account_prefix='cosmos' key_name='wallet' store_prefix='ibc' fee_denom='stake' [chains.trust_threshold] numerator = '1' denominator = '3' [[chains]] id='node3' rpc_addr='http://localhost:27040' grpc_addr='https://localhost:27042' websocket_addr='ws://localhost:27040/websocket' rpc_timeout='1s' account_prefix='cosmos' key_name='wallet' store_prefix='ibc' fee_denom='stake' [chains.trust_threshold] numerator = '1' denominator = '3' command added --- scripts/gm/bin/gm | 5 ++++ scripts/gm/bin/lib-gm | 53 ++++++++++++++++++++++++++++++++++-- scripts/gm/bin/shell-support | 2 +- scripts/gm/gm.toml | 20 +++++++++++++- 4 files changed, 76 insertions(+), 4 deletions(-) diff --git a/scripts/gm/bin/gm b/scripts/gm/bin/gm index 9f0eef2a58..91ff2daaec 100755 --- a/scripts/gm/bin/gm +++ b/scripts/gm/bin/gm @@ -26,6 +26,7 @@ usage() { echo "COMMANDS DESCRIPTION" echo echo "help print this help and exit" + echo "hermes print the hermes config.toml" echo "install install the script for the local user" echo "keys print the keys of validator nodes" echo "log print the log of a node" @@ -41,6 +42,10 @@ case "${1:-help}" in help) usage ;; + hermes) + load_config + hermes + ;; install) install ;; diff --git a/scripts/gm/bin/lib-gm b/scripts/gm/bin/lib-gm index d8df17079e..466437e38e 100644 --- a/scripts/gm/bin/lib-gm +++ b/scripts/gm/bin/lib-gm @@ -115,9 +115,11 @@ load_config() { #GLOBAL_PORTS_START_AT is deliberately not exported because it is updated sometimes during execution GLOBAL_HOME_DIR="$(eval echo "$(stoml -q "$CONFIG_FILE" global.home_dir || echo "${HOME}/.gm")")" GLOBAL_AUTO_MAINTAIN_CONFIG="$(stoml -q "$CONFIG_FILE" global.auto_maintain_config || echo "true")" + GLOBAL_ADD_TO_HERMES="$(stoml -q "$CONFIG_FILE" global.add_to_hermes || echo "true")" export GLOBAL_GAIAD_BINARY export GLOBAL_HOME_DIR export GLOBAL_AUTO_MAINTAIN_CONFIG + export GLOBAL_ADD_TO_HERMES if [ "$FRESH_CONFIG" = "yes" ]; then write_default_config VALIDATORS="" @@ -167,7 +169,7 @@ gaiad_binary="$GLOBAL_GAIAD_BINARY" ports_start_at=27000 home_dir="$GLOBAL_HOME_DIR" auto_maintain_config=$GLOBAL_AUTO_MAINTAIN_CONFIG -comment="Please add comments to all node definitions. (Mandatory.)" +add_to_hermes=$GLOBAL_ADD_TO_HERMES EOF set -e } @@ -243,6 +245,15 @@ get_network() { echo "$RESULT" } +get_add_to_hermes() { + RESULT="$(stoml -q "$CONFIG_FILE" "${1}.add_to_hermes")" + if [ -z "$RESULT" ]; then + test "$GLOBAL_ADD_TO_HERMES" = "true" + else + test "$RESULT" = "true" + fi +} + get_node_id() { GAIAD="$(get_gaiad_binary "$1")" HOME_DIR="$(get_home_dir "$1")" @@ -391,7 +402,7 @@ print_status_line() { NAME_LENGTH="${#NAME}" NAME_PAD="" if [ "$NAME_LENGTH" -lt 15 ]; then - for p in $(seq "$NAME_LENGTH" 15); + for _ in $(seq "$NAME_LENGTH" 15); do NAME_PAD="$NAME_PAD " done @@ -489,3 +500,41 @@ list_keys() { fi done } + +hermes() { + cat < Date: Wed, 12 May 2021 12:29:53 -0400 Subject: [PATCH 08/29] Config update, README added --- scripts/gm/README.md | 245 ++++++++++++++++++++++++++++++++++++++++++ scripts/gm/bin/lib-gm | 8 ++ scripts/gm/gm.toml | 22 ++-- 3 files changed, 266 insertions(+), 9 deletions(-) create mode 100644 scripts/gm/README.md diff --git a/scripts/gm/README.md b/scripts/gm/README.md new file mode 100644 index 0000000000..ffb05fd718 --- /dev/null +++ b/scripts/gm/README.md @@ -0,0 +1,245 @@ +# Gaiad Manager `gm` + +## TL;DR +* Tool to manage local gaiad instances - no Docker needed. +* `scripts/gm/bin/gm install` to install it. Follow the instructions there for dependencies. +* `gm start` to start the example configuration of one validator and one full node on your local machine. +* Config file is in `$HOME/.gm/gm.toml` play around and add more nodes. +* Tab completion is pretty good, use it! Or run `gm` by itself for help. +* Pre-1.0 warning: Got a shell error? [Raise an issue!](https://github.com/informalsystems/ibc-rs/issues/) + +## Overview +Gaiad Manager (`gm` from now on) is an easily configurable command-line tool (CLI) that helps manage local `gaiad` +networks. + +Typical problems running multiple `gaiad` instances involve: +* Identifying binaries and configurations for startup and nodes on the system for shutdown. +* Managing port allocations on the local machine. +* Copying and setting up configurations among nodes on the same network. +* Managing `hermes` configuration for IBC. + +`gm` solves this by using a unified configuration file that describes the nodes and their relationship and automating +configuration updates. + +## Requirements +* Bourne shell (`sh`) +* [`sconfig`](https://github.com/freshautomations/sconfig/releases) and + [`stoml`](https://github.com/freshautomations/stoml/releases) installed in your PATH (put them in `/usr/local/bin`) +* `sed` (trying to remove this dependency in the future) +* For shell-completion Bourne Again Shell (`bash`) for the local user (`zsh` shell-completion is coming) + +## How to run +1. Install the dependencies. + +On MacOS: +```bash +curl -o /usr/local/bin/sconfig https://github.com/freshautomations/sconfig/releases/download/v0.1.0/sconfig_darwin_amd64 +curl -o /usr/local/bin/stoml https://github.com/freshautomations/stoml/releases/download/v0.6.0/stoml_darwin_amd64 +``` +On Linux: +```bash +curl -o /usr/local/bin/sconfig https://github.com/freshautomations/sconfig/releases/download/v0.1.0/sconfig_linux_amd64 +curl -o /usr/local/bin/stoml https://github.com/freshautomations/stoml/releases/download/v0.6.0/stoml_linux_amd64 +``` +2. Install `gm` +```bash +git clone https://github.com/informal/ibc-rs +ibc-rs/scripts/gm/bin/gm install +``` +3. Activate `gm` +* Add `source $HOME/.gm/bin/shell-support` to a file that executes when a new terminal window comes up + (`$HOME/.bash_profile` or `$HOME/.bashrc`) +* (Optional) Enable auto-completion +On MacOS: +```bash +brew install bash-completion +``` +On Linux: +``` +apt install bash-completion || yum install bash-completion +``` +* Restart your terminal + +Note: The `shell-support` script allows bash-completion as well as creating a `gm` alias, so you don't need to add more +entries to your PATH environment variable. If you don't want to use this, you can always just add `$HOME/.gm/bin` to +your path. +## Folders and files +### The HOME folder +**Where**: `$HOME/.gm` + +**Description**: The hard-wired home folder for `gm` is `$HOME/.gm`. It contains the binaries required to run `gm` and +the `gm.toml` file for node configuration. By default, newly created node configuration also resides here under the +`$HOME/.gm/` folder but that can be configured in `gm.toml`. + +### The configuration: `gm.toml` +**Where**: `$HOME/.gm/gm.toml`. + +**Description**: This file contains all the high-level node configuration that `gm` is aware of. + +**Entries**: +* `[global]` - the global section defines application-wide defaults. All the entries here can be overridden on a + per-node basis. +* `gaiad_binary` - Path to the `gaiad` binary to use. +* `ports_start_at` + * In the global section this defines the first free port to use for newly created nodes. + The value will be incremented (by 10) when a new node requires ports. + * In the per-node configuration this is the first port to allocate for the node. + A total of 10 ports will be allocated per node. +* `home_dir` + * In the global section this defines the default folder where all node folders will be created. + * In the per-node configuration this is the folder for the node configuration. +* `auto_maintain_config` - automatically update the `persistent_peers` and `unconditional_peer_ids` sections of the node + configuration. +* `add_to_hermes` - this node should be part of the hermes config. +* `network` + * Only in subsections: Define the network (the validator connection) for the full node. + * **Mandatory for full nodes**, does not exist for validator nodes. + +Note: the user is welcome to create additional nodes outside the scope of `gm` on the local machine but `gm` will only +manage nodes that are added to the configuration file. + +### The network configuration +**Where**: Default is the folder `$HOME/.gm/`, but it can be configured in `gm.toml` using the `home_dir` +entry. + +**Description**: The configuration and data folder for a node. Partially resembles a gaiad home folder (`.gaia`) but +it has additional files to store the wallet mnemonics. + +**Entries**: +* `config` - The node configuration folder. If the node is a full-node, the genesis file was copied from a validator +config. The persistent_peers section is automatically managed if the node has the `auto_maintain_config` parameter + enabled in `gm.toml`. +* `data` - The data folder. +* `keyring-test` - the keyring folder as defined by `gaiad testnet` with the "test" keyring-backend. +* `key_seed.json` - the node's signing and wallet key as defined using the `gaiad testnet` command. +* `wallet_seed.json` - an extra wallet mnemonic defined on validator nodes with some tokens for developer use. +* `pid` - the file that contains the process ID of the running node. (a la `/var/run`) Use `gm status` to see. +* `log` - the log file that contains the output of the running node. Use `gm log ` to see. + +This setup allows developers to run a node outside of `gm` just by pointing the `gaiad --home-dir` to the folder. + +### Ports +Ports are defined by the `ports_start_at` parameter which will be the first port assigned. +Port assignment is as follows: +``` +| name | port redirection | +|==============|====================| +| RPC (26657) | ports_start_at + 0 | +| App (1317) | ports_start_at + 1 | +| GRPC (9090 | ports_start_at + 2 | +| P2P (26656) | ports_start_at + 3 | +| PPROF (6060) | ports_start_at + 4 | +``` + +Example output of `gm ports` command when `node4.ports_start_at=27050`: +``` +node4 RPC : http://localhost:27050 +node4 APP : http://localhost:27051 +node4 GRPC : http://localhost:27052 +node4 P2P : http://localhost:27053 +node4 PPROF: http://localhost:27054 +``` + +## Execution manual +### `gm help` +**Description**: shows the help screen + +### `gm hermes` +**Description**: generate the `hermes.toml` config and print it on the screen. + +Tip: `gm hermes > $HOME/.hermes/config.toml` if you know what you're doing. + +### `gm install` +**Description**: Install the `gm` command under `$HOME/.gm`, create a default configuration and warn about missing +dependencies. + +Tip: You can run this command multiple times to check if the dependencies are met. + +### `gm keys [ ...]` +**Description**: List the keys installed on a node. (It gets them from the `keyring-test` folder.) +If no node is specified then it lists all keys. + +Tip: it will show you the seed phrase if it can find it in the folder. + +### `gm log [ ...] [-f]` +**Description**: Show the log of the specified node(s). At least one node has to be specified. + +Tip: You can put `-f` anywhere after `log` to get `tail -f`-like functionality. + +### `gm ports [ ...]` +**Description**: List the ports assigned to a node. +If no node is specified then it lists all nodes' ports. + +Tip: When automation doesn't get you all the way, this helps in identifying your nodes on your local machine. + +### `gm start [ ...]` +**Description**: Start the node(s). This will use the defined `gaiad` binary and configuration. +If no node is specified then it will start all nodes. + +Tip: You can freely start nodes over-and-over. If they are proven running, the command will not do anything, if they +were killed for any reason, the `pid` configuration will be updated, and a fresh node started. + +### `gm status` +**Description**: List all nodes and their basic properties, such as: their PID if they are running, their home folder, +and the most common ports used. + +Tip: PIDs in brackets mean that the node is not running when `gm` started them. This could be because of a configuration +error or maybe the user simply killed the PID. No worries, `gm` will clean up when `start` or `stop` is invoked. + +### `gm stop [ ...]` +**Description**: Stop the node(s). This will use the defined `gaiad` binary and configuration. +If no node is specified then it will stop all nodes. + +Tip: If a node was killed, you can use `gm stop` to clean up the PID file. + +### `gm version` +**Description**: Display the version of `gm`. + +Tip: Congratulations in reaching the bottom of the command references. For your endurance, you are rewarded +with an unsupported yet useful command: `gm ss` will list the open ports between 27000-28000 (the default port set +used by `gm`) on your local machine. Use it when you get port-clashes because of rogue processes on your machine that +`gm` can't account for. + +## Example: 5 networks with 5 full nodes attached +This is an example that recreates the "full-mesh" tool's default network setup with the added twist that all networks +have a full node and the hermes config is going through the full nodes instead of the validator nodes. + +You can get the hermes configuration automatically. + +You need to manually run the `hermes create channel` commands to create the IBC paths as well as the relayer commands. + +(Note: maybe it's a good feature request to automate this in the next release.) + +You might need to replace the value of the `gaiad_binary` entry, if you don't set `$GOPATH` in your regular executions. +```toml +[global] +gaiad_binary="$GOPATH/bin/gaiad" +ports_start_at=27000 +home_dir="$HOME/.gm" +auto_maintain_config=true +add_to_hermes=false + +[network1] +[network2] +[network3] +[network4] +[network5] + +[node1] +network="network1" +add_to_hermes=true +[node2] +network="network2" +add_to_hermes=true +[node3] +network="network3" +add_to_hermes=true +[node4] +network="network4" +add_to_hermes=true +[node5] +network="network5" +add_to_hermes=true +``` + +Ports will be auto-assigned and written in the configuration file on the first start. diff --git a/scripts/gm/bin/lib-gm b/scripts/gm/bin/lib-gm index 466437e38e..533993a2cd 100644 --- a/scripts/gm/bin/lib-gm +++ b/scripts/gm/bin/lib-gm @@ -159,6 +159,14 @@ load_config() { export VALIDATORS export FULL_NODES export ALL_NODES="$VALIDATORS $FULL_NODES" + # Workaround for issue: https://github.com/spf13/viper/issues/1131 + # Fix "stoml" so it reads empty sections too. + for v in $VALIDATORS + do + # This will fill in the ports_start_at entry in all sections so empty sections have at least one entry. + get_ports_start_at "$v" 1> /dev/null + done + # End of workaround } write_default_config() { diff --git a/scripts/gm/gm.toml b/scripts/gm/gm.toml index b3848294ac..acb4cb9dde 100644 --- a/scripts/gm/gm.toml +++ b/scripts/gm/gm.toml @@ -1,30 +1,34 @@ # Example configuration with one validator and one full node -# The global section lists all possible parameters and their default value, when they are not explicitly set. +# the global section defines application-wide defaults. All the entries here can be overridden on a per-node basis. [global] -# The path to the gaiad binary path. +# Path to the `gaiad` binary to use. gaiad_binary="$GOPATH/bin/gaiad" -# The ports to start at for any previously undefined nodes. The value will be incremented (by 10) -# when a new node requires ports. +# The first free port to use for newly created nodes. +# The value will be incremented (by 10) when a new node requires ports. ports_start_at=27000 -# The folder that contains the chain configuration and data as well as any test keys. +# The default folder where all node folders will be created. home_dir="$HOME/.gm" -# Automatically update `persistent_peers` on a full node and `unconditional_peer_ids` on a validator. +# Automatically update the `persistent_peers` and `unconditional_peer_ids` sections of the node configuration. auto_maintain_config=true -# When generating the hermes config, the current section (except for [global]) will be copied as necessary. -add_to_hermes=true +# This node should be part of the hermes config. +add_to_hermes=false ## ## Sub-section (node) definitions and parameter redefinitons below. ## +# A validator node that uses the default configuration. [validator1] -add_to_hermes=false +# A full node with some configuration +# A full node is defined by adding the `network` property to the section. [node1] +# Define the network (the validator connection) for the full node. network="validator1" +add_to_hermes=true From e612afca9c0d3a2c1bbe6e492730893b37eee57a Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Wed, 12 May 2021 12:34:42 -0400 Subject: [PATCH 09/29] CHANGELOG update --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff07e6f9de..b853a0a41f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ Jongwhan Lee (@leejw51crypto) ([#878]). - [ibc-relayer-cli] - Added packet clearing and client refresh capabilities for the `start-multi` command ([#784], [#786]) +- [scripts] + - Created the Gaiad Manager `gm` CLI tool for managing gaiad instances on the local machine ([#902]) ### IMPROVEMENTS - [ibc] @@ -59,6 +61,7 @@ Jongwhan Lee (@leejw51crypto) ([#878]). [#869]: https://github.com/informalsystems/ibc-rs/issues/869 [#878]: https://github.com/informalsystems/ibc-rs/issues/878 [#873]: https://github.com/informalsystems/ibc-rs/issues/873 +[#902]: https://github.com/informalsystems/ibc-rs/issues/902 ## v0.2.0 *April 14th, 2021* From 9934f04c34bb1dad172ad3edaa0f777e2b282e21 Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Wed, 12 May 2021 12:57:58 -0400 Subject: [PATCH 10/29] fluff --- scripts/gm/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/gm/README.md b/scripts/gm/README.md index ffb05fd718..80e4a99e8f 100644 --- a/scripts/gm/README.md +++ b/scripts/gm/README.md @@ -46,6 +46,9 @@ curl -o /usr/local/bin/stoml https://github.com/freshautomations/stoml/releases/ git clone https://github.com/informal/ibc-rs ibc-rs/scripts/gm/bin/gm install ``` + +Alternatively, you can create the folder `$HOME/.gm/bin` and copy the files from `scripts/gm/bin` in there. +The rest is just fluff. 3. Activate `gm` * Add `source $HOME/.gm/bin/shell-support` to a file that executes when a new terminal window comes up (`$HOME/.bash_profile` or `$HOME/.bashrc`) From b24b449adb64a3f97d1fe26389af35b150c11441 Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Fri, 14 May 2021 08:22:26 -0400 Subject: [PATCH 11/29] colin's fixes --- scripts/gm/README.md | 12 ++++++++---- scripts/gm/bin/lib-gm | 19 ++++++++++++++----- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/scripts/gm/README.md b/scripts/gm/README.md index 80e4a99e8f..3d2f5f2483 100644 --- a/scripts/gm/README.md +++ b/scripts/gm/README.md @@ -33,13 +33,17 @@ configuration updates. On MacOS: ```bash -curl -o /usr/local/bin/sconfig https://github.com/freshautomations/sconfig/releases/download/v0.1.0/sconfig_darwin_amd64 -curl -o /usr/local/bin/stoml https://github.com/freshautomations/stoml/releases/download/v0.6.0/stoml_darwin_amd64 +curl -Lo /usr/local/bin/sconfig https://github.com/freshautomations/sconfig/releases/download/v0.1.0/sconfig_darwin_amd64 +curl -Lo /usr/local/bin/stoml https://github.com/freshautomations/stoml/releases/download/v0.6.1/stoml_darwin_amd64 +chmod 755 /usr/local/bin/sconfig +chmod 755 /usr/local/bin/stoml ``` On Linux: ```bash -curl -o /usr/local/bin/sconfig https://github.com/freshautomations/sconfig/releases/download/v0.1.0/sconfig_linux_amd64 -curl -o /usr/local/bin/stoml https://github.com/freshautomations/stoml/releases/download/v0.6.0/stoml_linux_amd64 +curl -Lo /usr/local/bin/sconfig https://github.com/freshautomations/sconfig/releases/download/v0.1.0/sconfig_linux_amd64 +curl -Lo /usr/local/bin/stoml https://github.com/freshautomations/stoml/releases/download/v0.6.1/stoml_linux_amd64 +chmod 755 /usr/local/bin/sconfig +chmod 755 /usr/local/bin/stoml ``` 2. Install `gm` ```bash diff --git a/scripts/gm/bin/lib-gm b/scripts/gm/bin/lib-gm index 533993a2cd..e7c4714899 100644 --- a/scripts/gm/bin/lib-gm +++ b/scripts/gm/bin/lib-gm @@ -6,6 +6,14 @@ version() { echo "v0.0.1" } +config_defaults() { + GLOBAL_GAIAD_BINARY="$(which gaiad || echo "${GOPATH:-./gaiad}")" + #GLOBAL_PORTS_START_AT is deliberately not exported because it is updated sometimes during execution + GLOBAL_HOME_DIR="${HOME}/.gm" + GLOBAL_AUTO_MAINTAIN_CONFIG="true" + GLOBAL_ADD_TO_HERMES="false" +} + install() { mkdir -p "${HOME}/.gm/bin" cp "$0" "${HOME}/.gm/bin/gm" @@ -16,6 +24,7 @@ install() { chmod 644 "${HOME}/.gm/bin/shell-support" CONFIG_FILE="${HOME}/.gm/gm.toml" if [ ! -f "$CONFIG_FILE" ]; then + config_defaults write_default_config fi if [ -z "$(which stoml)" ]; then @@ -111,11 +120,11 @@ load_config() { export CONFIG_FILE="${SCRIPT_DIR}gm.toml" fi fi - GLOBAL_GAIAD_BINARY="$(eval echo "$(stoml -q "$CONFIG_FILE" global.gaiad_binary || which gaiad)")" - #GLOBAL_PORTS_START_AT is deliberately not exported because it is updated sometimes during execution - GLOBAL_HOME_DIR="$(eval echo "$(stoml -q "$CONFIG_FILE" global.home_dir || echo "${HOME}/.gm")")" - GLOBAL_AUTO_MAINTAIN_CONFIG="$(stoml -q "$CONFIG_FILE" global.auto_maintain_config || echo "true")" - GLOBAL_ADD_TO_HERMES="$(stoml -q "$CONFIG_FILE" global.add_to_hermes || echo "true")" + config_defaults + GLOBAL_GAIAD_BINARY="$(eval echo "$(stoml -q "$CONFIG_FILE" global.gaiad_binary)")" + GLOBAL_HOME_DIR="$(eval echo "$(stoml -q "$CONFIG_FILE" global.home_dir)")" + GLOBAL_AUTO_MAINTAIN_CONFIG="$(stoml -q "$CONFIG_FILE" global.auto_maintain_config)" + GLOBAL_ADD_TO_HERMES="$(stoml -q "$CONFIG_FILE" global.add_to_hermes)" export GLOBAL_GAIAD_BINARY export GLOBAL_HOME_DIR export GLOBAL_AUTO_MAINTAIN_CONFIG From a8edcdef89608e3367f5cc4b4bf93799eb53e320 Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Fri, 14 May 2021 15:26:13 -0400 Subject: [PATCH 12/29] fix hemres config ID --- scripts/gm/bin/lib-gm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gm/bin/lib-gm b/scripts/gm/bin/lib-gm index e7c4714899..94614472c2 100644 --- a/scripts/gm/bin/lib-gm +++ b/scripts/gm/bin/lib-gm @@ -538,7 +538,7 @@ EOF fi cat < Date: Sat, 15 May 2021 02:14:46 -0400 Subject: [PATCH 13/29] fixes-fixes and new feature unsafe-reset --- scripts/gm/README.md | 38 ++++++++-- scripts/gm/bin/gm | 86 +++++++++++++++++---- scripts/gm/bin/lib-gm | 141 ++++++++++++++++++++++++++++++----- scripts/gm/bin/shell-support | 36 ++++++++- scripts/gm/gm.toml | 9 ++- 5 files changed, 273 insertions(+), 37 deletions(-) diff --git a/scripts/gm/README.md b/scripts/gm/README.md index 3d2f5f2483..2ebe6dec68 100644 --- a/scripts/gm/README.md +++ b/scripts/gm/README.md @@ -85,7 +85,7 @@ the `gm.toml` file for node configuration. By default, newly created node config **Entries**: * `[global]` - the global section defines application-wide defaults. All the entries here can be overridden on a - per-node basis. + per-node basis, except if indicated otherwise. * `gaiad_binary` - Path to the `gaiad` binary to use. * `ports_start_at` * In the global section this defines the first free port to use for newly created nodes. @@ -99,8 +99,19 @@ the `gm.toml` file for node configuration. By default, newly created node config configuration. * `add_to_hermes` - this node should be part of the hermes config. * `network` - * Only in subsections: Define the network (the validator connection) for the full node. + * This is a sub-sections-only variable. It will be ignored in the `global` section. + * Defines the network (the validator connection) for the full node. * **Mandatory for full nodes**, does not exist for validator nodes. +* `hermes_binary` + * Optional variable. + * Path to the `hermes` binary to use. + * Only used in the `hermes` sub-commands. + * This is a global-only variable. It will be ignored in sub-sections. +* `hermes_config` + * Optional variable. + * The hermes configuration file path. + * Only used in the `hermes` sub-commands. + * This is a global-only variable. It will be ignored in sub-sections. Note: the user is welcome to create additional nodes outside the scope of `gm` on the local machine but `gm` will only manage nodes that are added to the configuration file. @@ -151,10 +162,21 @@ node4 PPROF: http://localhost:27054 ### `gm help` **Description**: shows the help screen -### `gm hermes` -**Description**: generate the `hermes.toml` config and print it on the screen. +### `gm hermes cc` +**Description**: create the `hermes create client` commands for a fully interconnected set of IBC calls and print them +on the screen. -Tip: `gm hermes > $HOME/.hermes/config.toml` if you know what you're doing. +Tip: Pick and choose the ones you want to create. + +### `gm hermes config` +**Description**: generate the hermes `config.toml` config file and write it to the defined config path. + +Tip: Do not run this command, if you value your current hermes config. It will be overwritten. + +### `gm hermes keys` +**Description**: add network keys to a hermes configuration. + +Tip: Make sure you set the `global.hermes_binary` entry in the config to a valid binary path. ### `gm install` **Description**: Install the `gm` command under `$HOME/.gm`, create a default configuration and warn about missing @@ -199,6 +221,12 @@ If no node is specified then it will stop all nodes. Tip: If a node was killed, you can use `gm stop` to clean up the PID file. +### `gm unsafe-reset [ ...]` +**Description**: Run `unsafe-reset-all` on the node(s). This will use the defined `gaiad` binary and configuration. +If no node is specified then it will run for all nodes. + +Tip: It will skip nodes that are running. + ### `gm version` **Description**: Display the version of `gm`. diff --git a/scripts/gm/bin/gm b/scripts/gm/bin/gm index 91ff2daaec..ce983f58f2 100755 --- a/scripts/gm/bin/gm +++ b/scripts/gm/bin/gm @@ -23,18 +23,34 @@ usage() { echo "Usage:" echo " gm [[]...]" echo - echo "COMMANDS DESCRIPTION" + echo "COMMANDS DESCRIPTION" echo - echo "help print this help and exit" - echo "hermes print the hermes config.toml" - echo "install install the script for the local user" - echo "keys print the keys of validator nodes" - echo "log print the log of a node" - echo "ports print the ports of a (running) node" - echo "start start one or more nodes (starts all nodes if no parameter is given)" - echo "status print the status of nodes" - echo "stop stop one or more nodes (stops all nodes if no parameter is given)" - echo "version print the application version" + echo "help print this help and exit" + echo "hermes sub-command for hermes-related configuration" + echo "install install the script for the local user" + echo "keys print the keys of validator nodes" + echo "log print the log of a node" + echo "ports print the ports of a (running) node" + echo "start start one or more nodes (starts all nodes if no parameter is given)" + echo "status print the status of nodes" + echo "stop stop one or more nodes (stops all nodes if no parameter is given)" + echo "unsafe-reset unsafe-reset-all one or more nodes (resets all nodes if no parameter is given)" + echo "version print the application version" + echo +} + +hermes_usage() { + echo "Gaiad Manager $(version) - hermes subcommand" + echo + echo "Usage:" + echo " gm hermes [[]...]" + echo + echo "SUBCOMMANDS DESCRIPTION" + echo + echo "help print this help and exit" + echo "config print the hermes config.toml" + echo "keys add keys to $HOME/.hermes" + echo "cc print a list of hermes commands to create IBC connections" echo } @@ -43,8 +59,38 @@ case "${1:-help}" in usage ;; hermes) - load_config - hermes + shift + case "${1:-help}" in + help) + hermes_usage + ;; + config) + load_config + hermes_config + ;; + keys) + load_config + if [ $# -eq 1 ]; then + NODES="$ALL_HERMES_NODES" + else + shift + NODES="$*" + fi + for i in $NODES + do + warn_unknown_hermes_node "$i" || continue + hermes_keys "$i" + done + ;; + cc) + load_config + hermes_cc + ;; + *) + hermes_usage + exit_with_error "could not understand the command hermes \"$1\"" + ;; + esac ;; install) install @@ -148,6 +194,20 @@ case "${1:-help}" in stop "$i" done ;; + unsafe-reset) + load_config + if [ $# -eq 1 ]; then + NODES_TO_STOP="$ALL_NODES" + else + shift + NODES_TO_STOP="$*" + fi + for i in $NODES_TO_STOP + do + warn_unknown_node "$i" || continue + unsafe_reset "$i" + done + ;; version) version exit 0 diff --git a/scripts/gm/bin/lib-gm b/scripts/gm/bin/lib-gm index 94614472c2..b26f68a240 100644 --- a/scripts/gm/bin/lib-gm +++ b/scripts/gm/bin/lib-gm @@ -1,6 +1,8 @@ #!/usr/bin/env sh -#set -x set -eu +if [ -n "${DEBUG:-}" ]; then + set -x +fi version() { echo "v0.0.1" @@ -8,8 +10,10 @@ version() { config_defaults() { GLOBAL_GAIAD_BINARY="$(which gaiad || echo "${GOPATH:-./gaiad}")" + GLOBAL_HERMES_BINARY="$(which hermes || echo "./hermes")" #GLOBAL_PORTS_START_AT is deliberately not exported because it is updated sometimes during execution GLOBAL_HOME_DIR="${HOME}/.gm" + GLOBAL_HERMES_CONFIG="${HOME}/.hermes/hermes.toml" GLOBAL_AUTO_MAINTAIN_CONFIG="true" GLOBAL_ADD_TO_HERMES="false" } @@ -23,7 +27,9 @@ install() { cp "${0%%gm}shell-support" "${HOME}/.gm/bin/shell-support" chmod 644 "${HOME}/.gm/bin/shell-support" CONFIG_FILE="${HOME}/.gm/gm.toml" - if [ ! -f "$CONFIG_FILE" ]; then + if [ -f "$CONFIG_FILE" ]; then + warn "Config file already exists. Please double-check the documentation to see if all necessary configuration items are set." + else config_defaults write_default_config fi @@ -68,6 +74,13 @@ warn_unknown_node() { fi } +warn_unknown_hermes_node() { + if ! a_in_b "$1" "$ALL_HERMES_NODES"; then + warn "not a hermes node $1, skipping..." + return 1 + fi +} + exit_with_error() { echo "ERROR: $*, exiting..." return 1 @@ -102,6 +115,24 @@ a_in_b_comma_separated() { test -n "$(echo "$2" | grep '\(^\|,\) *'"${1}"' *\(,\|$\)')" } +# Return the n-th value from a comma-separated list. +# Starts with 1. +n_from_a() { + i=1 + N=$(($1)) + shift + LIST="$*" + for j in $LIST + do + if [ $((N-i)) -eq 0 ]; then + echo "$j" + return 0 + fi + i=$((i+1)) + done + return 1 +} + load_all_sections() { grep '^ *\[.\+\] *$' "$CONFIG_FILE" | sed 's/^ *\[\([^]]*\)\] *$/\1/' } @@ -121,18 +152,23 @@ load_config() { fi fi config_defaults - GLOBAL_GAIAD_BINARY="$(eval echo "$(stoml -q "$CONFIG_FILE" global.gaiad_binary)")" - GLOBAL_HOME_DIR="$(eval echo "$(stoml -q "$CONFIG_FILE" global.home_dir)")" - GLOBAL_AUTO_MAINTAIN_CONFIG="$(stoml -q "$CONFIG_FILE" global.auto_maintain_config)" - GLOBAL_ADD_TO_HERMES="$(stoml -q "$CONFIG_FILE" global.add_to_hermes)" + GLOBAL_GAIAD_BINARY="$(eval echo "$(stoml -q "$CONFIG_FILE" global.gaiad_binary || echo "$GLOBAL_GAIAD_BINARY")")" + GLOBAL_HERMES_BINARY="$(eval echo "$(stoml -q "$CONFIG_FILE" global.hermes_binary || echo "$GLOBAL_HERMES_BINARY")")" + GLOBAL_HOME_DIR="$(eval echo "$(stoml -q "$CONFIG_FILE" global.home_dir || echo "$GLOBAL_HOME_DIR")")" + GLOBAL_HERMES_CONFIG="$(eval echo "$(stoml -q "$CONFIG_FILE" global.hermes_config || echo "$GLOBAL_HERMES_CONFIG")")" + GLOBAL_AUTO_MAINTAIN_CONFIG="$(stoml -q "$CONFIG_FILE" global.auto_maintain_config || echo "$GLOBAL_AUTO_MAINTAIN_CONFIG")" + GLOBAL_ADD_TO_HERMES="$(stoml -q "$CONFIG_FILE" global.add_to_hermes || echo "$GLOBAL_ADD_TO_HERMES")" export GLOBAL_GAIAD_BINARY + export GLOBAL_HERMES_BINARY export GLOBAL_HOME_DIR + export GLOBAL_HERMES_CONFIG export GLOBAL_AUTO_MAINTAIN_CONFIG export GLOBAL_ADD_TO_HERMES if [ "$FRESH_CONFIG" = "yes" ]; then write_default_config VALIDATORS="" FULL_NODES="" + ALL_HERMES_NODES="" else stoml "$CONFIG_FILE" global 1> /dev/null || exit_with_error invalid config file. Make sure all strings are quoted test -x "$GLOBAL_GAIAD_BINARY" || exit_with_error "gaiad binary cannot be executed: $GLOBAL_GAIAD_BINARY" @@ -143,6 +179,7 @@ load_config() { # End of workaround VALIDATORS="" RAW_NODES="" + ALL_HERMES_NODES="" for i in $RAW_SECTIONS do if [ "$i" = "global" ]; then @@ -150,6 +187,9 @@ load_config() { fi if [ -z "$(stoml "$CONFIG_FILE" "${i}.network")" ]; then VALIDATORS="$VALIDATORS $i" + if get_add_to_hermes "$i"; then + ALL_HERMES_NODES="$ALL_HERMES_NODES $i" + fi else RAW_NODES="$RAW_NODES $i" fi @@ -163,10 +203,17 @@ load_config() { continue fi FULL_NODES="${FULL_NODES} $i" + if get_add_to_hermes "$i"; then + ALL_HERMES_NODES="$ALL_HERMES_NODES $i" + fi done fi + VALIDATORS="${VALIDATORS## }" + FULL_NODES="${FULL_NODES## }" + ALL_HERMES_NODES="${ALL_HERMES_NODES## }" export VALIDATORS export FULL_NODES + export ALL_HERMES_NODES export ALL_NODES="$VALIDATORS $FULL_NODES" # Workaround for issue: https://github.com/spf13/viper/issues/1131 # Fix "stoml" so it reads empty sections too. @@ -187,6 +234,8 @@ ports_start_at=27000 home_dir="$GLOBAL_HOME_DIR" auto_maintain_config=$GLOBAL_AUTO_MAINTAIN_CONFIG add_to_hermes=$GLOBAL_ADD_TO_HERMES +hermes_binary="$GLOBAL_HERMES_BINARY" +hermes_config="$GLOBAL_HERMES_CONFIG" EOF set -e } @@ -262,6 +311,16 @@ get_network() { echo "$RESULT" } +# Note: this depends on one-validator nodes. +# We might want to change it to `stoml -q "${HOME_DIR}/config/genesis.json" "chain_id"` later. +get_chain_id() { + if a_in_b "$1" "$VALIDATORS"; then + echo "$1" + else + get_network "$1" + fi +} + get_add_to_hermes() { RESULT="$(stoml -q "$CONFIG_FILE" "${1}.add_to_hermes")" if [ -z "$RESULT" ]; then @@ -374,10 +433,19 @@ start() { GAIAD="$(get_gaiad_binary "$1")" HOME_DIR="$(get_home_dir "$1")" GAIAD_LOG="${HOME_DIR}/log" + VALIDATION="$("$GAIAD" validate-genesis --home "$HOME_DIR" > "$GAIAD_LOG" 2>&1 || echo "ERR")" + if [ "$VALIDATION" = "ERR" ]; then + warn "invalid genesis.json for ${1}. Please check the log and fix manually. Skipping..." + return 0 + fi nohup "$GAIAD" start --x-crisis-skip-assert-invariants --home "$HOME_DIR" > "$GAIAD_LOG" 2>&1 & GAIAD_PID=$! echo "$GAIAD_PID" > "$HOME_DIR/pid" echo "$i started, PID: $GAIAD_PID, LOG: $GAIAD_LOG" + sleep 1 + if ! is_running "$1"; then + warn "$1 failed. Check the logs with \`gm log $1\`." + fi fi } @@ -389,11 +457,11 @@ stop() { kill -TERM "$GAIAD_PID" 2> /dev/null && rm -f "$HOME_DIR/pid" RESULT="$?" if [ "$RESULT" != "0" ]; then - warn "SIGTERM failed on PID ${GAIAD_PID}. Trying KILL..." + warn "SIGTERM failed on PID ${GAIAD_PID} for $1. Trying KILL..." kill -KILL "$GAIAD_PID" && rm -f "$HOME_DIR/pid" RESULT="$?" if [ "$RESULT" != "0" ]; then - warn "SIGKILL failed on PID ${GAIAD_PID}. Giving up..." + warn "SIGKILL failed on PID ${GAIAD_PID} for $1. Giving up..." fi fi else @@ -401,9 +469,9 @@ stop() { if [ -f "${HOME_DIR}/pid" ]; then GAIAD_PID="$(cat "${HOME_DIR}/pid")" if [ -n "$GAIAD_PID" ]; then - warn "no process with PID $GAIAD_PID. Removing stale data." + warn "no process with PID $GAIAD_PID found for $1. Removing stale data." else - warn "no process ID found. Removing stale data." + warn "no process ID found for $1. Removing stale data." fi rm -f "$HOME_DIR/pid" fi @@ -518,8 +586,8 @@ list_keys() { done } -hermes() { - cat < "$GLOBAL_HERMES_CONFIG" [global] strategy = 'naive' log_level = 'info' @@ -532,11 +600,8 @@ EOF fi RPC="$(get_rpc_port "$i")" GRPC="$(get_grpc_port "$i")" - ID="$(get_network "$i" || echo "")" - if [ -z "$ID" ]; then - ID="$i" - fi - cat <> "$GLOBAL_HERMES_CONFIG" [[chains]] id='${ID}' rpc_addr='http://localhost:${RPC}' @@ -555,3 +620,45 @@ denominator = '3' EOF done } + +hermes_keys() { + ID="$(get_chain_id "$1")" + NETWORK_HOME_DIR="$(get_home_dir "$ID")" + test -x "$GLOBAL_HERMES_BINARY" || exit_with_error "hermes binary \"${GLOBAL_HERMES_BINARY}\" not found, check your gm.toml config." + if [ -z "$GLOBAL_HERMES_CONFIG" ]; then + "$GLOBAL_HERMES_BINARY" keys add "$ID" -f "${NETWORK_HOME_DIR}/wallet_seed.json" + else + "$GLOBAL_HERMES_BINARY" -c "$GLOBAL_HERMES_CONFIG" keys add "$ID" -f "${NETWORK_HOME_DIR}/wallet_seed.json" + fi +} + +hermes_cc() { + CHAINS="" + for i in $ALL_HERMES_NODES + do + if ! a_in_b "$i" "$CHAINS"; then + ID="$(get_chain_id "$i")" + CHAINS="$CHAINS $ID" + fi + done + CHAINS="${CHAINS## }" + N="$(($(wc -w <<< "$CHAINS")))" + for i in $(seq 1 $((N-1))) + do + for j in $(seq $((i+1)) $N) + do + echo "\"${GLOBAL_HERMES_BINARY}\" create channel $(n_from_a "$i" "$CHAINS") $(n_from_a "$j" "$CHAINS") --port-a transfer --port-b transfer" + done + done +} + +unsafe_reset() { + if is_running "$1"; then + warn "$1 is already running, skipping" + else + GAIAD="$(get_gaiad_binary "$1")" + HOME_DIR="$(get_home_dir "$1")" + echo "Resetting $1..." + "$GAIAD" unsafe-reset-all --home "$HOME_DIR" + fi +} diff --git a/scripts/gm/bin/shell-support b/scripts/gm/bin/shell-support index a0add42e74..537d78667a 100644 --- a/scripts/gm/bin/shell-support +++ b/scripts/gm/bin/shell-support @@ -18,6 +18,26 @@ __gm_get_nodes() { fi } +__gm_hermes_get_nodes() { + if [ -f "$HOME/.gm/gm.toml" ] && [ -n "$(which stoml)" ]; then + local NODES="" + local GLOBAL_ADD_TO_HERMES="$(stoml "$HOME/.gm/gm.toml" global.add_to_hermes || echo "false")" + if [ "$GLOBAL_ADD_TO_HERMES" != "true" ]; then + GLOBAL_ADD_TO_HERMES="false" + fi + for node in $(__gm_get_nodes) + do + local ADD_TO_HERMES="$(stoml "$HOME/.gm/gm.toml" "${node}.add_to_hermes" || echo "${GLOBAL_ADD_TO_HERMES}")" + if [ "$ADD_TO_HERMES" == "true" ]; then + NODES="$NODES $node" + fi + done + echo "$NODES" + else + __gm_get_nodes + fi +} + _gm_nodes() { local cur="${COMP_WORDS[COMP_CWORD]}" local cmds="$(__gm_get_nodes)" @@ -30,9 +50,21 @@ _gm_nodes_f() { while read -r line; do COMPREPLY+=("$line"); done < <(compgen -W "$cmds" -- "$cur") } +_gm_hermes_nodes() { + local cur="${COMP_WORDS[COMP_CWORD]}" + local cmds="$(__gm_hermes_get_nodes)" + while read -r line; do COMPREPLY+=("$line"); done < <(compgen -W "$cmds" -- "$cur") +} + _gm_complete_commands() { local cur="${COMP_WORDS[COMP_CWORD]}" - local cmds="help hermes install keys log ports start status stop version" + local cmds="help hermes install keys log ports start status stop unsafe-reset version" + while read -r line; do COMPREPLY+=("$line"); done < <(compgen -W "$cmds" -- "$cur") +} + +_gm_hermes_complete_commands() { + local cur="${COMP_WORDS[COMP_CWORD]}" + local cmds="help config keys cc" while read -r line; do COMPREPLY+=("$line"); done < <(compgen -W "$cmds" -- "$cur") } @@ -66,6 +98,7 @@ _gm() { # subcommands have their own completion functions case "$cmd" in + hermes) _gm_hermes_complete_commands ;; keys) _gm_nodes ;; log) _gm_nodes_f ;; ports) _gm_nodes ;; @@ -74,6 +107,7 @@ _gm() { *) ;; esac } +# Todo: hermes subcommands can use some shell-completion love complete -o bashdefault -o default -F _gm gm diff --git a/scripts/gm/gm.toml b/scripts/gm/gm.toml index acb4cb9dde..53f8c3e8f5 100644 --- a/scripts/gm/gm.toml +++ b/scripts/gm/gm.toml @@ -1,6 +1,7 @@ # Example configuration with one validator and one full node -# the global section defines application-wide defaults. All the entries here can be overridden on a per-node basis. +# the global section defines application-wide defaults. All the entries here can be overridden on a per-node basis, +# except if otherwise noted. [global] # Path to the `gaiad` binary to use. @@ -19,6 +20,12 @@ auto_maintain_config=true # This node should be part of the hermes config. add_to_hermes=false +# (Optional) Path to the `hermes` binary to use. Global-only variable. Ignored in sub-sections. +hermes_binary="./hermes" + +# (Optional) Hermes configuration file path. Ignored in sub-sections. +hermes_config="$HOME/.hermes/config.toml" + ## ## Sub-section (node) definitions and parameter redefinitons below. ## From 77e552936567fe640c3466f71587213acabf8d00 Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Sat, 15 May 2021 02:16:44 -0400 Subject: [PATCH 14/29] default typo fix --- scripts/gm/bin/lib-gm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gm/bin/lib-gm b/scripts/gm/bin/lib-gm index b26f68a240..a4bfef346e 100644 --- a/scripts/gm/bin/lib-gm +++ b/scripts/gm/bin/lib-gm @@ -13,7 +13,7 @@ config_defaults() { GLOBAL_HERMES_BINARY="$(which hermes || echo "./hermes")" #GLOBAL_PORTS_START_AT is deliberately not exported because it is updated sometimes during execution GLOBAL_HOME_DIR="${HOME}/.gm" - GLOBAL_HERMES_CONFIG="${HOME}/.hermes/hermes.toml" + GLOBAL_HERMES_CONFIG="${HOME}/.hermes/config.toml" GLOBAL_AUTO_MAINTAIN_CONFIG="true" GLOBAL_ADD_TO_HERMES="false" } From f8ed012e3c3df4eb231a2fde395db35ee290081a Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Sat, 15 May 2021 03:01:30 -0400 Subject: [PATCH 15/29] README update: full-mesh config --- scripts/gm/README.md | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/scripts/gm/README.md b/scripts/gm/README.md index 2ebe6dec68..79dc3ac801 100644 --- a/scripts/gm/README.md +++ b/scripts/gm/README.md @@ -241,11 +241,11 @@ have a full node and the hermes config is going through the full nodes instead o You can get the hermes configuration automatically. -You need to manually run the `hermes create channel` commands to create the IBC paths as well as the relayer commands. +You might need to replace the value of the `gaiad_binary` entry, if you don't set `$GOPATH` in your regular executions. -(Note: maybe it's a good feature request to automate this in the next release.) +The same is true for `hermes_binary`. -You might need to replace the value of the `gaiad_binary` entry, if you don't set `$GOPATH` in your regular executions. +`gm.toml`: ```toml [global] gaiad_binary="$GOPATH/bin/gaiad" @@ -253,6 +253,8 @@ ports_start_at=27000 home_dir="$HOME/.gm" auto_maintain_config=true add_to_hermes=false +hermes_binary="./hermes" +hermes_config="$HOME/.hermes/config.toml" [network1] [network2] @@ -276,5 +278,20 @@ add_to_hermes=true network="network5" add_to_hermes=true ``` +(Ports will be auto-assigned and written in the configuration file on the first start.) + +Run the below: +```bash +gm start +gm hermes config +gm hermes keys +gm hermes cc +``` + +This will +* create the node configuration and start all nodes +* generate the keys for hermes +* generate the config for hermes +* print the `create client` commands for a full-mesh connection among the IBC node networks. -Ports will be auto-assigned and written in the configuration file on the first start. +Pick and choose the connections from the list that you want to create. From f219334e299638eea686d9bd1437ca6d619f126b Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Sat, 15 May 2021 03:26:03 -0400 Subject: [PATCH 16/29] fix dash issue --- scripts/gm/bin/lib-gm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gm/bin/lib-gm b/scripts/gm/bin/lib-gm index a4bfef346e..ee1142af26 100644 --- a/scripts/gm/bin/lib-gm +++ b/scripts/gm/bin/lib-gm @@ -351,7 +351,7 @@ create() { mv "${HOME_ROOT}/${i}v0" "${HOME_DIR}" mv "${HOME_ROOT}/gentxs" "${HOME_DIR}" $GAIAD keys add "wallet" --keyring-backend test --keyring-dir "${HOME_DIR}" --output json > "${HOME_DIR}/wallet_seed.json" - $GAIAD add-genesis-account "wallet" "10000000stake,10000${i}coin" --keyring-backend test --home "${HOME_DIR}" + $GAIAD add-genesis-account "wallet" "10000000stake,10000localcoin" --keyring-backend test --home "${HOME_DIR}" fi else NETWORK="$(get_network "$i")" From 6324fe9bfabb219a1057e0cdf436eb5990c0ffbf Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Tue, 18 May 2021 09:49:31 -0400 Subject: [PATCH 17/29] typo fix --- scripts/gm/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/gm/README.md b/scripts/gm/README.md index 79dc3ac801..4efd4214f0 100644 --- a/scripts/gm/README.md +++ b/scripts/gm/README.md @@ -53,6 +53,7 @@ ibc-rs/scripts/gm/bin/gm install Alternatively, you can create the folder `$HOME/.gm/bin` and copy the files from `scripts/gm/bin` in there. The rest is just fluff. + 3. Activate `gm` * Add `source $HOME/.gm/bin/shell-support` to a file that executes when a new terminal window comes up (`$HOME/.bash_profile` or `$HOME/.bashrc`) From 6e6a534d5a3f6b6598209a9153e67200d01cf7df Mon Sep 17 00:00:00 2001 From: Greg Szabo <16846635+greg-szabo@users.noreply.github.com> Date: Tue, 18 May 2021 13:41:33 -0400 Subject: [PATCH 18/29] Update scripts/gm/README.md Co-authored-by: Adi Seredinschi --- scripts/gm/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gm/README.md b/scripts/gm/README.md index 4efd4214f0..0447ebfac7 100644 --- a/scripts/gm/README.md +++ b/scripts/gm/README.md @@ -164,7 +164,7 @@ node4 PPROF: http://localhost:27054 **Description**: shows the help screen ### `gm hermes cc` -**Description**: create the `hermes create client` commands for a fully interconnected set of IBC calls and print them +**Description**: create and print the `hermes create channel` commands to obtain a fully interconnected IBC mesh on the screen. Tip: Pick and choose the ones you want to create. From a4c35551e857af8cd4cba6db1893dab4d7b2cbee Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Thu, 20 May 2021 15:00:00 -0400 Subject: [PATCH 19/29] Added rm command, renamed reset command, added wallet_mnemonics config param --- scripts/gm/README.md | 17 +++-- scripts/gm/bin/gm | 23 ++++++- scripts/gm/bin/lib-gm | 141 +++++++++++++++++++++++++++++++++++------- scripts/gm/gm.toml | 3 + 4 files changed, 155 insertions(+), 29 deletions(-) diff --git a/scripts/gm/README.md b/scripts/gm/README.md index 0447ebfac7..d463442a12 100644 --- a/scripts/gm/README.md +++ b/scripts/gm/README.md @@ -96,6 +96,7 @@ the `gm.toml` file for node configuration. By default, newly created node config * `home_dir` * In the global section this defines the default folder where all node folders will be created. * In the per-node configuration this is the folder for the node configuration. +* `wallet_mnemonic` - define a 24-word seed phrase to use for wallet address(es) instead of creating random wallets. * `auto_maintain_config` - automatically update the `persistent_peers` and `unconditional_peer_ids` sections of the node configuration. * `add_to_hermes` - this node should be part of the hermes config. @@ -213,6 +214,9 @@ were killed for any reason, the `pid` configuration will be updated, and a fresh **Description**: List all nodes and their basic properties, such as: their PID if they are running, their home folder, and the most common ports used. +Home folders in brackets mean the configuration was not created yet. Configuration is automatically created during +startup of a node. + Tip: PIDs in brackets mean that the node is not running when `gm` started them. This could be because of a configuration error or maybe the user simply killed the PID. No worries, `gm` will clean up when `start` or `stop` is invoked. @@ -222,11 +226,16 @@ If no node is specified then it will stop all nodes. Tip: If a node was killed, you can use `gm stop` to clean up the PID file. -### `gm unsafe-reset [ ...]` -**Description**: Run `unsafe-reset-all` on the node(s). This will use the defined `gaiad` binary and configuration. -If no node is specified then it will run for all nodes. +### `gm reset [ ...]` +**Description**: Run `unsafe-reset-all` on the node(s) and reset the node database. This will use the defined +`gaiad` binary and configuration. If no node is specified then it will run for all nodes. + +Tip: It will stop nodes that are running and restart them after the database reset. + +### `gm rm [ ...]` +**Description**: Delete the node configuration. At least one node has to be specified. -Tip: It will skip nodes that are running. +Tip: It will stop nodes that are running. ### `gm version` **Description**: Display the version of `gm`. diff --git a/scripts/gm/bin/gm b/scripts/gm/bin/gm index ce983f58f2..f77bd42b85 100755 --- a/scripts/gm/bin/gm +++ b/scripts/gm/bin/gm @@ -34,7 +34,8 @@ usage() { echo "start start one or more nodes (starts all nodes if no parameter is given)" echo "status print the status of nodes" echo "stop stop one or more nodes (stops all nodes if no parameter is given)" - echo "unsafe-reset unsafe-reset-all one or more nodes (resets all nodes if no parameter is given)" + echo "reset reset one or more nodes' database (resets all nodes if no parameter is given)" + echo "rm delete the configuration of a node" echo "version print the application version" echo } @@ -194,7 +195,7 @@ case "${1:-help}" in stop "$i" done ;; - unsafe-reset) + reset) load_config if [ $# -eq 1 ]; then NODES_TO_STOP="$ALL_NODES" @@ -205,7 +206,23 @@ case "${1:-help}" in for i in $NODES_TO_STOP do warn_unknown_node "$i" || continue - unsafe_reset "$i" + reset "$i" + done + ;; + rm) + load_config + shift + if [ "$#" -eq 0 ]; then + exit_with_error "no node was given" + fi + while [ "$#" -gt 0 ]; + do + if is_running "$1"; then + stop "$1" + sleep 1 + fi + safer_rm "$1" + shift done ;; version) diff --git a/scripts/gm/bin/lib-gm b/scripts/gm/bin/lib-gm index ee1142af26..544167910e 100644 --- a/scripts/gm/bin/lib-gm +++ b/scripts/gm/bin/lib-gm @@ -16,6 +16,7 @@ config_defaults() { GLOBAL_HERMES_CONFIG="${HOME}/.hermes/config.toml" GLOBAL_AUTO_MAINTAIN_CONFIG="true" GLOBAL_ADD_TO_HERMES="false" + GLOBAL_WALLET_MNEMONIC="" } install() { @@ -133,6 +134,27 @@ n_from_a() { return 1 } +# Todo: do we want to use this? +is_configuration_valid() { + if ! a_in_b "$1" "$ALL_NODES"; then + return 1 + fi + HOME_DIR="$(get_home_dir "$1")" + for d in "$HOME_DIR" "${HOME_DIR}/config" "${HOME_DIR}/data" + do + if [ ! -d "$d" ]; then + return 1 + fi + done + for f in "${HOME_DIR}/config/config.toml" "${HOME_DIR}/config/app.toml" "${HOME_DIR}/config/node_key.json" + do + if [ ! -f "$f" ]; then + return 1 + fi + done + return 0 +} + load_all_sections() { grep '^ *\[.\+\] *$' "$CONFIG_FILE" | sed 's/^ *\[\([^]]*\)\] *$/\1/' } @@ -158,12 +180,14 @@ load_config() { GLOBAL_HERMES_CONFIG="$(eval echo "$(stoml -q "$CONFIG_FILE" global.hermes_config || echo "$GLOBAL_HERMES_CONFIG")")" GLOBAL_AUTO_MAINTAIN_CONFIG="$(stoml -q "$CONFIG_FILE" global.auto_maintain_config || echo "$GLOBAL_AUTO_MAINTAIN_CONFIG")" GLOBAL_ADD_TO_HERMES="$(stoml -q "$CONFIG_FILE" global.add_to_hermes || echo "$GLOBAL_ADD_TO_HERMES")" + GLOBAL_WALLET_MNEMONIC="$(stoml -q "$CONFIG_FILE" global.wallet_mnemonic || echo "$GLOBAL_WALLET_MNEMONIC")" export GLOBAL_GAIAD_BINARY export GLOBAL_HERMES_BINARY export GLOBAL_HOME_DIR export GLOBAL_HERMES_CONFIG export GLOBAL_AUTO_MAINTAIN_CONFIG export GLOBAL_ADD_TO_HERMES + export GLOBAL_WALLET_MNEMONIC if [ "$FRESH_CONFIG" = "yes" ]; then write_default_config VALIDATORS="" @@ -236,6 +260,7 @@ auto_maintain_config=$GLOBAL_AUTO_MAINTAIN_CONFIG add_to_hermes=$GLOBAL_ADD_TO_HERMES hermes_binary="$GLOBAL_HERMES_BINARY" hermes_config="$GLOBAL_HERMES_CONFIG" +wallet_mnemonic="$GLOBAL_WALLET_MNEMONIC" EOF set -e } @@ -331,31 +356,52 @@ get_add_to_hermes() { } get_node_id() { - GAIAD="$(get_gaiad_binary "$1")" + GAIAD_BINARY="$(get_gaiad_binary "$1")" HOME_DIR="$(get_home_dir "$1")" - $GAIAD tendermint show-node-id --home "$HOME_DIR" + # Todo: Shall we change this to is_configuration_valid? + if [ ! -f "${HOME_DIR}/config/node_key.json" ]; then + warn "Invalid configuration: no node key found for $1" + echo "DEADBEEFDEADBEEF" + else + "$GAIAD_BINARY" tendermint show-node-id --home "$HOME_DIR" + fi +} + +get_wallet_mnemonic() { + RESULT="$(stoml -q "$CONFIG_FILE" "${1}.wallet_mnemonic")" + if [ -z "$RESULT" ]; then + echo "$GLOBAL_WALLET_MNEMONIC" + else + echo "$RESULT" + fi } create() { i="$1" echo "Creating $i config..." - GAIAD="$(get_gaiad_binary "$i")" + GAIAD_BINARY="$(get_gaiad_binary "$i")" HOME_DIR="$(get_home_dir "$i")" HOME_ROOT="${HOME_DIR%%${i}}" if a_in_b "$i" "$VALIDATORS"; then - EXEC_RESULT="$("$GAIAD" testnet --chain-id "$i" --keyring-backend test --node-dir-prefix "${i}v" -o "$HOME_ROOT" --node-daemon-home . --v 1 2>&1)" + EXEC_RESULT="$("$GAIAD_BINARY" testnet --chain-id "$i" --keyring-backend test --node-dir-prefix "${i}v" -o "$HOME_ROOT" --node-daemon-home . --v 1 2>&1)" if [ "$EXEC_RESULT" != "Successfully initialized 1 node directories" ]; then warn "could not create config for ${i}: \"$EXEC_RESULT\", skipping..." return 1 else mv "${HOME_ROOT}/${i}v0" "${HOME_DIR}" mv "${HOME_ROOT}/gentxs" "${HOME_DIR}" - $GAIAD keys add "wallet" --keyring-backend test --keyring-dir "${HOME_DIR}" --output json > "${HOME_DIR}/wallet_seed.json" - $GAIAD add-genesis-account "wallet" "10000000stake,10000localcoin" --keyring-backend test --home "${HOME_DIR}" + WALLET_MNEMONIC="$(get_wallet_mnemonic "$i")" + if [ -z "$WALLET_MNEMONIC" ]; then + "$GAIAD_BINARY" keys add "wallet" --keyring-backend test --keyring-dir "${HOME_DIR}" --output json > "${HOME_DIR}/wallet_seed.json" + else + echo "$WALLET_MNEMONIC" | "$GAIAD_BINARY" keys add "wallet" --recover --keyring-backend test --keyring-dir "${HOME_DIR}" --output json > "${HOME_DIR}/wallet_seed.json" + sconfig "${HOME_DIR}/wallet_seed.json" -t string "mnemonic=${WALLET_MNEMONIC}" + fi + "$GAIAD_BINARY" add-genesis-account "wallet" "10000000stake,10000localcoin" --keyring-backend test --home "${HOME_DIR}" fi else NETWORK="$(get_network "$i")" - EXEC_RESULT="$("$GAIAD" testnet --chain-id "$NETWORK" --keyring-backend test --node-dir-prefix "${i}n" -o "$HOME_ROOT" --node-daemon-home . --v 1 2>&1)" + EXEC_RESULT="$("$GAIAD_BINARY" testnet --chain-id "$NETWORK" --keyring-backend test --node-dir-prefix "${i}n" -o "$HOME_ROOT" --node-daemon-home . --v 1 2>&1)" if [ "$EXEC_RESULT" != "Successfully initialized 1 node directories" ]; then warn "could not create config for ${i}: \"$EXEC_RESULT\", skipping..." return 1 @@ -430,15 +476,15 @@ start() { if is_running "$1"; then warn "$1 is already running, skipping" else - GAIAD="$(get_gaiad_binary "$1")" + GAIAD_BINARY="$(get_gaiad_binary "$1")" HOME_DIR="$(get_home_dir "$1")" GAIAD_LOG="${HOME_DIR}/log" - VALIDATION="$("$GAIAD" validate-genesis --home "$HOME_DIR" > "$GAIAD_LOG" 2>&1 || echo "ERR")" + VALIDATION="$("$GAIAD_BINARY" validate-genesis --home "$HOME_DIR" > "$GAIAD_LOG" 2>&1 || echo "ERR")" if [ "$VALIDATION" = "ERR" ]; then warn "invalid genesis.json for ${1}. Please check the log and fix manually. Skipping..." return 0 fi - nohup "$GAIAD" start --x-crisis-skip-assert-invariants --home "$HOME_DIR" > "$GAIAD_LOG" 2>&1 & + nohup "$GAIAD_BINARY" start --x-crisis-skip-assert-invariants --home "$HOME_DIR" > "$GAIAD_LOG" 2>&1 & GAIAD_PID=$! echo "$GAIAD_PID" > "$HOME_DIR/pid" echo "$i started, PID: $GAIAD_PID, LOG: $GAIAD_LOG" @@ -479,7 +525,7 @@ stop() { } print_header_line() { - echo "NODE PID RPC APP GRPC HOME_DIR" + echo "NODE PID RPC APP GRPC HOME_DIR" } print_status_line() { @@ -493,6 +539,10 @@ print_status_line() { done fi HOME_DIR="$(get_home_dir "$1")" + HOME_DIR_PRINTED=" $HOME_DIR " + if [ ! -d "$HOME_DIR" ]; then + HOME_DIR_PRINTED="(${HOME_DIR})" + fi GAIAD_PID_FILE="${HOME_DIR}/pid" if [ -f "$GAIAD_PID_FILE" ]; then GAIAD_PID="$(cat "$GAIAD_PID_FILE")" @@ -500,12 +550,12 @@ print_status_line() { GAIAD_PID=" N/A " fi if [ "$(ps -p "$GAIAD_PID" -o pid | wc -l)" -eq 2 ]; then - echo "${NAME}${NAME_PAD} $GAIAD_PID $(get_rpc_port "$1") $(get_app_port "$1") $(get_grpc_port "$1") $HOME_DIR" + echo "${NAME}${NAME_PAD} $GAIAD_PID $(get_rpc_port "$1") $(get_app_port "$1") $(get_grpc_port "$1") $HOME_DIR_PRINTED" else - echo "${NAME}${NAME_PAD}($GAIAD_PID) - - - $HOME_DIR" + echo "${NAME}${NAME_PAD}($GAIAD_PID) - - - $HOME_DIR_PRINTED" fi else - echo "${NAME}${NAME_PAD} - - - - $HOME_DIR" + echo "${NAME}${NAME_PAD} - - - - $HOME_DIR_PRINTED" fi } @@ -541,6 +591,11 @@ ports() { list_keys() { HOME_DIR="$(get_home_dir "$1")" + # Todo: Should we change this to is_configuration_valid? + if [ ! -d "$HOME_DIR" ]; then + warn "No configuration at ${HOME_DIR}. Skipping..." + return 0 + fi GAIAD_BINARY="$(get_gaiad_binary "$1")" echo "\"$GAIAD_BINARY\" keys list --keyring-backend test --keyring-dir \"$HOME_DIR\"" KEY_NAME="" @@ -642,7 +697,7 @@ hermes_cc() { fi done CHAINS="${CHAINS## }" - N="$(($(wc -w <<< "$CHAINS")))" + N="$(($(echo "$CHAINS" | wc -w)))" for i in $(seq 1 $((N-1))) do for j in $(seq $((i+1)) $N) @@ -652,13 +707,55 @@ hermes_cc() { done } -unsafe_reset() { - if is_running "$1"; then - warn "$1 is already running, skipping" +reset() { + WAS_RUNNING="$(is_running "$1" || echo "no")" + if [ -z "$WAS_RUNNING" ]; then + if a_in_b "$1" "$VALIDATORS"; then + warn "After reset all full nodes will need to be reset too." + fi + stop "$1" + fi + GAIAD_BINARY="$(get_gaiad_binary "$1")" + HOME_DIR="$(get_home_dir "$1")" + echo "Resetting $1..." + # Todo: Should we change this to is_configuration_valid? + if [ ! -d "$HOME_DIR" ]; then + warn "No configuration at ${HOME_DIR}. Skipping..." + return 0 + fi + "$GAIAD_BINARY" unsafe-reset-all --home "$HOME_DIR" + if [ -z "$WAS_RUNNING" ]; then + start "$1" + fi +} + +# Guard against removing arbitrary folders/files, only remove folders that have at least a half-baked node configuration. +safer_rm() { + if [ $# -eq 1 ]; then + if a_in_b "$1" "$ALL_NODES"; then + HOME_DIR="$(get_home_dir "$1")" + if [ -d "$HOME_DIR" ]; then + if [ -d "$HOME_DIR/config" ]; then + if [ -f "$HOME_DIR/config/config.toml" ]; then + if [ -d "$HOME_DIR/data" ]; then + echo "Executing \"rm -r $HOME_DIR\"..." + rm -r "$HOME_DIR" + else + warn "Anxious to run \"rm -r ${HOME_DIR}\" automatically, folder does not contain data folder, skipping..." + fi + else + warn "Anxious to run \"rm -r ${HOME_DIR}\" automatically, folder does not contain config file, skipping..." + fi + else + warn "Anxious to run \"rm -r ${HOME_DIR}\" automatically, folder does not contain config folder, skipping..." + fi + else + warn "Anxious to run \"rm -r ${HOME_DIR}\" automatically, folder does not exist, skipping..." + fi + else + warn "Anxious to delete \"${1}\" automatically, node not in configuration, skipping..." + fi else - GAIAD="$(get_gaiad_binary "$1")" - HOME_DIR="$(get_home_dir "$1")" - echo "Resetting $1..." - "$GAIAD" unsafe-reset-all --home "$HOME_DIR" + warn "Anxious to delete \"${*}\" automatically, looks like multiple paths, skipping..." fi } diff --git a/scripts/gm/gm.toml b/scripts/gm/gm.toml index 53f8c3e8f5..8de1711856 100644 --- a/scripts/gm/gm.toml +++ b/scripts/gm/gm.toml @@ -14,6 +14,9 @@ ports_start_at=27000 # The default folder where all node folders will be created. home_dir="$HOME/.gm" +# (Optional) The wallet address will be created from this 24-word seed phrase instead of a random address. +wallet_mnemonic="" + # Automatically update the `persistent_peers` and `unconditional_peer_ids` sections of the node configuration. auto_maintain_config=true From d15a90264aad96c1f12f3e7b554be18761f21d57 Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Thu, 20 May 2021 15:13:00 -0400 Subject: [PATCH 20/29] shell suport fix --- scripts/gm/bin/shell-support | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gm/bin/shell-support b/scripts/gm/bin/shell-support index 537d78667a..9d2ba82ddc 100644 --- a/scripts/gm/bin/shell-support +++ b/scripts/gm/bin/shell-support @@ -58,7 +58,7 @@ _gm_hermes_nodes() { _gm_complete_commands() { local cur="${COMP_WORDS[COMP_CWORD]}" - local cmds="help hermes install keys log ports start status stop unsafe-reset version" + local cmds="help hermes install keys log ports start status stop reset rm version" while read -r line; do COMPREPLY+=("$line"); done < <(compgen -W "$cmds" -- "$cur") } From 956c95f8f3f07689a5b919ec063ccbb23a2e3f9e Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Thu, 20 May 2021 15:26:45 -0400 Subject: [PATCH 21/29] better log command, goes to end normally --- scripts/gm/README.md | 4 ++-- scripts/gm/bin/gm | 28 ++++++++++++++++++++-------- scripts/gm/bin/shell-support | 6 +++--- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/scripts/gm/README.md b/scripts/gm/README.md index d463442a12..34f94560e1 100644 --- a/scripts/gm/README.md +++ b/scripts/gm/README.md @@ -192,10 +192,10 @@ If no node is specified then it lists all keys. Tip: it will show you the seed phrase if it can find it in the folder. -### `gm log [ ...] [-f]` +### `gm log [ ...] [-f|-r]` **Description**: Show the log of the specified node(s). At least one node has to be specified. -Tip: You can put `-f` anywhere after `log` to get `tail -f`-like functionality. +Tip: You can put `-f` and `-r` anywhere after `log` to get `tail -f` or `tail -r`-like functionality. ### `gm ports [ ...]` **Description**: List the ports assigned to a node. diff --git a/scripts/gm/bin/gm b/scripts/gm/bin/gm index f77bd42b85..89b7b66ed9 100755 --- a/scripts/gm/bin/gm +++ b/scripts/gm/bin/gm @@ -114,29 +114,41 @@ case "${1:-help}" in load_config shift F_FLAG="" + R_FLAG="" LIST="" while [ "$#" -gt 0 ]; do case "$1" in "-f") F_FLAG="-f" ;; + "-r") + R_FLAG="-r" ;; *) LIST="$LIST $(get_home_dir "$1")/log" esac shift done - if [ -z "$F_FLAG" ]; then - echo "less -R $LIST" + if [ -n "$F_FLAG" ] && [ -n "$R_FLAG" ]; then + exit_with_error "-f and -r is too much even for the tail command" + fi + if [ -z "$F_FLAG" ] && [ -z "$R_FLAG" ]; then + echo "less -R +G $LIST" # Todo: spaces in folder names and file names are not supported yet. # shellcheck disable=SC2086 - less -R $LIST + less -R +G $LIST else - echo "tail -f $LIST" - # Todo: spaces in folder names and file names are not supported yet. - # shellcheck disable=SC2086 - tail -f $LIST + if [ -n "$F_FLAG" ]; then + echo "tail -f $LIST" + # Todo: spaces in folder names and file names are not supported yet. + # shellcheck disable=SC2086 + tail -f $LIST + elif [ -n "$R_FLAG" ]; then + echo "tail -r $LIST" + # Todo: spaces in folder names and file names are not supported yet. + # shellcheck disable=SC2086 + tail -r $LIST + fi fi - ;; ports) load_config diff --git a/scripts/gm/bin/shell-support b/scripts/gm/bin/shell-support index 9d2ba82ddc..bb607bc787 100644 --- a/scripts/gm/bin/shell-support +++ b/scripts/gm/bin/shell-support @@ -44,9 +44,9 @@ _gm_nodes() { while read -r line; do COMPREPLY+=("$line"); done < <(compgen -W "$cmds" -- "$cur") } -_gm_nodes_f() { +_gm_nodes_log() { local cur="${COMP_WORDS[COMP_CWORD]}" - local cmds="$(__gm_get_nodes) -f" + local cmds="$(__gm_get_nodes) -f -r" while read -r line; do COMPREPLY+=("$line"); done < <(compgen -W "$cmds" -- "$cur") } @@ -100,7 +100,7 @@ _gm() { case "$cmd" in hermes) _gm_hermes_complete_commands ;; keys) _gm_nodes ;; - log) _gm_nodes_f ;; + log) _gm_nodes_log ;; ports) _gm_nodes ;; start) _gm_nodes ;; stop) _gm_nodes ;; From 9872c5cba33c8c05b1df0ccfe0503eb6d162bd4a Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Thu, 20 May 2021 15:28:02 -0400 Subject: [PATCH 22/29] log command fix --- scripts/gm/bin/gm | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/gm/bin/gm b/scripts/gm/bin/gm index 89b7b66ed9..7f2531f7de 100755 --- a/scripts/gm/bin/gm +++ b/scripts/gm/bin/gm @@ -128,6 +128,9 @@ case "${1:-help}" in esac shift done + if [ -z "$LIST" ]; then + exit_with_error "no node name was given" + fi if [ -n "$F_FLAG" ] && [ -n "$R_FLAG" ]; then exit_with_error "-f and -r is too much even for the tail command" fi From 827704c4db106afbb918c4ffd474ae29d2a65c86 Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Thu, 20 May 2021 20:05:39 -0400 Subject: [PATCH 23/29] wallet_hdpath added --- scripts/gm/README.md | 7 ++++++- scripts/gm/bin/lib-gm | 28 +++++++++++++++++++++++++--- scripts/gm/gm.toml | 3 +++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/scripts/gm/README.md b/scripts/gm/README.md index 34f94560e1..235ea6cbeb 100644 --- a/scripts/gm/README.md +++ b/scripts/gm/README.md @@ -96,7 +96,12 @@ the `gm.toml` file for node configuration. By default, newly created node config * `home_dir` * In the global section this defines the default folder where all node folders will be created. * In the per-node configuration this is the folder for the node configuration. -* `wallet_mnemonic` - define a 24-word seed phrase to use for wallet address(es) instead of creating random wallets. +* `wallet_mnemonic` + * Optional variable. + * define a 24-word seed phrase to use for wallet address(es) instead of creating random wallets. +* `wallet_hdpath` + * Optional variable. + * Change the HD derivation path for the wallet address. * `auto_maintain_config` - automatically update the `persistent_peers` and `unconditional_peer_ids` sections of the node configuration. * `add_to_hermes` - this node should be part of the hermes config. diff --git a/scripts/gm/bin/lib-gm b/scripts/gm/bin/lib-gm index 544167910e..6ddf6759da 100644 --- a/scripts/gm/bin/lib-gm +++ b/scripts/gm/bin/lib-gm @@ -17,6 +17,7 @@ config_defaults() { GLOBAL_AUTO_MAINTAIN_CONFIG="true" GLOBAL_ADD_TO_HERMES="false" GLOBAL_WALLET_MNEMONIC="" + GLOBAL_WALLET_HDPATH="" } install() { @@ -181,6 +182,7 @@ load_config() { GLOBAL_AUTO_MAINTAIN_CONFIG="$(stoml -q "$CONFIG_FILE" global.auto_maintain_config || echo "$GLOBAL_AUTO_MAINTAIN_CONFIG")" GLOBAL_ADD_TO_HERMES="$(stoml -q "$CONFIG_FILE" global.add_to_hermes || echo "$GLOBAL_ADD_TO_HERMES")" GLOBAL_WALLET_MNEMONIC="$(stoml -q "$CONFIG_FILE" global.wallet_mnemonic || echo "$GLOBAL_WALLET_MNEMONIC")" + GLOBAL_WALLET_HDPATH="$(stoml -q "$CONFIG_FILE" global.wallet_hdpath || echo "$GLOBAL_WALLET_HDPATH")" export GLOBAL_GAIAD_BINARY export GLOBAL_HERMES_BINARY export GLOBAL_HOME_DIR @@ -188,6 +190,7 @@ load_config() { export GLOBAL_AUTO_MAINTAIN_CONFIG export GLOBAL_ADD_TO_HERMES export GLOBAL_WALLET_MNEMONIC + export GLOBAL_WALLET_HDPATH if [ "$FRESH_CONFIG" = "yes" ]; then write_default_config VALIDATORS="" @@ -261,6 +264,7 @@ add_to_hermes=$GLOBAL_ADD_TO_HERMES hermes_binary="$GLOBAL_HERMES_BINARY" hermes_config="$GLOBAL_HERMES_CONFIG" wallet_mnemonic="$GLOBAL_WALLET_MNEMONIC" +wallet_hdpath="$GLOBAL_WALLET_HDPATH" EOF set -e } @@ -376,6 +380,15 @@ get_wallet_mnemonic() { fi } +get_wallet_hdpath() { + RESULT="$(stoml -q "$CONFIG_FILE" "${1}.wallet_hdpath")" + if [ -z "$RESULT" ]; then + echo "$GLOBAL_WALLET_HDPATH" + else + echo "$RESULT" + fi +} + create() { i="$1" echo "Creating $i config..." @@ -391,11 +404,20 @@ create() { mv "${HOME_ROOT}/${i}v0" "${HOME_DIR}" mv "${HOME_ROOT}/gentxs" "${HOME_DIR}" WALLET_MNEMONIC="$(get_wallet_mnemonic "$i")" + WALLET_HDPATH="$(get_wallet_hdpath "$i")" if [ -z "$WALLET_MNEMONIC" ]; then - "$GAIAD_BINARY" keys add "wallet" --keyring-backend test --keyring-dir "${HOME_DIR}" --output json > "${HOME_DIR}/wallet_seed.json" + if [ -z "$WALLET_HDPATH" ]; then + "$GAIAD_BINARY" keys add "wallet" --keyring-backend test --keyring-dir "${HOME_DIR}" --output json > "${HOME_DIR}/wallet_seed.json" + else + "$GAIAD_BINARY" keys add "wallet" --hd-path "$WALLET_HDPATH" --keyring-backend test --keyring-dir "${HOME_DIR}" --output json > "${HOME_DIR}/wallet_seed.json" + fi else - echo "$WALLET_MNEMONIC" | "$GAIAD_BINARY" keys add "wallet" --recover --keyring-backend test --keyring-dir "${HOME_DIR}" --output json > "${HOME_DIR}/wallet_seed.json" - sconfig "${HOME_DIR}/wallet_seed.json" -t string "mnemonic=${WALLET_MNEMONIC}" + if [ -z "$WALLET_HDPATH" ]; then + echo "$WALLET_MNEMONIC" | "$GAIAD_BINARY" keys add "wallet" --recover --keyring-backend test --keyring-dir "${HOME_DIR}" --output json > "${HOME_DIR}/wallet_seed.json" + else + echo "$WALLET_MNEMONIC" | "$GAIAD_BINARY" keys add "wallet" --hd-path "$WALLET_HDPATH" --recover --keyring-backend test --keyring-dir "${HOME_DIR}" --output json > "${HOME_DIR}/wallet_seed.json" + fi + sconfig "${HOME_DIR}/wallet_seed.json" -t string "mnemonic=${WALLET_MNEMONIC}" 1> /dev/null fi "$GAIAD_BINARY" add-genesis-account "wallet" "10000000stake,10000localcoin" --keyring-backend test --home "${HOME_DIR}" fi diff --git a/scripts/gm/gm.toml b/scripts/gm/gm.toml index 8de1711856..85d2cda9c7 100644 --- a/scripts/gm/gm.toml +++ b/scripts/gm/gm.toml @@ -17,6 +17,9 @@ home_dir="$HOME/.gm" # (Optional) The wallet address will be created from this 24-word seed phrase instead of a random address. wallet_mnemonic="" +# (Optional) The HD derivation path for the wallet (default is "m/44'/118'/0'/0/0" for ATOM.) +wallet_hdpath="" + # Automatically update the `persistent_peers` and `unconditional_peer_ids` sections of the node configuration. auto_maintain_config=true From 6c2101a5cf766839e2c78a3ba7a20b52e8bd7597 Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Thu, 20 May 2021 20:40:16 -0400 Subject: [PATCH 24/29] denom issue workaround --- scripts/gm/README.md | 2 +- scripts/gm/bin/lib-gm | 23 +++++++++++++---------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/scripts/gm/README.md b/scripts/gm/README.md index 235ea6cbeb..0caa466171 100644 --- a/scripts/gm/README.md +++ b/scripts/gm/README.md @@ -25,7 +25,7 @@ configuration updates. * Bourne shell (`sh`) * [`sconfig`](https://github.com/freshautomations/sconfig/releases) and [`stoml`](https://github.com/freshautomations/stoml/releases) installed in your PATH (put them in `/usr/local/bin`) -* `sed` (trying to remove this dependency in the future) +* `sed`, `tr` (trying to remove these in the future) * For shell-completion Bourne Again Shell (`bash`) for the local user (`zsh` shell-completion is coming) ## How to run diff --git a/scripts/gm/bin/lib-gm b/scripts/gm/bin/lib-gm index 6ddf6759da..ba559e3087 100644 --- a/scripts/gm/bin/lib-gm +++ b/scripts/gm/bin/lib-gm @@ -1,6 +1,6 @@ #!/usr/bin/env sh set -eu -if [ -n "${DEBUG:-}" ]; then +if [ "${DEBUG:-}" == "2" ]; then set -x fi @@ -396,12 +396,13 @@ create() { HOME_DIR="$(get_home_dir "$i")" HOME_ROOT="${HOME_DIR%%${i}}" if a_in_b "$i" "$VALIDATORS"; then - EXEC_RESULT="$("$GAIAD_BINARY" testnet --chain-id "$i" --keyring-backend test --node-dir-prefix "${i}v" -o "$HOME_ROOT" --node-daemon-home . --v 1 2>&1)" + NODE_DIR_PREFIX="$(echo "${i}v" | tr -d '-')" + EXEC_RESULT="$("$GAIAD_BINARY" testnet --chain-id "$i" --keyring-backend test --node-dir-prefix "$NODE_DIR_PREFIX" -o "$HOME_ROOT" --node-daemon-home . --v 1 2>&1)" if [ "$EXEC_RESULT" != "Successfully initialized 1 node directories" ]; then warn "could not create config for ${i}: \"$EXEC_RESULT\", skipping..." return 1 else - mv "${HOME_ROOT}/${i}v0" "${HOME_DIR}" + mv "${HOME_ROOT}/${NODE_DIR_PREFIX}0" "${HOME_DIR}" mv "${HOME_ROOT}/gentxs" "${HOME_DIR}" WALLET_MNEMONIC="$(get_wallet_mnemonic "$i")" WALLET_HDPATH="$(get_wallet_hdpath "$i")" @@ -423,12 +424,13 @@ create() { fi else NETWORK="$(get_network "$i")" - EXEC_RESULT="$("$GAIAD_BINARY" testnet --chain-id "$NETWORK" --keyring-backend test --node-dir-prefix "${i}n" -o "$HOME_ROOT" --node-daemon-home . --v 1 2>&1)" + NODE_DIR_PREFIX="$(echo "${i}n" | tr -d '-')" + EXEC_RESULT="$("$GAIAD_BINARY" testnet --chain-id "$NETWORK" --keyring-backend test --node-dir-prefix "$NODE_DIR_PREFIX" -o "$HOME_ROOT" --node-daemon-home . --v 1 2>&1)" if [ "$EXEC_RESULT" != "Successfully initialized 1 node directories" ]; then warn "could not create config for ${i}: \"$EXEC_RESULT\", skipping..." return 1 fi - mv "${HOME_ROOT}/${i}n0" "${HOME_DIR}" + mv "${HOME_ROOT}/${NODE_DIR_PREFIX}0" "${HOME_DIR}" rm -rf "${HOME_ROOT}/gentxs" fi configure "$i" @@ -461,6 +463,10 @@ configure() { if ! a_in_b "$i" "$VALIDATORS"; then NETWORK="$(get_network "$1")" NETWORK_HOME_DIR="$(get_home_dir "$NETWORK")" + if [ ! -f "$NETWORK_HOME_DIR/config/genesis.json" ]; then + warn "${NETWORK} does not have a genesis.json. Start ${NETWORK} first." + return 1 + fi cp "$NETWORK_HOME_DIR/config/genesis.json" "$HOME_DIR/config/genesis.json" NETWORK_NODE="$(get_node_id "$NETWORK")@localhost:$(get_p2p_port "$NETWORK")" sconfig "$HOME_DIR/config/config.toml" "p2p.persistent_peers=$NETWORK_NODE" 1> /dev/null @@ -637,15 +643,12 @@ list_keys() { echo "$line" fi elif [ "$MNEMONIC" != "$line" ]; then - if a_in_b "${KEY_NAME%%v0}" "$VALIDATORS"; then + if a_in_b "${KEY_NAME%%v0}" "$VALIDATORS" || a_in_b "${KEY_NAME%%v0}" "$(echo "$VALIDATORS" | tr -d '-')"; then echo "mnemonic: \"$(stoml "${HOME_DIR}/key_seed.json" secret)\"" - elif a_in_b "${KEY_NAME%%n0}" "$FULL_NODES"; then + elif a_in_b "${KEY_NAME%%n0}" "$FULL_NODES" || a_in_b "${KEY_NAME%%n0}" "$(echo "$FULL_NODES" | tr -d '-')"; then echo "mnemonic: \"$(stoml "${HOME_DIR}/key_seed.json" secret)\"" elif [ -f "${HOME_DIR}/${KEY_NAME}_seed.json" ]; then echo "mnemonic: \"$(stoml "${HOME_DIR}/${KEY_NAME}_seed.json" mnemonic)\"" - # Compatibility with the full-mesh tool - elif a_in_b "${KEY_NAME%%c0}" "$VALIDATORS" && [ -f "${HOME_DIR}/client_wallet_seed.json" ]; then - echo "mnemonic: \"$(stoml "${HOME_DIR}/client_wallet_seed.json" mnemonic)\"" else echo "mnemonic: \"\"" fi From 62c3475fd276fcf92ff2001ca6153504b69cca39 Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Thu, 20 May 2021 20:53:05 -0400 Subject: [PATCH 25/29] comments about tr --- scripts/gm/bin/lib-gm | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/gm/bin/lib-gm b/scripts/gm/bin/lib-gm index ba559e3087..cf6eb089c7 100644 --- a/scripts/gm/bin/lib-gm +++ b/scripts/gm/bin/lib-gm @@ -396,6 +396,7 @@ create() { HOME_DIR="$(get_home_dir "$i")" HOME_ROOT="${HOME_DIR%%${i}}" if a_in_b "$i" "$VALIDATORS"; then + # Removing '-' to work around https://github.com/cosmos/cosmos-sdk/issues/9373 NODE_DIR_PREFIX="$(echo "${i}v" | tr -d '-')" EXEC_RESULT="$("$GAIAD_BINARY" testnet --chain-id "$i" --keyring-backend test --node-dir-prefix "$NODE_DIR_PREFIX" -o "$HOME_ROOT" --node-daemon-home . --v 1 2>&1)" if [ "$EXEC_RESULT" != "Successfully initialized 1 node directories" ]; then @@ -424,6 +425,7 @@ create() { fi else NETWORK="$(get_network "$i")" + # Removing '-' to work around https://github.com/cosmos/cosmos-sdk/issues/9373 NODE_DIR_PREFIX="$(echo "${i}n" | tr -d '-')" EXEC_RESULT="$("$GAIAD_BINARY" testnet --chain-id "$NETWORK" --keyring-backend test --node-dir-prefix "$NODE_DIR_PREFIX" -o "$HOME_ROOT" --node-daemon-home . --v 1 2>&1)" if [ "$EXEC_RESULT" != "Successfully initialized 1 node directories" ]; then @@ -643,8 +645,10 @@ list_keys() { echo "$line" fi elif [ "$MNEMONIC" != "$line" ]; then + # Removing '-' to work around https://github.com/cosmos/cosmos-sdk/issues/9373 if a_in_b "${KEY_NAME%%v0}" "$VALIDATORS" || a_in_b "${KEY_NAME%%v0}" "$(echo "$VALIDATORS" | tr -d '-')"; then echo "mnemonic: \"$(stoml "${HOME_DIR}/key_seed.json" secret)\"" + # Removing '-' to work around https://github.com/cosmos/cosmos-sdk/issues/9373 elif a_in_b "${KEY_NAME%%n0}" "$FULL_NODES" || a_in_b "${KEY_NAME%%n0}" "$(echo "$FULL_NODES" | tr -d '-')"; then echo "mnemonic: \"$(stoml "${HOME_DIR}/key_seed.json" secret)\"" elif [ -f "${HOME_DIR}/${KEY_NAME}_seed.json" ]; then From 777ab75df5dc6f80cf9a89f864d5608c9d68f9ae Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Thu, 20 May 2021 21:15:49 -0400 Subject: [PATCH 26/29] remove supply from genesis as a workaround --- scripts/gm/bin/lib-gm | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/gm/bin/lib-gm b/scripts/gm/bin/lib-gm index cf6eb089c7..94060e81d3 100644 --- a/scripts/gm/bin/lib-gm +++ b/scripts/gm/bin/lib-gm @@ -57,6 +57,9 @@ enforce_requirements() { if [ -z "$(which sed)" ]; then exit_with_error "missing sed, please install it (this requirement will be removed in the future)" fi + if [ -z "$(which tr)" ]; then + exit_with_error "missing tr, please install it (this requirement will be removed in the future)" + fi } debug() { @@ -156,6 +159,7 @@ is_configuration_valid() { return 0 } +# Function to work around https://github.com/spf13/viper/issues/1131 load_all_sections() { grep '^ *\[.\+\] *$' "$CONFIG_FILE" | sed 's/^ *\[\([^]]*\)\] *$/\1/' } @@ -422,6 +426,15 @@ create() { sconfig "${HOME_DIR}/wallet_seed.json" -t string "mnemonic=${WALLET_MNEMONIC}" 1> /dev/null fi "$GAIAD_BINARY" add-genesis-account "wallet" "10000000stake,10000localcoin" --keyring-backend test --home "${HOME_DIR}" + # Workaround for https://github.com/cosmos/cosmos-sdk/issues/9372 : remove supply from genesis + sconfig "${HOME_DIR}/config/genesis.json" -t stringSlice app_state.bank.supply=remove_supply 1> /dev/null + OS="$(uname -s)" + if [ "$OS" = "Darwin" ]; then + sed -i '' 's/\"remove_supply\"//' "$HOME_DIR/config/genesis.json" + else + sed -i'' 's/\"remove_supply\"//' "$HOME_DIR/config/genesis.json" + fi + # End of workaround fi else NETWORK="$(get_network "$i")" From d1a3c2534ed738ac7c0bb0dce5ae65f5939684d1 Mon Sep 17 00:00:00 2001 From: Greg Szabo Date: Thu, 20 May 2021 21:30:23 -0400 Subject: [PATCH 27/29] Added GRPC-Web port support --- scripts/gm/README.md | 18 +++++++++++------- scripts/gm/bin/lib-gm | 8 ++++++++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/scripts/gm/README.md b/scripts/gm/README.md index 0caa466171..9a4e334433 100644 --- a/scripts/gm/README.md +++ b/scripts/gm/README.md @@ -147,13 +147,14 @@ This setup allows developers to run a node outside of `gm` just by pointing the Ports are defined by the `ports_start_at` parameter which will be the first port assigned. Port assignment is as follows: ``` -| name | port redirection | -|==============|====================| -| RPC (26657) | ports_start_at + 0 | -| App (1317) | ports_start_at + 1 | -| GRPC (9090 | ports_start_at + 2 | -| P2P (26656) | ports_start_at + 3 | -| PPROF (6060) | ports_start_at + 4 | +| name | port redirection | +|=================|====================| +| RPC (26657) | ports_start_at + 0 | +| App (1317) | ports_start_at + 1 | +| GRPC (9090 | ports_start_at + 2 | +| P2P (26656) | ports_start_at + 3 | +| PPROF (6060) | ports_start_at + 4 | +| GRPC-WEB (9091) | ports_start_at + 5 | ``` Example output of `gm ports` command when `node4.ports_start_at=27050`: @@ -163,8 +164,11 @@ node4 APP : http://localhost:27051 node4 GRPC : http://localhost:27052 node4 P2P : http://localhost:27053 node4 PPROF: http://localhost:27054 +node4 GRPCW: http://localhost:27055 ``` +Note: The GRPC-Web port was recently introduced (after gaiad v4.2.1). It will be ignored in earlier versions. + ## Execution manual ### `gm help` **Description**: shows the help screen diff --git a/scripts/gm/bin/lib-gm b/scripts/gm/bin/lib-gm index 94060e81d3..b6646d5c20 100644 --- a/scripts/gm/bin/lib-gm +++ b/scripts/gm/bin/lib-gm @@ -314,6 +314,10 @@ get_pprof_port() { echo "$(($(get_ports_start_at "$1")+4))" } +get_grpcw_port() { + echo "$(($(get_ports_start_at "$1")+5))" +} + get_home_dir() { RESULT="$(stoml -q "$CONFIG_FILE" "${1}.home_dir")" if [ -z "$RESULT" ]; then @@ -460,6 +464,7 @@ configure() { APP="$(get_app_port "$1")" GRPC="$(get_grpc_port "$1")" PPROF="$(get_pprof_port "$1")" + GRPCW="$(get_grpcw_port "$1")" sconfig "$HOME_DIR/config/config.toml" "p2p.laddr=tcp://0.0.0.0:${P2P}" 1> /dev/null sconfig "$HOME_DIR/config/config.toml" "rpc.laddr=tcp://0.0.0.0:${RPC}" 1> /dev/null sconfig "$HOME_DIR/config/config.toml" "rpc.pprof_laddr=0.0.0.0:${PPROF}" 1> /dev/null @@ -475,6 +480,7 @@ configure() { # End of workaround sconfig "$HOME_DIR/config/app.toml" "api.address=tcp://0.0.0.0:${APP}" 1> /dev/null sconfig "$HOME_DIR/config/app.toml" "grpc.address=0.0.0.0:${GRPC}" 1> /dev/null + sconfig "$HOME_DIR/config/app.toml" "grpc-web.address=0.0.0.0:${GRPCW}" 1> /dev/null if ! a_in_b "$i" "$VALIDATORS"; then NETWORK="$(get_network "$1")" NETWORK_HOME_DIR="$(get_home_dir "$NETWORK")" @@ -625,11 +631,13 @@ ports() { APP="$(get_app_port "$1")" GRPC="$(get_grpc_port "$1")" PPROF="$(get_pprof_port "$1")" + GRPCW="$(get_grpcw_port "$1")" echo "${1} RPC : http://localhost:${RPC}" echo "${1} APP : http://localhost:${APP}" echo "${1} GRPC : http://localhost:${GRPC}" echo "${1} P2P : http://localhost:${P2P}" echo "${1} PPROF: http://localhost:${PPROF}" + echo "${1} GRPCW: http://localhost:${GRPCW}" } list_keys() { From 2ccf8aed843671ff3d9bd977b8365793e3f05f2b Mon Sep 17 00:00:00 2001 From: Greg Szabo <16846635+greg-szabo@users.noreply.github.com> Date: Fri, 21 May 2021 11:29:07 -0400 Subject: [PATCH 28/29] Update scripts/gm/bin/gm Co-authored-by: Adi Seredinschi --- scripts/gm/bin/gm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gm/bin/gm b/scripts/gm/bin/gm index 7f2531f7de..6ca07cd6db 100755 --- a/scripts/gm/bin/gm +++ b/scripts/gm/bin/gm @@ -49,7 +49,7 @@ hermes_usage() { echo "SUBCOMMANDS DESCRIPTION" echo echo "help print this help and exit" - echo "config print the hermes config.toml" + echo "config update hermes config.toml with the current gm network details" echo "keys add keys to $HOME/.hermes" echo "cc print a list of hermes commands to create IBC connections" echo From ecc07f62312013659f601561cf872c9ada07e8a6 Mon Sep 17 00:00:00 2001 From: Greg Szabo <16846635+greg-szabo@users.noreply.github.com> Date: Fri, 21 May 2021 11:50:06 -0400 Subject: [PATCH 29/29] Update scripts/gm/bin/lib-gm --- scripts/gm/bin/lib-gm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gm/bin/lib-gm b/scripts/gm/bin/lib-gm index b6646d5c20..3da978d930 100644 --- a/scripts/gm/bin/lib-gm +++ b/scripts/gm/bin/lib-gm @@ -429,7 +429,7 @@ create() { fi sconfig "${HOME_DIR}/wallet_seed.json" -t string "mnemonic=${WALLET_MNEMONIC}" 1> /dev/null fi - "$GAIAD_BINARY" add-genesis-account "wallet" "10000000stake,10000localcoin" --keyring-backend test --home "${HOME_DIR}" + "$GAIAD_BINARY" add-genesis-account "wallet" "10000000stake,100000000samoleans" --keyring-backend test --home "${HOME_DIR}" # Workaround for https://github.com/cosmos/cosmos-sdk/issues/9372 : remove supply from genesis sconfig "${HOME_DIR}/config/genesis.json" -t stringSlice app_state.bank.supply=remove_supply 1> /dev/null OS="$(uname -s)"