Skip to content

Commit

Permalink
fix(installer): Change snapshot URL calculating logic (#374)
Browse files Browse the repository at this point in the history 
* fix: Change snapshot URL calculating logic
* Change archive format to lz4
* Fixed default paths in state sync file
* Change the directory for untarring

Signed-off-by: Andrew Nikitin <andrew.nikitin@evernym.com>

Signed-off-by: Andrew Nikitin <andrew.nikitin@evernym.com>
Co-authored-by: Ankur Banerjee <ankurdotb@users.noreply.github.com>
  • Loading branch information
@ankurdotb
Andrew Nikitin and ankurdotb committed Aug 19, 2022
1 parent 48d1512 commit c279317
Show file tree
Hide file tree
Showing 2 changed files with 36 additions and 28 deletions.
48 changes: 28 additions & 20 deletions installer/installer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,11 +62,11 @@
###############################################################
### Node snapshots ###
###############################################################
DEFAULT_SNAPSHOT_SERVER = "https://snapshots.cheqd.net"
DEFAULT_SNAPSHOT_SERVER = "https://snapshots-cdn.cheqd.net"
DEFAULT_INIT_FROM_SNAPSHOT = "yes"
TESTNET_SNAPSHOT = "https://cheqd-node-backups.ams3.cdn.digitaloceanspaces.com/testnet/latest/cheqd-testnet-4_{}.tar.gz"
MAINNET_SNAPSHOT = "https://cheqd-node-backups.ams3.cdn.digitaloceanspaces.com/mainnet/latest/cheqd-mainnet-1_{}.tar.gz"
CHECKSUM_URL_BASE = "https://cheqd-node-backups.ams3.cdn.digitaloceanspaces.com/"
TESTNET_SNAPSHOT = "https://snapshots-cdn.cheqd.net/testnet/{}/cheqd-testnet-4_{}.tar.lz4"
MAINNET_SNAPSHOT = "https://snapshots-cdn.cheqd.net/mainnet/{}/cheqd-mainnet-1_{}.tar.lz4"
MAX_SNAPSHOT_DAYS = 7

###############################################################
### Default node configuration ###
Expand Down Expand Up @@ -244,8 +244,8 @@ def get_binary(self):
fname = os.path.basename(binary_url)
try:
self.exec(f"wget -c {binary_url}")
if fname.find(".tar.gz") != -1:
self.exec(f"tar -xzf {fname}")
if fname.find(".tar.lz4") != -1:
self.exec(f"tar -I -xf {fname}")
self.remove_safe(fname)
self.exec(f"chmod +x {DEFAULT_BINARY_NAME}")
except:
Expand Down Expand Up @@ -511,7 +511,7 @@ def setup_cosmovisor(self):

def compare_checksum(self, file_path):
# Set URL for correct checksum file for snapshot
checksum_url = os.path.join(CHECKSUM_URL_BASE, self.interviewer.chain, "latest/md5sum.txt")
checksum_url = os.path.join(os.path.dirname(self.interviewer.snapshot_url), "md5sum.txt")
# Get checksum file
published_checksum = self.exec(f"curl -s {checksum_url} | tail -1 | cut -d' ' -f 1").stdout.strip()
self.log(f"Comparing published checksum with local checksum")
Expand Down Expand Up @@ -540,7 +540,7 @@ def download_snapshot(self):
self.mkdir_p(self.cheqd_data_dir)
self.exec(f"chown -R {DEFAULT_CHEQD_USER}:{DEFAULT_CHEQD_USER} {self.cheqd_data_dir}")
# Fetch size of snapshot archive. Uses curl to fetch headers and looks for Content-Length.
archive_size = self.exec(f"curl -s --head {self.interviewer.snapshot_url} | awk '/Length/ {{print $2}}'").stdout.strip()
archive_size = self.exec(f"curl -s --head {self.interviewer.snapshot_url} | awk '/content-length/ {{print $2}}'").stdout.strip()
# Check how much free disk space is available wherever the cheqd root directory is mounted
free_disk_space = self.exec(f"df -P -B1 {self.cheqd_root_dir} | tail -1 | awk '{{print $4}}'").stdout.strip()
if int(archive_size) < int(free_disk_space):
Expand All @@ -567,7 +567,7 @@ def untar_from_snapshot(self):
self.log(f"Extracting snapshot archive. This may take a while...")

# Extract to cheqd node data directory EXCEPT for validator state
self.exec(f"sudo su -c 'pv {archive_path} | tar xzf - -C {self.cheqd_data_dir} --exclude priv_validator_state.json' {DEFAULT_CHEQD_USER}")
self.exec(f"sudo su -c 'pv {archive_path} | tar --use-compress-program=lz4 -xf - -C {self.cheqd_root_dir} --exclude priv_validator_state.json' {DEFAULT_CHEQD_USER}")

# Delete snapshot archive file
self.log(f"Snapshot extraction was successful. Deleting snapshot archive.")
Expand Down Expand Up @@ -596,7 +596,7 @@ def __init__(self,
self._release = None
self._chain = chain
self.verbose = True
self._snapshot_url = self.prepare_url_for_latest()
self._snapshot_url = ""
self._is_setup_needed = False
self._moniker = ""
self._external_address = ""
Expand Down Expand Up @@ -774,11 +774,11 @@ def is_systemd_config_exists(self) -> bool:
os.path.exists(DEFAULT_STANDALONE_SERVICE_FILE_PATH)

@post_process
def exec(self, cmd, use_stdout=True, suppress_err=False):
def exec(self, cmd, use_stdout=True, suppress_err=False, check=True):
self.log(f"Executing command: {cmd}")
kwargs = {
"shell": True,
"check": True,
"check": check,
}
if use_stdout:
kwargs["stdout"] = subprocess.PIPE
Expand Down Expand Up @@ -959,18 +959,26 @@ def ask_for_gas_price(self):
def prepare_url_for_latest(self) -> str:
template = TESTNET_SNAPSHOT if self.chain == "testnet" else MAINNET_SNAPSHOT
_date = datetime.date.today()
_url = template.format(_date.strftime("%Y-%m-%d"))
while not self.is_url_exists(_url):
_days_counter = 0
_is_url_valid = False

while not _is_url_valid and _days_counter <= MAX_SNAPSHOT_DAYS:
_url = template.format(_date.strftime("%Y-%m-%d"), _date.strftime("%Y-%m-%d"))
_is_url_valid = self.is_url_exists(_url)
_days_counter += 1
_date -= datetime.timedelta(days=1)
_url = template.format(_date.strftime("%Y-%m-%d"))

if not _is_url_valid:
failure_exit("Could not find the valid snapshot for the last {} days".format(MAX_SNAPSHOT_DAYS))
return _url

def is_url_exists(self, url):
try:
request.urlopen(request.Request(url))
except urllib.error.HTTPError:
return False
return True
curr_verbose = self.verbose
self.verbose = False
if self.exec("curl --output /dev/null --silent --head --fail {url}".format(url=url), check=False).returncode == 0:
return True
self.verbose = curr_verbose
return False

if __name__ == '__main__':

Expand Down
16 changes: 8 additions & 8 deletions scripts/state-sync.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,16 @@ fi
# set environment variables
export GOPATH=~/go
export PATH=$PATH:~/go/bin
export RPC=https://rpc.cheqd.net:443
export RPCN=https://rpc.cheqd.net:443
export RPC=https://eu-rpc.cheqd.net:443
export RPCN=https://ap-rpc.cheqd.net:443
export APPNAME=CHEQD_NODED

# Install Gaia
go install -tags rocksdb ./...
go install -tags goleveldb ./...

# MAKE HOME FOLDER AND GET GENESIS
cheqd-noded init "$NODE_MONIKER" --home /others/cheqd
wget -O /others/cheqd/config/genesis.json https://github.com/cheqd/cheqd-node/raw/main/networks/mainnet/genesis.json
cheqd-noded init "$NODE_MONIKER" --home /home/cheqd
wget -O /home/cheqd/.cheqdnode/config/genesis.json https://github.com/cheqd/cheqd-node/raw/main/networks/mainnet/genesis.json
wget -O seeds.txt https://github.com/cheqd/cheqd-node/raw/main/networks/mainnet/seeds.txt

INTERVAL=1000
Expand All @@ -48,15 +48,15 @@ export ${APPNAME}_STATESYNC_TRUST_HASH="$TRUST_HASH"
export ${APPNAME}_P2P_SEEDS="$(cat seeds.txt)"


cheqd-noded start --x-crisis-skip-assert-invariants --home /others/cheqd --grpc-web.address 127.0.0.1:5050
cheqd-noded start --x-crisis-skip-assert-invariants --home /home/cheqd/.cheqdnode --grpc-web.address 127.0.0.1:9091


# THIS WILL FIX THE APP VERSION, contributed by callum and claimens
git clone https://github.com/tendermint/tendermint
cd tendermint
git checkout remotes/origin/callum/app-version
go install ./...
tendermint set-app-version 1 --home ~/others/cheqd
tendermint set-app-version 1 --home ~/home/cheqd/.cheqdnode


cheqd-noded start --x-crisis-skip-assert-invariants --home /others/cheqd --grpc-web.address 127.0.0.1:5050
cheqd-noded start --x-crisis-skip-assert-invariants --home /home/cheqd/.cheqdnode --grpc-web.address 127.0.0.1:9091

0 comments on commit c279317

Please sign in to comment.