diff --git a/scripts/cnode-helper-scripts/cncli.sh b/scripts/cnode-helper-scripts/cncli.sh index 25e6fb1b3..92b303b41 100755 --- a/scripts/cnode-helper-scripts/cncli.sh +++ b/scripts/cnode-helper-scripts/cncli.sh @@ -44,10 +44,13 @@ usage() { force Manually force leaderlog calculation and overwrite even if already done, exits after leaderlog is calculated validate Continously monitor and confirm that the blocks made actually was accepted and adopted by chain (deployed as service) all One-time re-validation of all blocks in blocklog db - epoch One-time re-validation of blocks in blocklog db for the specified epoch + epoch One-time re-validation of blocks in blocklog db for the specified epoch + epochdata Manually re-calculate leaderlog to load stakepool history into epochdata table of blocklog db. Needs completion of cncli.sh sync and cncli.sh validate processes. + all One-time re-calculation of all epochs (avg execution duration: 1hr / 50 epochs) + epoch One-time re-calculation for the specified epoch ptsendtip Send node tip to PoolTool for network analysis and to show that your node is alive and well with a green badge (deployed as service) ptsendslots Securely sends PoolTool the number of slots you have assigned for an epoch and validates the correctness of your past epochs (deployed as service) - force Manually force pooltool sendslots submission ignoring configured time window + force Manually force pooltool sendslots submission ignoring configured time window init One-time initialization adding all minted and confirmed blocks to blocklog metrics Print cncli block metrics in Prometheus format deploy Install dependencies and deploy cncli monitoring agent service (available through port specified by CNCLI_PROM_PORT) @@ -126,11 +129,15 @@ getLedgerData() { # getNodeMetrics expected to have been already run } getConsensus() { - getProtocolParams + if isNumber "$1"; then + getProtocolParamsHist "$(( $1 - 1 ))" || return 1 + else + getProtocolParams || return 1 + fi if versionCheck "9.0" "${PROT_VERSION}"; then consensus="cpraos" stability_window_factor=3 - elif versionCheck "8.0" "${PROT_VERSION}"; then + elif versionCheck "7.0" "${PROT_VERSION}"; then consensus="praos" stability_window_factor=2 else @@ -794,10 +801,184 @@ cncliPTsendslots() { done } +################################# +# epochdata table load process # +################################# + +getCurrNextEpoch() { + getNodeMetrics + curr_epoch=${epochnum} + next_epoch=$((curr_epoch+1)) +} + +runCurrentEpoch() { + getKoiosData + echo "Processing current epoch: ${1}" + stake_param_curr="--active-stake ${active_stake_set} --pool-stake ${pool_stake_set}" + + ${CNCLI} leaderlog ${cncliParams} --consensus "${consensus}" --epoch="${1}" ${stake_param_curr} | + jq -r '[.epoch, .epochNonce, .poolId, .sigma, .d, .epochSlotsIdeal, .maxPerformance, .activeStake, .totalActiveStake] | @csv' | + tr -d '"' >> "$tmpcsv" +} + +runNextEpoch() { + getKoiosData + getNodeMetrics + getConsensus + slot_for_next_nonce=$(echo "(${slotnum} - ${slot_in_epoch} + ${EPOCH_LENGTH}) - (${stability_window_factor} * ${BYRON_K} / ${ACTIVE_SLOTS_COEFF})" | bc) + curr_epoch=${epochnum} + next_epoch=$((curr_epoch+1)) + + if [[ ${slotnum} -gt ${slot_for_next_nonce} ]]; then + if [[ $(sqlite3 "${BLOCKLOG_DB}" "SELECT COUNT(*) FROM epochdata WHERE epoch=${next_epoch};" 2>/dev/null) -eq 1 ]]; then + echo "Leaderlogs already calculated for epoch ${next_epoch}, skipping!" && return 1 + else + echo "Processing next epoch: ${1}" + stake_param_next="--active-stake ${active_stake_mark} --pool-stake ${pool_stake_mark}" + ${CNCLI} leaderlog ${cncliParams} --consensus "${consensus}" --epoch="${1}" ${stake_param_next} | + jq -r '[.epoch, .epochNonce, .poolId, .sigma, .d, .epochSlotsIdeal, .maxPerformance, .activeStake, .totalActiveStake] | @csv' | + tr -d '"' >> "$tmpcsv" + fi + fi +} + +runPreviousEpochs() { + [[ -z ${KOIOS_API} ]] && return 1 + if ! pool_hist=$(curl -sSL -f "${KOIOS_API}/pool_history?_pool_bech32=${POOL_ID_BECH32}&_epoch_no=${1}" 2>&1); then + echo "ERROR: Koios pool_stake_snapshot history query failed." + return 1 + fi + + if ! epoch_hist=$(curl -sSL -f "${KOIOS_API}/epoch_info?_epoch_no=${1}" 2>&1); then + echo "ERROR: Koios epoch_stake_snapshot history query failed." + return 1 + fi + + pool_stake_hist=$(jq -r '.[].active_stake' <<< "${pool_hist}") + active_stake_hist=$(jq -r '.[].active_stake' <<< "${epoch_hist}") + + echo "Processing previous epoch: ${1}" + stake_param_prev="--active-stake ${active_stake_hist} --pool-stake ${pool_stake_hist}" + + ${CNCLI} leaderlog ${cncliParams} --consensus "${consensus}" --epoch="${1}" ${stake_param_prev} | + jq -r '[.epoch, .epochNonce, .poolId, .sigma, .d, .epochSlotsIdeal, .maxPerformance, .activeStake, .totalActiveStake] | @csv' | + tr -d '"' >> "$tmpcsv" + + return 0 +} + +processAllEpochs() { + getCurrNextEpoch + IFS=' ' read -r -a epochs_array <<< "$EPOCHS" + + for epoch in "${epochs_array[@]}"; do + if ! getConsensus "${epoch}"; then echo "ERROR: Failed to fetch protocol parameters for epoch ${epoch}."; return 1; fi + if [[ "$epoch" == "$curr_epoch" ]]; then + runCurrentEpoch ${epoch} + elif [[ "$epoch" == "$next_epoch" ]]; then + runNextEpoch ${epoch} + else + runPreviousEpochs ${epoch} + fi + done + + id=1 + while IFS= read -r row; do + echo "$id,$row" >> "$csvfile" + ((id++)) + done < "$tmpcsv" + + sqlite3 "$BLOCKLOG_DB" < "$onerow_csv" + + sqlite3 "$BLOCKLOG_DB" "DELETE FROM epochdata WHERE epoch = ${1};" + sqlite3 "$BLOCKLOG_DB" < "$tmpcsv" ; true > "$csvfile" ; true > "$onerow_csv" + + proc_msg="~ CNCLI epochdata table load started ~" + + if ! cncliDBinSync; then + echo ${proc_msg} + echo "CNCLI DB out of sync :( [$(printf "%2.4f %%" ${cncli_sync_prog})] ... check cncli sync service!" + exit 1 + else + echo ${proc_msg} + getLedgerData + + if [[ "${subcommand}" == "epochdata" ]]; then + if [[ ${subarg} == "all" ]]; then + processAllEpochs + elif isNumber "${subarg}"; then + processSingleEpoch "${subarg}" + else + echo + echo "ERROR: unknown argument passed to validate command, valid options incl the string 'all' or the epoch number to recalculate" + echo + exit 1 + fi + fi + fi +} + ################################# case ${subcommand} in - sync ) + sync ) cncliInit && cncliSync ;; leaderlog ) cncliInit && cncliLeaderlog ;; @@ -811,5 +992,8 @@ case ${subcommand} in cncliInit && cncliInitBlocklogDB ;; metrics ) cncliMetrics ;; # no cncliInit needed - * ) usage ;; + epochdata ) + cncliInit && cncliEpochData ;; + * ) + usage ;; esac diff --git a/scripts/cnode-helper-scripts/env b/scripts/cnode-helper-scripts/env index 1335988ed..9f50a1475 100644 --- a/scripts/cnode-helper-scripts/env +++ b/scripts/cnode-helper-scripts/env @@ -874,6 +874,24 @@ getProtocolParams() { [[ ${PROT_MAJOR} -eq 0 ]] && return 1 || return 0 } +getProtocolParamsHist() { + if [[ -z ${KOIOS_API} ]]; then + return 1 + fi + + if ! PROT_PARAMS=$(curl -sSL -f -X GET -H "accept: application/json" "${KOIOS_API}/epoch_params?_epoch_no=${1}" 2>&1); then + return 2 + fi + + # Extract historical protocol major and minor versions in TSV format + read -r PROT_MAJOR PROT_MINOR <<<"$(jq -r '.[0] | [ + .protocol_major //0, + .protocol_minor //0 + ] | @tsv' <<<"${PROT_PARAMS}" 2>/dev/null)" + PROT_VERSION="${PROT_MAJOR}.${PROT_MINOR}" + +} + telegramSend() { if [[ -z "${TG_BOT_TOKEN}" ]] || [[ -z "${TG_CHAT_ID}" ]]; then echo "Warn: to use the telegramSend function you must first set the bot and chat id in the env file"