Merge pull request #788 from guydavis/develop

Latest fixes from testing.

CHANGELOG.md

@@ -7,8 +7,8 @@ All notable changes to this project will be documented in this file. The format
  - Chart memory usage per container (GiB) as well as total host memory usage (%) for OS and all apps.
  - Enhanced Forktools to optionally decrease a blockchain's full_node process count, which greatly limits memory usage.
  - Improve Plotting page to display configured tmp, dst, and archiving directories before starting to plot. Support `site_path_filter` for archive folders under `site_root`.
- - Fixes: Enhance 'NFT Reward Recovery' tool to support v2 databases. Fix for invalid Chiadog harvester alerts.
- - Update: [Chia](https://github.com/Chia-Network/chia-blockchain/releases/tag/1.6.1) to v1.6.1, [Chinilla](https://github.com/Chinilla/chinilla-blockchain/releases/tag/1.3.0 to v1.3.0, [Littlelambocoin](https://github.com/BTCgreen-Network/littlelambocoin-blockchain/releases/tag/1.6.1) to v1.6.1, [Maize](https://github.com/Maize-Network/maize-blockchain/releases/tag/1.6.0) to v1.6.0, [MMX](https://github.com/madMAx43v3r/mmx-node) to `testnet7`.
+ - Fixes: Enhance 'NFT Reward Recovery' tool to support v2 databases. Fix for invalid Chiadog harvester alerts. Fix for bladebit diskplot mode (NOTE: ramplot mode under development again by original author.)
+ - Update: [Chia](https://github.com/Chia-Network/chia-blockchain/releases/tag/1.6.1) to v1.6.1, [Chinilla](https://github.com/Chinilla/chinilla-blockchain/releases/tag/1.3.0 to v1.3.0, [Littlelambocoin](https://github.com/BTCgreen-Network/littlelambocoin-blockchain/releases/tag/1.6.1) to v1.6.1, [Maize](https://github.com/Maize-Network/maize-blockchain/releases/tag/1.6.0) to v1.6.0, [MMX](https://github.com/madMAx43v3r/mmx-node) to `testnet8`.
 
 ## [0.8.4] - 2022-09-21
 - Scaling-Down: Optional mode where wallets are synced daily, not run 24/7.  Saves ~35% memory so smaller farmers can farm more blockchains concurrently on the same machine. See Wallets page, top-right Settings.

CREDITS.md

@@ -100,6 +100,7 @@ A big thanks to all that contributed with dev and test including:
 * @fabriziocacicia (Fabrizio Cacicia)
 * @bdeprez (Bernie Deprez)
 * Gnomuz
+* borifrmr
 
 ## Trademark Notice
 CHIA NETWORK INC, CHIA™, the CHIA BLOCKCHAIN™, the CHIA PROTOCOL™, CHIALISP™ and the “leaf Logo” (including the leaf logo alone when it refers to or indicates Chia), are trademarks or registered trademarks of Chia Network, Inc., a Delaware corporation.  *There is no affliation between this Machinaris project and the main Chia Network project.*

api/commands/log_parser.py

@@ -18,6 +18,7 @@
 
 from common.config import globals
 from api.models import log
+from . import plotman_cli
 from api import app
 
 # Rough number of challenges arriving per minute on a blockchain
@@ -114,25 +115,6 @@ def recent_farmed_blocks(blockchain):
     #app.logger.info(blocks.rows)
     return blocks
 
-def find_plotting_job_log(plot_id):
-    dir_path = '/root/.chia/plotman/logs'
-    directory = os.fsencode(dir_path)
-    for file in os.listdir(directory):
-        filename = os.fsdecode(file)
-        try:
-            if filename.endswith(".log") and not filename.startswith('plotman.'):
-                with open(os.path.join(str(dir_path), filename)) as logfile:
-                    for line in itertools.islice(logfile, 0, 20):
-                        if plot_id in line:
-                            return os.path.join(str(dir_path), filename)
-                continue
-            else:
-                continue
-        except:
-            app.logger.info("find_plotting_job_log: Skipping error when reading head of {0}".format(filename))
-            app.logger.info(traceback.format_exc())
-    return None
-
 def get_farming_log_file(blockchain):
     mainnet_folder = globals.get_blockchain_network_path(blockchain)
     if blockchain == 'mmx':
@@ -144,7 +126,7 @@ def get_log_lines(log_type, log_id=None, blockchain=None):
         log_file = "/root/.chia/chiadog/logs/chiadog.log"
     elif log_type == "plotting":
         if log_id:
-            log_file = find_plotting_job_log(log_id)
+            log_file = plotman_cli.find_plotting_job_log(log_id)
         else:
             log_file = "/root/.chia/plotman/logs/plotman.log"
     elif log_type == "archiving":

api/commands/plotman_cli.py

@@ -238,9 +238,9 @@ def find_plotting_job_log(plot_id):
     for file in os.listdir(directory):
         filename = os.fsdecode(file)
         try:
-            if filename.endswith(".log") and not filename.startswith('plotman.'):
+            if filename.endswith(".log") and not filename.startswith('plotman.') and not filename.startswith('archiver.'):
                 with open(os.path.join(str(dir_path), filename)) as logfile:
-                    for line in itertools.islice(logfile, 0, 20):
+                    for line in itertools.islice(logfile, 0, 35):
                         if plot_id in line:
                             return os.path.join(str(dir_path), filename)
                 continue

api/translations/de_DE/LC_MESSAGES/messages.po

@@ -7,7 +7,7 @@ msgstr ""
 "Project-Id-Version: Machinaris VERSION\n"
 "Report-Msgid-Bugs-To: https://github.com/guydavis/machinaris"
 "/issuesCreation-Date: 2022-02-16 15:00-0700\n"
-"POT-Creation-Date: 2022-10-19 14:25-0600\n"
+"POT-Creation-Date: 2022-10-22 06:07-0600\n"
 "PO-Revision-Date: 2022-02-15 14:29-0700\n"
 "Last-Translator: ApfelBirneKreis\n"
 "Language: de\n"

api/translations/fr_FR/LC_MESSAGES/messages.po

@@ -7,7 +7,7 @@ msgstr ""
 "Project-Id-Version: Machinaris VERSION\n"
 "Report-Msgid-Bugs-To: https://github.com/guydavis/machinaris"
 "/issuesCreation-Date: 2022-02-16 15:00-0700\n"
-"POT-Creation-Date: 2022-10-19 14:25-0600\n"
+"POT-Creation-Date: 2022-10-22 06:07-0600\n"
 "PO-Revision-Date: 2022-02-15 14:29-0700\n"
 "Last-Translator: Guy Davis\n"
 "Language: fr\n"

api/translations/it_IT/LC_MESSAGES/messages.po

@@ -7,7 +7,7 @@ msgstr ""
 "Project-Id-Version: Machinaris VERSION\n"
 "Report-Msgid-Bugs-To: https://github.com/guydavis/machinaris"
 "/issuesCreation-Date: 2022-02-16 15:00-0700\n"
-"POT-Creation-Date: 2022-10-19 14:25-0600\n"
+"POT-Creation-Date: 2022-10-22 06:07-0600\n"
 "PO-Revision-Date: 2022-02-15 14:29-0700\n"
 "Last-Translator: Fabrizio Cacicia\n"
 "Language: it\n"

api/translations/nl_NL/LC_MESSAGES/messages.po

@@ -7,7 +7,7 @@ msgstr ""
 "Project-Id-Version: Machinaris VERSION\n"
 "Report-Msgid-Bugs-To: https://github.com/guydavis/machinaris/issuesPOT-"
 "Creation-Date: 2022-03-13 10:30-0600\n"
-"POT-Creation-Date: 2022-10-19 14:25-0600\n"
+"POT-Creation-Date: 2022-10-22 06:07-0600\n"
 "PO-Revision-Date: 2022-03-13 10:29-0600\n"
 "Last-Translator: Bernie Deprez\n"
 "Language: nl_NL\n"

api/translations/pt_PT/LC_MESSAGES/messages.po

@@ -7,7 +7,7 @@ msgstr ""
 "Project-Id-Version: Machinaris VERSION\n"
 "Report-Msgid-Bugs-To: https://github.com/guydavis/machinaris"
 "/issuesCreation-Date: 2022-02-16 15:00-0700\n"
-"POT-Creation-Date: 2022-10-19 14:25-0600\n"
+"POT-Creation-Date: 2022-10-22 06:07-0600\n"
 "PO-Revision-Date: 2022-02-13 13:45-0700\n"
 "Last-Translator: Antonio Casqueiro\n"
 "Language: pt\n"

api/translations/zh/LC_MESSAGES/messages.po

@@ -7,7 +7,7 @@ msgstr ""
 "Project-Id-Version: Machinaris VERSION\n"
 "Report-Msgid-Bugs-To: https://github.com/guydavis/machinaris"
 "/issuesCreation-Date: 2022-02-16 15:00-0700\n"
-"POT-Creation-Date: 2022-10-19 14:25-0600\n"
+"POT-Creation-Date: 2022-10-22 06:07-0600\n"
 "PO-Revision-Date: 2022-02-15 14:29-0700\n"
 "Last-Translator: Guy Davis\n"
 "Language: zh\n"

common/config/blockchains.json

@@ -275,8 +275,8 @@
         "name": "MMX",
         "symbol": "MMX" ,
         "binary":  "/mmx-node/build/mmx",
-        "network_path": "/root/.mmx/testnet7",
-        "network_name": "testnet7",
+        "network_path": "/root/.mmx/testnet8",
+        "network_name": "testnet8",
         "network_port": 12334,
         "farmer_port": 11330,
         "worker_port": 8940,

docker/dockerfile

@@ -8,7 +8,7 @@ ARG PLOTMAN_BRANCH=development
 ARG CHIADOG_BRANCH=main
 ARG FDCLI_BRANCH=master
 ARG FORKTOOLS_BRANCH=testing
-ARG BLADEBIT_BRANCH=master
+ARG BLADEBIT_BRANCH=2.0.0-beta1
 
 ARG APPLE_BRANCH
 ARG BPX_BRANCH
@@ -126,6 +126,7 @@ ENV CHIADOG_BRANCH=${CHIADOG_BRANCH}
 ENV FDCLI_BRANCH=${FDCLI_BRANCH}
 ENV FORKTOOLS_BRANCH=${FORKTOOLS_BRANCH}
 ENV BLADEBIT_BRANCH=${BLADEBIT_BRANCH}
+ENV MMX_HOME=/root/.mmx/
 
 VOLUME [ "/id_rsa" ]
 

scripts/forks/btcgreen_launch.sh

@@ -18,6 +18,7 @@ if [[ "${blockchain_db_download}" == 'true' ]] \
   && [[ ! -f /root/.btcgreen/mainnet/db/blockchain_v2_mainnet.sqlite ]]; then
   mkdir -p /root/.btcgreen/mainnet/db/ && cd /root/.btcgreen/mainnet/db/
   # Mega links for BTCGreen blockchain DB from their Discord 2022-08-23
+  /usr/bin/bash /machinaris/scripts/megacmd_setup.sh > /tmp/megacmd_setup.log 2>&1
   mega-get https://mega.nz/folder/YllTjSYR#S-YDbL6Y98qPpAN81UTs-g
   mv BTCgreen_8_21_22/* . && rm -rf BTCgreen_8_21_22
 fi

scripts/forks/chives_launch.sh

@@ -16,13 +16,8 @@ if [[ "${blockchain_db_download}" == 'true' ]] \
   && [[ "${mode}" == 'fullnode' ]] \
   && [[ ! -f /root/.chives/mainnet/db/blockchain_v1_mainnet.sqlite ]] \
   && [[ ! -f /root/.chives/mainnet/db/blockchain_v2_mainnet.sqlite ]]; then
-  echo "Downloading Chives blockchain DB (many GBs in size) on first launch..."
-  echo "Please be patient as takes minutes now, but saves days of syncing time later."
-  mkdir -p /root/.chives/mainnet/db/ && cd /root/.chives/mainnet/db/
-  # Download link from https://node-hk.chivescoin.org/
-  gdown 1meeY39NbcNcI40OG-WZWo4Rnwi6BHLEw
-  unzip blockchain_v2_mainnet*.zip
-  rm -f blockchain_v2_mainnet*.zip
+  echo "Sorry, Chives does not offer a recent blockchain DB for download.  Standard sync will happen over a few days."
+  echo "It is recommended to add some peer node connections on the Connections page of Machinaris from: https://alltheblocks.net/chives"
 fi
 
 mkdir -p /root/.chives/mainnet/log

scripts/forks/cryptodoge_launch.sh

@@ -20,8 +20,8 @@ if [[ "${blockchain_db_download}" == 'true' ]] \
   echo "Downloading Cryptodoge blockchain DB (many GBs in size) on first launch..."
   echo "Please be patient as takes minutes now, but saves days of syncing time later."
   mkdir -p /root/.cryptodoge/mainnet/db/ && cd /root/.cryptodoge/mainnet/db/
-  # Latest Blockchain DB download from https://gamefi.cryptodoge.cc
-  curl -skLJO https://gamefi.cryptodoge.cc/blockchain_v2_mainnet.sqlite
+  # Latest Blockchain DB download from their Discord - Oct 2022
+  curl -skLJO https://dbs.ctek.cc/$/oXE5V
 fi
 
 mkdir -p /root/.cryptodoge/mainnet/log

scripts/forks/ecostake_launch.sh

@@ -16,12 +16,8 @@ if [[ "${blockchain_db_download}" == 'true' ]] \
   && [[ "${mode}" == 'fullnode' ]] \
   && [[ ! -f /root/.ecostake/mainnet/db/blockchain_v1_mainnet.sqlite ]] \
   && [[ ! -f /root/.ecostake/mainnet/db/blockchain_v2_mainnet.sqlite ]]; then
-  echo "Downloading Ecostake blockchain DB (many GBs in size) on first launch..."
-  echo "Please be patient as takes minutes now, but saves days of syncing time later."
-  mkdir -p /root/.ecostake/mainnet/db/ && cd /root/.ecostake/mainnet/db/
-  # Download link from their Discord on 2022-08-20, not compressed, 6 GB
-  gdown 1MmbxJRvWXdN317Ikv7JQe370yx8m0SDB
-  #p7zip --decompress --force blockchain_v1_mainnet*.7z
+  echo "Sorry, Ecostake does not offer a recent blockchain DB for download.  Standard sync will happen over a few weeks."
+  echo "It is recommended to add some peer node connections on the Connections page of Machinaris from: https://alltheblocks.net/ecostake"
 fi
 
 mkdir -p /root/.ecostake/mainnet/log

scripts/forks/gold_launch.sh

@@ -18,11 +18,12 @@ if [[ "${blockchain_db_download}" == 'true' ]] \
   && [[ ! -f /root/.gold/mainnet/db/blockchain_v2_mainnet.sqlite ]]; then
   echo "Downloading Gold blockchain DB (many GBs in size) on first launch..."
   echo "Please be patient as takes minutes now, but saves days of syncing time later."
+  /usr/bin/bash /machinaris/scripts/megacmd_setup.sh > /tmp/megacmd_setup.log 2>&1
   mkdir -p /root/.gold/mainnet/db/ && cd /root/.gold/mainnet/db/
-  # Latest Blockchain DB download as per the Gold Discord
-  curl -skJLO http://58.7.212.211/C%3A/http/Gold/951456/Gold.blockchain_v1_mainnet.height.951456.db.rar
-  unrar e *.rar 
-  rm -f *.rar
+  # Mega links for Gold blockchain DB from their Discord 2022-10-24
+  mega-get https://mega.nz/folder/XAhB1bbC#vvlz5NKwtL0iS_WXGsVg2w
+  cd db/ && unzip "db-$(date +'%Y-%m-%d').zip" && mv *.sqlite ../
+  cd ../ && rm -rf db
 fi
 
 mkdir -p /root/.gold/mainnet/log

scripts/forks/mmx_install.sh

@@ -4,8 +4,8 @@
 #
 
 MMX_BRANCH=$1
-# On 2022-10-19
-HASH=85f7f41245e92827afb6d8db8873cc820af401cd
+# On 2022-10-24
+HASH=744dfe66729099f12dc697e8576feb0910d6ecd9
 
 if [ -z ${MMX_BRANCH} ]; then
 	echo 'Skipping MMX install as not requested.'

scripts/forks/mmx_launch.sh

@@ -5,21 +5,20 @@
 
 cd /mmx-node
 
-rm -rf ./logs
 mkdir -p /root/.chia/mmx/logs
-ln -s /root/.chia/mmx/logs /mmx-node/logs
 if [ ! -L /root/.mmx ]; then
 	ln -s /root/.chia/mmx /root/.mmx
 fi
 
+echo "Launching MMX with storage at: ${MMX_HOME}"
+
 IFS=':' read -r -a array <<< "$plots_dir"
 joined=$(printf ", \"%s\"" "${array[@]}")
 plot_dirs=${joined:1}
 echo "Adding plot directories at: ${plot_dirs}"
 
 # Setup configuration for MMX inside a Docker container
 if [ ! -d /root/.chia/mmx/config ]; then
-	mv ./config /root/.chia/mmx/
 	mkdir -p /root/.chia/mmx/config/local
 	tee /root/.chia/mmx/config/local/Harvester.json >/dev/null <<EOF
 {
@@ -28,14 +27,9 @@ if [ ! -d /root/.chia/mmx/config ]; then
 EOF
 	# For a fresh install of Machinaris-MMX, disable timelord by default to save CPU usage
 	echo false > /root/.chia/mmx/config/local/timelord
-elif [ ! -d /root/.chia/mmx/config/testnet7 ]; then # Handle an upgrade from older testnet
-	echo 'Copying over new testnet configs to /root/.chia/mmx/config/'
-	cp -r ./config/testnet7 /root/.chia/mmx/config/
 fi
-rm -rf ./config
-ln -s /root/.chia/mmx/config /mmx-node/config
 escaped_plot_dirs=$(printf '%s\n' "$plot_dirs" | sed -e 's/[\/&]/\\&/g')
-sed -i "s/\"plot_dirs\":.*$/\"plot_dirs\": [ $escaped_plot_dirs ]/g" ./config/local/Harvester.json
+sed -i "s/\"plot_dirs\":.*$/\"plot_dirs\": [ $escaped_plot_dirs ]/g" /root/.chia/mmx/config/local/Harvester.json
 
 if [[ ${OPENCL_GPU} == 'nvidia' ]]; then    
     mkdir -p /etc/OpenCL/vendors
@@ -54,33 +48,22 @@ else
 	echo "No OPENCL_GPU provided.  MMX blockchain will use use CPU instead."
 fi
 
-# Symlink the NETWORK file, use 'testnet7' for now
-#if [ ! -f /root/.chia/mmx/NETWORK ]; then
-echo 'testnet7' > /root/.chia/mmx/NETWORK
-#fi
-rm -f ./NETWORK
-ln -s /root/.chia/mmx/NETWORK /mmx-node/NETWORK
+echo 'testnet8' > /root/.chia/mmx/NETWORK
 
-# Symlink the testnet7 folder
-if [ ! -d /root/.chia/mmx/testnet7 ]; then
-	mkdir /root/.chia/mmx/testnet7
+# Symlink the testnet8 folder
+if [ ! -d /root/.chia/mmx/testnet8 ]; then
+	mkdir /root/.chia/mmx/testnet8
 fi
-rm -rf ./testnet7
-ln -s /root/.chia/mmx/testnet7 /mmx-node/testnet7
 
 # Create a key if none found from previous runs
 if [[ ${mode} == 'fullnode' ]]; then
 	if [ ! -f /root/.chia/mmx/wallet.dat ]; then
 		echo "Creating key at path: /root/.chia/mmx/wallet.dat"
 		mmx wallet create
-		mv wallet.dat /root/.chia/mmx/
 	else
 		echo "Adding key at path: /root/.chia/mmx/wallet.dat"
 	fi
-	if [ ! -L /mmx-node/wallet.dat ]; then
-		ln -s /root/.chia/mmx/wallet.dat /mmx-node/wallet.dat
-	fi
-		# Setup log rotation
+	# Setup log rotation
 	tee /etc/logrotate.d/mmx-node >/dev/null <<EOF
 	/root/.chia/mmx/logs/mmx_node.log {
 	rotate 3

scripts/forks/profit_launch.sh

@@ -16,12 +16,8 @@ if [[ "${blockchain_db_download}" == 'true' ]] \
   && [[ "${mode}" == 'fullnode' ]] \
   && [[ ! -f /root/.profit/mainnet/db/blockchain_v1_mainnet.sqlite ]] \
   && [[ ! -f /root/.profit/mainnet/db/blockchain_v2_mainnet.sqlite ]]; then
-  echo "Downloading Profit blockchain DB (many GBs in size) on first launch..."
-  echo "Please be patient as takes minutes now, but saves days of syncing time later."
-  mkdir -p /root/.profit/mainnet/db/ && cd /root/.profit/mainnet/db/
-   # Download link from their Discord on 2022-08-20, not compressed 7 tb
-  gdown 1wE-opD-lwF5rmXL3D2kw1slBZvxXS-BE
-  #p7zip --decompress --force blockchain_v1_mainnet*.7z
+  echo "Sorry, Profit does not offer a recent blockchain DB for download.  Standard sync will happen over a few weeks."
+  echo "It is recommended to add some peer node connections on the Connections page of Machinaris from: https://alltheblocks.net/profit"
 fi
 
 mkdir -p /root/.profit/mainnet/log

scripts/forks/shibgreen_launch.sh

@@ -18,6 +18,7 @@ if [[ "${blockchain_db_download}" == 'true' ]] \
   && [[ ! -f /root/.shibgreen/mainnet/db/blockchain_v2_mainnet.sqlite ]]; then
   mkdir -p /root/.shibgreen/mainnet/db/ && cd /root/.shibgreen/mainnet/db/
   # Mega links for SHIBGreen blockchain DB from their Discord 2022-08-23
+  /usr/bin/bash /machinaris/scripts/megacmd_setup.sh > /tmp/megacmd_setup.log 2>&1
   mega-get https://mega.nz/folder/QpU3UbgQ#J76hG6GE5cYioiLyCweV1w
   mv shibgreen_8_21_22/* . && rm -rf shibgreen_8_21_22
 fi

scripts/forks/staicoin_install.sh

@@ -4,8 +4,8 @@
 #
 
 STAICOIN_BRANCH=$1
-# On 2022-09-17
-HASH=4808632d8147b1fb8ec9ec7aad5a80f86a509b3d
+# On 2022-10-24
+HASH=2fad4e4f12e374eff6e83b2830876902139156f9
 
 if [ -z ${STAICOIN_BRANCH} ]; then
 	echo 'Skipping Staicoin install as not requested.'

scripts/plotman_setup.sh

@@ -13,5 +13,6 @@ if [[ (${mode} == 'fullnode' || ${mode} =~ "plotter") && (${blockchains} == 'chi
         cd plotman
         /chia-blockchain/venv/bin/python setup.py install
         apt update && apt install -y rsync
+        mkdir -p /root/.chia/plotman/logs/archiving
     fi
 fi

scripts/start_machinaris.sh

@@ -41,7 +41,7 @@ fi
 . /machinaris/scripts/setup_databases.sh
 
 mkdir -p /root/.chia/machinaris/config
-mkdir -p /root/.chia/machinaris/logs/archiving
+mkdir -p /root/.chia/machinaris/logs
 cd /machinaris
 if [ $FLASK_ENV == "development" ];
 then

web/actions/chia.py

@@ -155,7 +155,7 @@ def load_keys():
 def load_farmers():
     farmers = wk.load_worker_summary().farmers_harvesters()
     for farmer in farmers:
-        app.logger.info("Load farmer statistics for {0}".format(farmer.displayname))
+        #app.logger.info("Load farmer statistics for {0}".format(farmer.displayname))
         farmer.plot_counts = str(stats.count_plots_by_ksize(farmer.hostname))[1:-1].replace("'", "")
         farmer.plot_types = str(stats.count_plots_by_type(farmer.hostname))[1:-1].replace("'", "")
         farmer.drive_count = stats.count_drives(farmer.hostname)