-
Notifications
You must be signed in to change notification settings - Fork 8
/
forkpatchq
197 lines (181 loc) · 10.9 KB
/
forkpatchq
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
#! /usr/bin/env bash
#! /usr/bin/env bash
function print_usage () {
echo "forkpatch help:"
echo
echo "SYNOPSIS: This tool edits the python fork source code to apply popular and time-tested patches"
echo " to chia or forks that for whatever reason the fork developer has not yet applied."
echo " forkpatch does extensive validation to make sure the code it is modifying hasn't been"
echo " altered in any way that might interfere with the automated application of the patch, but it"
echo " is still highly recommended to confirm that farming continues well following running forkpatch."
echo
echo "SUPPORTED PATCHES: -multiproc and -logwinningplots."
echo
echo " -multiproc: the excellent grayfallstown 'multiprocessing_limit' patch which, depending on"
echo " the number of cores of your CPU, greatly reduces CPU and memory usage by starting fewer full"
echo " node worker processes. The default number of workers started by chia code is roughly equivalent"
echo " to the number of threads of your CPU. For me, this results in 20 workers. This is huge overkill"
echo " for a fork that is already synced, as even setting a multiprocessing_limit of 2 is enough to keep"
echo " a chain synced without issue. -multiproc patches forkname/consensus/blockchain.py"
echo
echo " -logwinningplots: when a proof is found, this patch will create an additional line in the logs"
echo " identifying the specific plot in which that proof was found. -logwinningplots patches"
echo " forkname/harvester/harvester_api.py"
echo
echo "PARAMETERS:"
echo " forkname | all Required. Apply patch to a single fork or all forks running a farmer process."
echo " -patchname Required. Currently only -multiproc as described above is supported."
echo " -help | --help Show this information again."
echo
exit 0
}
if [[ $1 != 'all' ]]; then
VALIDATEFORKNAME='Yes'
fi
. ftinit.sh
if [[ $2 != '-multiproc' && $2 != '-logwinningplots' ]]; then
echo "forkpatch: Patch name must be specified as second parameter. Available patches are -multiproc and -logwinningplots"
echo "Aborting forkpatch."
exit
fi
. ftbuildrunlists.sh
if [[ $FTBASECOMMAND == 'forkupdate' ]]; then
FARMERLIST=$1
elif [[ $1 != 'all' ]]; then
OLDIFS=$IFS
IFS=''
FORKNAMERUNNING=$( echo $FARMERLIST | grep "^$1$" )
if [[ $FORKNAMERUNNING == '' ]]; then
echo "forkpatch: Specified fork is not currently running a farmer process. The process should be started and forkpatch run again."
echo "Aborting forkpatch."
exit
fi
IFS=$OLDIFS
fi
if [[ $2 == '-multiproc' ]]; then
for FORKNAME in $FARMERLIST; do
if [[ $1 != 'all' && $1 != $FORKNAME ]]; then
continue
fi
CURRENTCONFIG=$FORKTOOLSHIDDENDIRS/.$FORKNAME/mainnet/config/config.yaml
if [[ ! -f $CURRENTCONFIG ]]; then
continue
fi
echo
PROGRAMTOMOD=$FORKTOOLSBLOCKCHAINDIRS/$FORKNAME-blockchain/$FORKNAME/consensus/blockchain.py
if [[ -f $PROGRAMTOMOD ]]; then
echo "forkpatch: Analyzing suitability of $FORKNAME for automated -multiproc patching."
. $FORKTOOLSDIR/ftcheckprocs.sh
FORKNAMEINCODE=$FORKNAME
CODESTILLCHIA=$( grep -c '^from chia.util' $PROGRAMTOMOD )
if [[ $CODESTILLCHIA > 0 ]]; then
FORKNAMEINCODE='chia'
fi
FINDMULTIPROC=$( grep -c 'multiprocessing_limit' $PROGRAMTOMOD )
if [[ $FINDMULTIPROC > 0 ]]; then
echo "forkpatch: $FORKNAME failed code validation - already patched. Skipping $FORKNAME."
continue
fi
FINDCONFIGLOAD=$( grep -c 'import load_config' $PROGRAMTOMOD )
if [[ $FINDCONFIGLOAD > 0 ]]; then
echo "forkpatch: $FORKNAME failed code validation - loads config without loading multiprocessing_limit. Skipping $FORKNAME."
continue
fi
FINDDEFAULTPATH=$( grep -c 'DEFAULT_ROOT_PATH' $PROGRAMTOMOD )
if [[ $FINDDEFAULTPATH > 0 ]]; then
echo "forkpatch: $FORKNAME failed code validation - loads DEFAULT_ROOT_PATH without loading multiprocessing_limit. Skipping $FORKNAME."
continue
fi
FINDNUMWORKERSKEY=$( grep -c '^ num_workers = max(cpu_count - 2, 1)$' $PROGRAMTOMOD )
if [[ $FINDNUMWORKERSKEY > 0 ]]; then
CHIAPRE130=1
else
CHIAPRE130=0
FINDNUMWORKERSKEY=$( grep -c '^ num_workers = max(cpu_count - reserved_cores, 1)$' $PROGRAMTOMOD )
if [[ $FINDNUMWORKERSKEY > 0 ]]; then
CHIAPRE134=1
else
CHIAPRE134=0
FINDNUMWORKERSKEY=$( grep -c '^ num_workers = max(cpu_count - reserved_cores, 1)$' $PROGRAMTOMOD )
fi
fi
if [[ $FINDNUMWORKERSKEY == 0 ]]; then
echo "forkpatch: $FORKNAME failed code validation - num_workers code missing or has non-standard indendation. Skipping $FORKNAME."
continue
fi
FINDIMPORTKEY=$( grep -c "^from ${FORKNAMEINCODE}.util.setproctitle import getproctitle, setproctitle$" $PROGRAMTOMOD )
if [[ $FINDIMPORTKEY == 0 ]]; then
echo "forkpatch: $FORKNAME failed code validation - exact key needed for proper placement of from-import lines missing. Skipping $FORKNAME."
continue
fi
echo "forkpatch: $FORKNAME deemed suitable for automated -multiproc patching. Attempting to mod $FORKNAME/consensus/blockchain.py."
echo " blockchain.py will be backed up to $FORKNAME/consensus/blockchain.py.ftbackup"
if [[ $CHIAPRE130 == 1 ]]; then
sed -i.ftbackup "/^from ${FORKNAMEINCODE}.util.setproctitle import getproctitle, setproctitle/a # Next two lines added by forkpatch -multiproc\nfrom ${FORKNAMEINCODE}.util.default_root import DEFAULT_ROOT_PATH\nfrom ${FORKNAMEINCODE}.util.config import load_config" $PROGRAMTOMOD
sed -i.ftbak "/^ num_workers = max(cpu_count - 2, 1)/a\ # Next three lines added by forkpatch -multiproc\n config = load_config(DEFAULT_ROOT_PATH, \"config.yaml\")\n if 'multiprocessing_limit' in config.keys():\n num_workers = min(num_workers, int(config[\"multiprocessing_limit\"]));" $PROGRAMTOMOD
elif [[ $CHIAPRE134 == 1 ]]; then
sed -i.ftbackup "/^from ${FORKNAMEINCODE}.util.setproctitle import getproctitle, setproctitle/a # Next two lines added by forkpatch -multiproc\nfrom ${FORKNAMEINCODE}.util.default_root import DEFAULT_ROOT_PATH\nfrom ${FORKNAMEINCODE}.util.config import load_config" $PROGRAMTOMOD
sed -i.ftbak "/^ num_workers = max(cpu_count - reserved_cores, 1)/a\ # Next three lines added by forkpatch -multiproc\n config = load_config(DEFAULT_ROOT_PATH, \"config.yaml\")\n if 'multiprocessing_limit' in config.keys():\n num_workers = min(num_workers, int(config[\"multiprocessing_limit\"]));" $PROGRAMTOMOD
else
sed -i.ftbackup "/^from ${FORKNAMEINCODE}.util.setproctitle import getproctitle, setproctitle/a # Next two lines added by forkpatch -multiproc\nfrom ${FORKNAMEINCODE}.util.default_root import DEFAULT_ROOT_PATH\nfrom ${FORKNAMEINCODE}.util.config import load_config" $PROGRAMTOMOD
sed -i.ftbak "/^ num_workers = max(cpu_count - reserved_cores, 1)/a\ # Next three lines added by forkpatch -multiproc\n config = load_config(DEFAULT_ROOT_PATH, \"config.yaml\")\n if 'multiprocessing_limit' in config.keys():\n num_workers = min(num_workers, int(config[\"multiprocessing_limit\"]));" $PROGRAMTOMOD
fi
rm ${PROGRAMTOMOD}.ftbak
else
echo "forkpatch: $FORKNAME code $PROGRAMTOMOD not found. Skipping $FORKNAME."
continue
fi
if [[ $FTBASECOMMAND != 'forkupdate' ]]; then
. $FORKTOOLSDIR/ftrestart.sh
fi
echo "forkpatch complete: Verify patched forks have lowered fullnode worker counts in forkmon. If not, try running forkfixconfig to add the parameter."
done
fi
if [[ $2 == '-logwinningplots' ]]; then
for FORKNAME in $FARMERLIST; do
if [[ $1 != 'all' && $1 != $FORKNAME ]]; then
continue
fi
CURRENTCONFIG=$FORKTOOLSHIDDENDIRS/.$FORKNAME/mainnet/config/config.yaml
if [[ ! -f $CURRENTCONFIG ]]; then
continue
fi
echo
PROGRAMTOMOD=$FORKTOOLSBLOCKCHAINDIRS/$FORKNAME-blockchain/$FORKNAME/harvester/harvester_api.py
if [[ -f $PROGRAMTOMOD ]]; then
echo "forkpatch: Analyzing suitability of $FORKNAME for automated -logwinningplots patching."
. $FORKTOOLSDIR/ftcheckprocs.sh
FINDPROOFSINFILENAME=$( grep -c 'proofs in {filename}' $PROGRAMTOMOD )
if [[ $FINDPROOFSINFILENAME > 0 ]]; then
echo "forkpatch: $FORKNAME failed code validation - already patched. Skipping $FORKNAME."
continue
fi
SENDMSGLINE=$( grep -n '^ await peer.send_message(msg)$' $PROGRAMTOMOD | sed 's/:.*//' )
if [[ $SENDMSGLINE == '' ]]; then
echo "forkpatch: $FORKNAME failed code validation - first placement key code missing or has non-standard indendation. Skipping $FORKNAME."
continue
fi
NOWKEYLINE=$( grep -n '^ now = uint64(int(time.time()))$' $PROGRAMTOMOD | sed 's/:.*//' )
if [[ $NOWKEYLINE != '' ]]; then
let LINEDIFF=$NOWKEYLINE-$SENDMSGLINE
if [[ $LINEDIFF != 2 ]]; then
echo "forkpatch: $FORKNAME failed code validation - placement keys exist but non-standard separation. Skipping $FORKNAME."
continue
fi
else
echo "forkpatch: $FORKNAME failed code validation - second placement key code missing or has non-standard indendation. Skipping $FORKNAME."
continue
fi
echo "forkpatch: $FORKNAME deemed suitable for automated -logwinningplots patching. Attempting to mod $FORKNAME/harvester/harvester_api.py."
echo " harvester_api.py will be backed up to $FORKNAME/harvester/harvester_api.py.ftbackup"
sed -i.ftbackup "/^ await peer.send_message(msg)$/a\ # Next three lines added by forkpatch -logwinningplots\n if sublist:\n self.harvester.log.info(\n f\"Found {len(sublist)} proofs in {filename} in {time_taken:.5f} s\"\n )" $PROGRAMTOMOD
else
echo "forkpatch: $FORKNAME code $PROGRAMTOMOD not found. Skipping $FORKNAME."
continue
fi
if [[ $FTBASECOMMAND != 'forkupdate' ]]; then
. $FORKTOOLSDIR/ftrestart.sh
fi
echo "forkpatch complete: An additional line in the logs will identify the specific plot proofs are found in for all patched forks."
done
fi