backport: merge bitcoin#19192, #18968, #19191, #18991, #19658, #19697, #20000, #19638, #20120, #20757, #20965, #20284, #22025, #22179 (auxiliary backports: part 10)

CONTRIBUTING.md

@@ -89,7 +89,7 @@ include:
 Examples:
 
     feat(consensus): add new opcode for BIP-XXXX OP_CHECKAWESOMESIG
-    feat(net): automatically create hidden service, listen on Tor
+    feat(net): automatically create onion service, listen on Tor
     feat(qt): add feed bump button
     fix(log): fix typo in log message
     feat(rpc)!: modify gettransaction parameter type

doc/JSON-RPC-interface.md

@@ -60,7 +60,7 @@ RPC interface will be abused.
   are sent as clear text that can be read by anyone on your network
   path.  Additionally, the RPC interface has not been hardened to
   withstand arbitrary Internet traffic, so changing the above settings
-  to expose it to the Internet (even using something like a Tor hidden
+  to expose it to the Internet (even using something like a Tor onion
   service) could expose you to unconsidered vulnerabilities.  See
   `dashd -help` for more information about these settings and other
   settings described in this document.

doc/files.md

@@ -65,7 +65,7 @@ Subdirectory       | File(s)               | Description
 `./`               | `fee_estimates.dat`   | Stores statistics used to estimate minimum transaction fees and priorities required for confirmation
 `./`               | `guisettings.ini.bak` | Backup of former [GUI settings](#gui-settings) after `-resetguisettings` option is used
 `./`               | `mempool.dat`         | Dump of the mempool's transactions
-`./`               | `onion_v3_private_key` | Cached Tor hidden service private key for `-listenonion` option
+`./`               | `onion_v3_private_key` | Cached Tor onion service private key for `-listenonion` option
 `./`               | `peers.dat`           | Peer IP address database (custom format)
 `./`               | `settings.json`       | Read-write settings set through GUI or RPC interfaces, augmenting manual settings from [dash.conf](dash-conf.md). File is created automatically if read-write settings storage is not disabled with `-nosettings` option. Path can be specified with `-settings` option
 `./`               | `.cookie`             | Session RPC authentication cookie; if used, created at start and deleted on shutdown; can be specified by `-rpccookiefile` option

doc/reduce-traffic.md

@@ -23,7 +23,7 @@ longer serving historic blocks (blocks older than one week).
 Keep in mind that new nodes require other nodes that are willing to serve
 historic blocks.
 
-Whitelisted peers will never be disconnected, although their traffic counts for
+Peers with the `download` permission will never be disconnected, although their traffic counts for
 calculating the target.
 
 ## 2. Disable "listening" (`-listen=0`)

doc/release-notes-19191.md

@@ -0,0 +1,7 @@
+Updated settings
+----------------
+
+- A `download` permission has been extracted from the `noban` permission. For
+  compatibility, `noban` implies the `download` permission, but this may change
+  in future releases. Refer to the help of the affected settings `-whitebind`
+  and `-whitelist` for more details. (#19191)

doc/tor.md

@@ -1,6 +1,6 @@
 # TOR SUPPORT IN DASH CORE
 
-It is possible to run Dash Core as a Tor hidden service, and connect to such services.
+It is possible to run Dash Core as a Tor onion service, and connect to such services.
 
 The following directions assume you have a Tor proxy running on port 9050. Many
 distributions default to having a SOCKS proxy listening on port 9050, but others
@@ -16,14 +16,19 @@ outgoing connections, but more is possible.
 
 	-proxy=ip:port  Set the proxy server. If SOCKS5 is selected (default), this proxy
 	                server will be used to try to reach .onion addresses as well.
+	                You need to use -noonion or -onion=0 to explicitly disable
+	                outbound access to onion services.
 
-	-onion=ip:port  Set the proxy server to use for Tor hidden services. You do not
-	                need to set this if it's the same as -proxy. You can use -noonion
-	                to explicitly disable access to hidden services.
+	-onion=ip:port  Set the proxy server to use for Tor onion services. You do not
+	                need to set this if it's the same as -proxy. You can use -onion=0
+	                to explicitly disable access to onion services.
+	                Note: Only the -proxy option sets the proxy for DNS requests;
+	                with -onion they will not route over Tor, so use -proxy if you
+	                have privacy concerns.
 
 	-listen         When using -proxy, listening is disabled by default. If you want
-	                to run a hidden service (see next section), you'll need to enable
-	                it explicitly.
+	                to manually configure an onion service (see section 3), you'll
+	                need to enable it explicitly.
 
 	-connect=X      When behind a Tor proxy, you can specify .onion addresses instead
 	-addnode=X      of IP addresses or hostnames in these parameters. It requires
@@ -33,7 +38,11 @@ outgoing connections, but more is possible.
 	-onlynet=onion  Make outgoing connections only to .onion addresses. Incoming
 	                connections are not affected by this option. This option can be
 	                specified multiple times to allow multiple network types, e.g.
-	                ipv4, ipv6, or onion.
+	                ipv4, ipv6 or onion. If you use this option with values other
+	                than onion you *cannot* disable onion connections; outgoing onion
+	                connections will be enabled when you use -proxy or -onion. Use
+	                -noonion or -onion=0 if you want to be sure there are no outbound
+	                onion connections over the default proxy or your defined -proxy.
 
 An example how to start the client if the Tor proxy is running on local host on
 port 9050 and only allows .onion nodes to connect:
@@ -44,8 +53,99 @@ In a typical situation, this suffices to run behind a Tor proxy:
 
 	./dashd -proxy=127.0.0.1:9050
 
+## 2. Automatically create a Dash Core onion service
 
-## 2. Run a Dash Core hidden server
+Dash Core makes use of Tor's control socket API to create and destroy
+ephemeral onion services programmatically. This means that if Tor is running and
+proper authentication has been configured, Dash Core automatically creates an
+onion service to listen on. The goal is to increase the number of available
+onion nodes.
+
+This feature is enabled by default if Dash Core is listening (`-listen`) and
+it requires a Tor connection to work. It can be explicitly disabled with
+`-listenonion=0`. If it is not disabled, it can be configured using the
+`-torcontrol` and `-torpassword` settings.
+
+To see verbose Tor information in the dashd debug log, pass `-debug=tor`.
+
+### Control Port
+
+You may need to set up the Tor Control Port. On Linux distributions there may be
+some or all of the following settings in `/etc/tor/torrc`, generally commented
+out by default (if not, add them):
+
+```
+ControlPort 9051
+CookieAuthentication 1
+CookieAuthFileGroupReadable 1
+```
+
+Add or uncomment those, save, and restart Tor (usually `systemctl restart tor`
+or `sudo systemctl restart tor` on most systemd-based systems, including recent
+Debian and Ubuntu, or just restart the computer).
+
+On some systems (such as Arch Linux), you may also need to add the following
+line:
+
+```
+DataDirectoryGroupReadable 1
+```
+
+### Authentication
+
+Connecting to Tor's control socket API requires one of two authentication
+methods to be configured: cookie authentication or dashd's `-torpassword`
+configuration option.
+
+#### Cookie authentication
+
+For cookie authentication, the user running dashd must have read access to
+the `CookieAuthFile` specified in the Tor configuration. In some cases this is
+preconfigured and the creation of an onion service is automatic. Don't forget to
+use the `-debug=tor` dashd configuration option to enable Tor debug logging.
+
+If a permissions problem is seen in the debug log, e.g. `tor: Authentication
+cookie /run/tor/control.authcookie could not be opened (check permissions)`, it
+can be resolved by adding both the user running Tor and the user running
+dashd to the same Tor group and setting permissions appropriately.
+
+On Debian-derived systems, the Tor group will likely be `debian-tor` and one way
+to verify could be to list the groups and grep for a "tor" group name:
+
+```
+getent group | cut -d: -f1 | grep -i tor
+```
+
+You can also check the group of the cookie file. On most Linux systems, the Tor
+auth cookie will usually be `/run/tor/control.authcookie`:
+
+```
+stat -c '%G' /run/tor/control.authcookie
+```
+
+Once you have determined the `${TORGROUP}` and selected the `${USER}` that will
+run dashd, run this as root:
+
+```
+usermod -a -G ${TORGROUP} ${USER}
+```
+
+Then restart the computer (or log out) and log in as the `${USER}` that will run
+dashd.
+
+#### `torpassword` authentication
+
+For the `-torpassword=password` option, the password is the clear text form that
+was used when generating the hashed password for the `HashedControlPassword`
+option in the Tor configuration file.
+
+The hashed password can be obtained with the command `tor --hash-password
+password` (refer to the [Tor Dev
+Manual](https://2019.www.torproject.org/docs/tor-manual.html.en) for more
+details).
+
+
+## 3. Manually create a Dash Core onion service
 
 If you configure your Tor system accordingly, it is possible to make your node also
 reachable from the Tor network. Add these lines to your /etc/tor/torrc (or equivalent
@@ -101,7 +201,7 @@ for normal IPv4/IPv6 communication, use:
 	./dashd -onion=127.0.0.1:9050 -externalip=ssapp53tmftyjmjb.onion -discover
 
 
-## 3. List of known Dash Core Tor relays
+## 3.1. List of known Dash Core Tor relays
 
 Note: All these nodes are hosted by masternodehosting.com
 
@@ -116,45 +216,10 @@ Note: All these nodes are hosted by masternodehosting.com
 * ys5upbdeotplam3y.onion
 * fijy6aikzxfea54i.onion
 
+## 4. Privacy recommendations
 
-## 4. Automatically listen on Tor
-
-Starting with Tor version 0.2.7.1 it is possible, through Tor's control socket
-API, to create and destroy 'ephemeral' hidden services programmatically.
-Dash Core has been updated to make use of this.
-
-This means that if Tor is running (and proper authentication has been configured),
-Dash Core automatically creates a hidden service to listen on. This will positively
-affect the number of available .onion nodes.
-
-This new feature is enabled by default if Dash Core is listening (`-listen`), and
-requires a Tor connection to work. It can be explicitly disabled with `-listenonion=0`
-and, if not disabled, configured using the `-torcontrol` and `-torpassword` settings.
-To show verbose debugging information, pass `-debug=tor`.
-
-Connecting to Tor's control socket API requires one of two authentication methods to be
-configured. It also requires the control socket to be enabled, e.g. put `ControlPort 9051`
-in `torrc` config file. For cookie authentication the user running dashd must have read
-access to the `CookieAuthFile` specified in Tor configuration. In some cases this is
-preconfigured and the creation of a hidden service is automatic. If permission problems
-are seen with `-debug=tor` they can be resolved by adding both the user running Tor and
-the user running dashd to the same group and setting permissions appropriately. On
-Debian-based systems the user running dashd can be added to the debian-tor group,
-which has the appropriate permissions. Before starting dashd you will need to re-login
-to allow debian-tor group to be applied. Otherwise you will see the following notice: "tor:
-Authentication cookie /run/tor/control.authcookie could not be opened (check permissions)"
-on debug.log.
-
-An alternative authentication method is the use
-of the `-torpassword=password` option. The `password` is the clear text form that
-was used when generating the hashed password for the `HashedControlPassword` option
-in the tor configuration file. The hashed password can be obtained with the command
-`tor --hash-password password` (read the tor manual for more details).
-
-## 5. Privacy recommendations
-
-- Do not add anything but Dash Core ports to the hidden service created in section 2.
-  If you run a web service too, create a new hidden service for that.
-  Otherwise it is trivial to link them, which may reduce privacy. Hidden
-  services created automatically (as in section 3) always have only one port
+- Do not add anything but Dash Core ports to the onion service created in section 3.
+  If you run a web service too, create a new onion service for that.
+  Otherwise it is trivial to link them, which may reduce privacy. Onion
+  services created automatically (as in section 2) always have only one port
   open.

src/addrman.cpp

@@ -73,6 +73,38 @@ double CAddrInfo::GetChance(int64_t nNow) const
     return fChance;
 }
 
+void CAddrMan::RemoveInvalid()
+{
+    for (size_t bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; ++bucket) {
+        for (size_t i = 0; i < ADDRMAN_BUCKET_SIZE; ++i) {
+            const auto id = vvNew[bucket][i];
+            if (id != -1 && !mapInfo[id].IsValid()) {
+                ClearNew(bucket, i);
+            }
+        }
+    }
+
+    for (size_t bucket = 0; bucket < ADDRMAN_TRIED_BUCKET_COUNT; ++bucket) {
+        for (size_t i = 0; i < ADDRMAN_BUCKET_SIZE; ++i) {
+            const auto id = vvTried[bucket][i];
+            if (id == -1) {
+                continue;
+            }
+            const auto& addr_info = mapInfo[id];
+            if (addr_info.IsValid()) {
+                continue;
+            }
+            vvTried[bucket][i] = -1;
+            --nTried;
+            SwapRandom(addr_info.nRandomPos, vRandom.size() - 1);
+            vRandom.pop_back();
+            mapAddr.erase(addr_info);
+            mapInfo.erase(id);
+            m_tried_collisions.erase(id);
+        }
+    }
+}
+
 CAddrInfo* CAddrMan::Find(const CService& addr, int* pnId)
 {
     CService addr2 = addr;
@@ -500,11 +532,15 @@ int CAddrMan::Check_()
 }
 #endif
 
-void CAddrMan::GetAddr_(std::vector<CAddress>& vAddr)
+void CAddrMan::GetAddr_(std::vector<CAddress>& vAddr, size_t max_addresses, size_t max_pct)
 {
-    unsigned int nNodes = ADDRMAN_GETADDR_MAX_PCT * vRandom.size() / 100;
-    if (nNodes > ADDRMAN_GETADDR_MAX)
-        nNodes = ADDRMAN_GETADDR_MAX;
+    size_t nNodes = vRandom.size();
+    if (max_pct != 0) {
+        nNodes = max_pct * nNodes / 100;
+    }
+    if (max_addresses != 0) {
+        nNodes = std::min(nNodes, max_addresses);
+    }
 
     // gather a list of random nodes, skipping those of low quality
     for (unsigned int n = 0; n < vRandom.size(); n++) {