Merge pull request #5574 from kwvg/auxports11

backport: merge bitcoin#19326, bitcoin#20464, bitcoin#23451, bitcoin#20852, bitcoin#20355, bitcoin#22050, bitcoin#23115, bitcoin#23409, bitcoin#18468, bitcoin#23413, partial bitcoin#21560, bitcoin#23438 (auxiliary backports: part 11)

contrib/seeds/generate-seeds.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright (c) 2014-2017 Wladimir J. van der Laan
+# Copyright (c) 2014-2021 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 '''
@@ -13,44 +13,47 @@
 
 These files must consist of lines in the format
 
-    <ip>
     <ip>:<port>
-    [<ipv6>]
     [<ipv6>]:<port>
-    <onion>.onion
-    0xDDBBCCAA (IPv4 little-endian old pnSeeds format)
+    <onion>.onion:<port>
 
 The output will be two data structures with the peers in binary format:
 
-   static SeedSpec6 pnSeed6_main[]={
-   ...
-   }
-   static SeedSpec6 pnSeed6_test[]={
+   static const uint8_t chainparams_seed_{main,test}[]={
    ...
    }
 
 These should be pasted into `src/chainparamsseeds.h`.
 '''
 
 from base64 import b32decode
-from binascii import a2b_hex
+from enum import Enum
+import struct
 import sys
 import os
 import re
 
-# ipv4 in ipv6 prefix
-pchIPv4 = bytearray([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff])
-# tor-specific ipv6 prefix
-pchOnionCat = bytearray([0xFD,0x87,0xD8,0x7E,0xEB,0x43])
-
-def name_to_ipv6(addr):
-    if len(addr)>6 and addr.endswith('.onion'):
+class BIP155Network(Enum):
+    IPV4 = 1
+    IPV6 = 2
+    TORV2 = 3  # no longer supported
+    TORV3 = 4
+    I2P = 5
+    CJDNS = 6
+
+def name_to_bip155(addr):
+    '''Convert address string to BIP155 (networkID, addr) tuple.'''
+    if addr.endswith('.onion'):
         vchAddr = b32decode(addr[0:-6], True)
-        if len(vchAddr) != 16-len(pchOnionCat):
+        if len(vchAddr) == 35:
+            assert vchAddr[34] == 3
+            return (BIP155Network.TORV3, vchAddr[:32])
+        elif len(vchAddr) == 10:
+            return (BIP155Network.TORV2, vchAddr)
+        else:
             raise ValueError('Invalid onion %s' % vchAddr)
-        return pchOnionCat + vchAddr
     elif '.' in addr: # IPv4
-        return pchIPv4 + bytearray((int(x) for x in addr.split('.')))
+        return (BIP155Network.IPV4, bytes((int(x) for x in addr.split('.'))))
     elif ':' in addr: # IPv6
         sub = [[], []] # prefix, suffix
         x = 0
@@ -67,13 +70,12 @@ def name_to_ipv6(addr):
                 sub[x].append(val & 0xff)
         nullbytes = 16 - len(sub[0]) - len(sub[1])
         assert((x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0))
-        return bytearray(sub[0] + ([0] * nullbytes) + sub[1])
-    elif addr.startswith('0x'): # IPv4-in-little-endian
-        return pchIPv4 + bytearray(reversed(a2b_hex(addr[2:])))
+        return (BIP155Network.IPV6, bytes(sub[0] + ([0] * nullbytes) + sub[1]))
     else:
         raise ValueError('Could not parse address %s' % addr)
 
-def parse_spec(s, defaultport):
+def parse_spec(s):
+    '''Convert endpoint string to BIP155 (networkID, addr, port) tuple.'''
     match = re.match(r'\[([0-9a-fA-F:]+)\](?::([0-9]+))?$', s)
     if match: # ipv6
         host = match.group(1)
@@ -85,32 +87,57 @@ def parse_spec(s, defaultport):
         (host,_,port) = s.partition(':')
 
     if not port:
-        port = defaultport
+        port = 0
     else:
         port = int(port)
 
-    host = name_to_ipv6(host)
-
-    return (host,port)
+    host = name_to_bip155(host)
 
-def process_nodes(g, f, structname, defaultport):
-    g.write('static SeedSpec6 %s[] = {\n' % structname)
-    first = True
+    if host[0] == BIP155Network.TORV2:
+        return None  # TORV2 is no longer supported, so we ignore it
+    else:
+        return host + (port, )
+
+def ser_compact_size(l):
+    r = b""
+    if l < 253:
+        r = struct.pack("B", l)
+    elif l < 0x10000:
+        r = struct.pack("<BH", 253, l)
+    elif l < 0x100000000:
+        r = struct.pack("<BI", 254, l)
+    else:
+        r = struct.pack("<BQ", 255, l)
+    return r
+
+def bip155_serialize(spec):
+    '''
+    Serialize (networkID, addr, port) tuple to BIP155 binary format.
+    '''
+    r = b""
+    r += struct.pack('B', spec[0].value)
+    r += ser_compact_size(len(spec[1]))
+    r += spec[1]
+    r += struct.pack('>H', spec[2])
+    return r
+
+def process_nodes(g, f, structname):
+    g.write('static const uint8_t %s[] = {\n' % structname)
     for line in f:
         comment = line.find('#')
         if comment != -1:
             line = line[0:comment]
         line = line.strip()
         if not line:
             continue
-        if not first:
-            g.write(',\n')
-        first = False
 
-        (host,port) = parse_spec(line, defaultport)
-        hoststr = ','.join(('0x%02x' % b) for b in host)
-        g.write('    {{%s}, %i}' % (hoststr, port))
-    g.write('\n};\n')
+        spec = parse_spec(line)
+        if spec is None:  # ignore this entry (e.g. no longer supported addresses like TORV2)
+            continue
+        blob = bip155_serialize(spec)
+        hoststr = ','.join(('0x%02x' % b) for b in blob)
+        g.write(f'    {hoststr},\n')
+    g.write('};\n')
 
 def main():
     if len(sys.argv)<2:
@@ -124,14 +151,13 @@ def main():
     g.write(' * List of fixed seed nodes for the dash network\n')
     g.write(' * AUTOGENERATED by contrib/seeds/generate-seeds.py\n')
     g.write(' *\n')
-    g.write(' * Each line contains a 16-byte IPv6 address and a port.\n')
-    g.write(' * IPv4 as well as onion addresses are wrapped inside an IPv6 address accordingly.\n')
+    g.write(' * Each line contains a BIP155 serialized (networkID, addr, port) tuple.\n')
     g.write(' */\n')
     with open(os.path.join(indir,'nodes_main.txt'), 'r', encoding="utf8") as f:
-        process_nodes(g, f, 'pnSeed6_main', 9999)
+        process_nodes(g, f, 'chainparams_seed_main')
     g.write('\n')
     with open(os.path.join(indir,'nodes_test.txt'), 'r', encoding="utf8") as f:
-        process_nodes(g, f, 'pnSeed6_test', 19999)
+        process_nodes(g, f, 'chainparams_seed_test')
     g.write('#endif // BITCOIN_CHAINPARAMSSEEDS_H\n')
 
 if __name__ == '__main__':

doc/files.md

@@ -53,9 +53,10 @@ Subdirectory       | File(s)               | Description
 `indexes/blockfilter/basic/db/` | LevelDB database      | Blockfilter index LevelDB database for the basic filtertype; *optional*, used if `-blockfilterindex=basic`
 `indexes/blockfilter/basic/`    | `fltrNNNNN.dat`<sup>[\[2\]](#note2)</sup> | Blockfilter index filters for the basic filtertype; *optional*, used if `-blockfilterindex=basic`
 `wallets/`         |                       | [Contains wallets](#multi-wallet-environment); can be specified by `-walletdir` option; if `wallets/` subdirectory does not exist, a wallet resides in the data directory
+`./`               | `anchors.dat`         | Anchor IP address database, created on shutdown and deleted at startup. Anchors are last known outgoing block-relay-only peers that are tried to re-connect to on startup
 `evodb/`         |                       |special txes and quorums database
 `llmq/`          |                       |quorum signatures database
-`./`               | `banlist.dat`         | Stores the IPs/subnets of banned nodes
+`./`               | `banlist.json`        | Stores the addresses/subnets of banned nodes.
 `./`               | `dash.conf`        | User-defined [configuration settings](dash-conf.md) for `dashd` or `dash-qt`. File is not written to by the software and must be created manually. Path can be specified by `-conf` option
 `./`               | `dashd.pid`        | Stores the process ID (PID) of `dashd` or `dash-qt` while running; created at start and deleted on shutdown; can be specified by `-pid` option
 `./`               | `debug.log`           | Contains debug information and general logging generated by `dashd` or `dash-qt`; can be specified by `-debuglogfile` option
@@ -110,10 +111,11 @@ Subdirectory | File                 | Description
 
 ## Legacy subdirectories and files
 
-These subdirectories and files are no longer used by the Dash Core:
+These subdirectories and files are no longer used by Dash Core:
 
 Path           | Description | Repository notes
 ---------------|-------------|-----------------
+`banlist.dat`  | Stores the addresses/subnets of banned nodes; completely ignored and superseded by `banlist.json` in 20.0 | [PR #5574](https://github.com/dashpay/dash/pull/5574)
 `blktree/`     | Blockchain index; replaced by `blocks/index/` in [0.8.0](https://github.com/dash/dash/blob/master/doc/release-notes/release-notes-0.8.0.md#improvements) | [PR #2231](https://github.com/dash/dash/pull/2231), [`8fdc94cc`](https://github.com/dash/dash/commit/8fdc94cc8f0341e96b1edb3a5b56811c0b20bd15)
 `coins/`       | Unspent transaction output database; replaced by `chainstate/` in 0.8.0 | [PR #2231](https://github.com/dash/dash/pull/2231), [`8fdc94cc`](https://github.com/dash/dash/commit/8fdc94cc8f0341e96b1edb3a5b56811c0b20bd15)
 `blkindex.dat` | Blockchain index BDB database; replaced by {`chainstate/`, `blocks/index/`, `blocks/revNNNNN.dat`<sup>[\[2\]](#note2)</sup>} in 0.8.0 | [PR #1677](https://github.com/dash/dash/pull/1677)