Merge branch 'auxpow'

.github/workflows/ci.yml

@@ -104,8 +104,8 @@ jobs:
           HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1
         run: |
           # A workaround for "The `brew link` step did not complete successfully" error.
-          brew install python@3 || brew link --overwrite python@3
-          brew install automake libtool pkg-config gnu-getopt ccache boost libevent miniupnpc libnatpmp zeromq qt@5 qrencode
+          brew install --quiet python@3 || brew link --overwrite python@3
+          brew install --quiet automake libtool pkg-config gnu-getopt ccache boost libevent miniupnpc libnatpmp zeromq qt@5 qrencode
 
       - name: Set Ccache directory
         run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV"

ci/lint/04_install.sh

@@ -53,6 +53,7 @@ ${CI_RETRY_EXE} pip3 install \
   lief==0.13.2 \
   mypy==1.4.1 \
   pyzmq==25.1.0 \
+  ruff==0.5.5 \
   vulture==2.6
 
 SHELLCHECK_VERSION=v0.8.0

contrib/asmap/asmap-tool.py

@@ -6,7 +6,9 @@
 import argparse
 import sys
 import ipaddress
+import json
 import math
+from collections import defaultdict
 
 import asmap
 
@@ -113,6 +115,18 @@ def main():
     parser_diff.add_argument('infile2', type=argparse.FileType('rb'),
                              help="second file to compare (text or binary)")
 
+    parser_diff_addrs = subparsers.add_parser("diff_addrs",
+                                              help="compute difference between two asmap files for a set of addresses")
+    parser_diff_addrs.add_argument('-s', '--show-addresses', dest="show_addresses", default=False, action="store_true",
+                                   help="include reassigned addresses in the output")
+    parser_diff_addrs.add_argument("infile1", type=argparse.FileType("rb"),
+                                   help="first file to compare (text or binary)")
+    parser_diff_addrs.add_argument("infile2", type=argparse.FileType("rb"),
+                                   help="second file to compare (text or binary)")
+    parser_diff_addrs.add_argument("addrs_file", type=argparse.FileType("r"),
+                                   help="address file containing getnodeaddresses output to use in the comparison "
+                                   "(make sure to set the count parameter to zero to get all node addresses, "
+                                   "e.g. 'bitcoin-cli getnodeaddresses 0 > addrs.json')")
     args = parser.parse_args()
     if args.subcommand is None:
         parser.print_help()
@@ -148,6 +162,33 @@ def main():
             f"# {ipv4_changed}{ipv4_change_str} IPv4 addresses changed; "
             f"{ipv6_changed}{ipv6_change_str} IPv6 addresses changed"
         )
+    elif args.subcommand == "diff_addrs":
+        state1 = load_file(args.infile1)
+        state2 = load_file(args.infile2)
+        address_info = json.load(args.addrs_file)
+        addrs = {a["address"] for a in address_info if a["network"] in ["ipv4", "ipv6"]}
+        reassignments = defaultdict(list)
+        for addr in addrs:
+            net = ipaddress.ip_network(addr)
+            prefix = asmap.net_to_prefix(net)
+            old_asn = state1.lookup(prefix)
+            new_asn = state2.lookup(prefix)
+            if new_asn != old_asn:
+                reassignments[(old_asn, new_asn)].append(addr)
+        reassignments = sorted(reassignments.items(), key=lambda item: len(item[1]), reverse=True)
+        num_reassignment_type = defaultdict(int)
+        for (old_asn, new_asn), reassigned_addrs in reassignments:
+            num_reassigned = len(reassigned_addrs)
+            num_reassignment_type[(bool(old_asn), bool(new_asn))] += num_reassigned
+            old_asn_str = f"AS{old_asn}" if old_asn else "unassigned"
+            new_asn_str = f"AS{new_asn}" if new_asn else "unassigned"
+            opt = ": " + ", ".join(reassigned_addrs) if args.show_addresses else ""
+            print(f"{num_reassigned} address(es) reassigned from {old_asn_str} to {new_asn_str}{opt}")
+        num_reassignments = sum(len(addrs) for _, addrs in reassignments)
+        share = num_reassignments / len(addrs) if len(addrs) > 0 else 0
+        print(f"Summary: {num_reassignments:,} ({share:.2%}) of {len(addrs):,} addresses were reassigned "
+              f"(migrations={num_reassignment_type[True, True]}, assignments={num_reassignment_type[False, True]}, "
+              f"unassignments={num_reassignment_type[True, False]})")
     else:
         parser.print_help()
         sys.exit("No command provided.")

contrib/guix/INSTALL.md

@@ -671,6 +671,8 @@ More information: https://github.com/python/cpython/issues/81765
 OpenSSL includes tests that will fail once some certificate has expired.
 The workarounds from the GnuTLS section immediately below can be used.
 
+For openssl-1.1.1l use 2022-05-01 as the date.
+
 ### GnuTLS: test-suite FAIL: status-request-revoked
 
 *The derivation is likely identified by: `/gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv`*
@@ -705,11 +707,12 @@ authorized.
 This workaround was described [here](https://issues.guix.gnu.org/44559#5).
 
 Basically:
-2. Turn off NTP
-3. Set system time to 2020-10-01
-4. guix build --no-substitutes /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv
-5. Set system time back to accurate current time
-6. Turn NTP back on
+
+1. Turn off NTP
+2. Set system time to 2020-10-01
+3. guix build --no-substitutes /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv
+4. Set system time back to accurate current time
+5. Turn NTP back on
 
 For example,
 

contrib/linearize/linearize-data.py

@@ -76,6 +76,16 @@ def getFirstBlockFileId(block_dir_path):
     blkId = int(firstBlkFn[3:8])
     return blkId
 
+def read_xor_key(blocks_path):
+    NUM_XOR_BYTES = 8  # From InitBlocksdirXorKey::xor_key.size()
+    try:
+        xor_filename = os.path.join(blocks_path, "xor.dat")
+        with open(xor_filename, "rb") as xor_file:
+            return xor_file.read(NUM_XOR_BYTES)
+    # support also blockdirs created with pre-v28 versions, where no xor key exists yet
+    except FileNotFoundError:
+        return bytes([0] * NUM_XOR_BYTES)
+
 # Block header and extent on disk
 BlockExtent = namedtuple('BlockExtent', ['fn', 'offset', 'inhdr', 'blkhdr', 'size'])
 
@@ -95,6 +105,7 @@ def __init__(self, settings, blkindex, blkmap):
         self.outFname = None
         self.blkCountIn = 0
         self.blkCountOut = 0
+        self.xor_key = read_xor_key(self.settings['input'])
 
         self.lastDate = datetime.datetime(2000, 1, 1)
         self.highTS = 1408893517 - 315360000
@@ -113,6 +124,13 @@ def __init__(self, settings, blkindex, blkmap):
         self.outOfOrderData = {}
         self.outOfOrderSize = 0 # running total size for items in outOfOrderData
 
+    def read_xored(self, f, size):
+        offset = f.tell()
+        data = bytearray(f.read(size))
+        for i in range(len(data)):
+            data[i] ^= self.xor_key[(i + offset) % len(self.xor_key)]
+        return bytes(data)
+
     def writeBlock(self, inhdr, blk_hdr, rawblock):
         blockSizeOnDisk = len(inhdr) + len(blk_hdr) + len(rawblock)
         if not self.fileOutput and ((self.outsz + blockSizeOnDisk) > self.maxOutSz):
@@ -165,7 +183,7 @@ def fetchBlock(self, extent):
         '''Fetch block contents from disk given extents'''
         with open(self.inFileName(extent.fn), "rb") as f:
             f.seek(extent.offset)
-            return f.read(extent.size)
+            return self.read_xored(f, extent.size)
 
     def copyOneBlock(self):
         '''Find the next block to be written in the input, and copy it to the output.'''
@@ -190,7 +208,7 @@ def run(self):
                     print("Premature end of block data")
                     return
 
-            inhdr = self.inF.read(8)
+            inhdr = self.read_xored(self.inF, 8)
             if (not inhdr or (inhdr[0] == "\0")):
                 self.inF.close()
                 self.inF = None
@@ -207,7 +225,7 @@ def run(self):
             inLenLE = inhdr[4:]
             su = struct.unpack("<I", inLenLE)
             inLen = su[0] - 80 # length without header
-            blk_hdr = self.inF.read(80)
+            blk_hdr = self.read_xored(self.inF, 80)
             inExtent = BlockExtent(self.inFn, self.inF.tell(), inhdr, blk_hdr, inLen)
 
             self.hash_str = calc_hash_str(blk_hdr)
@@ -224,7 +242,7 @@ def run(self):
 
             if self.blkCountOut == blkHeight:
                 # If in-order block, just copy
-                rawblock = self.inF.read(inLen)
+                rawblock = self.read_xored(self.inF, inLen)
                 self.writeBlock(inhdr, blk_hdr, rawblock)
 
                 # See if we can catch up to prior out-of-order blocks
@@ -237,7 +255,7 @@ def run(self):
                     # If there is space in the cache, read the data
                     # Reading the data in file sequence instead of seeking and fetching it later is preferred,
                     # but we don't want to fill up memory
-                    self.outOfOrderData[blkHeight] = self.inF.read(inLen)
+                    self.outOfOrderData[blkHeight] = self.read_xored(self.inF, inLen)
                     self.outOfOrderSize += inLen
                 else: # If no space in cache, seek forward
                     self.inF.seek(inLen, os.SEEK_CUR)

depends/packages/miniupnpc.mk

@@ -1,6 +1,6 @@
 package=miniupnpc
 $(package)_version=2.2.7
-$(package)_download_path=http://miniupnp.free.fr/files/
+$(package)_download_path=https://miniupnp.tuxfamily.org/files/
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
 $(package)_sha256_hash=b0c3a27056840fd0ec9328a5a9bac3dc5e0ec6d2e8733349cf577b0aa1e70ac1
 $(package)_patches=dont_leak_info.patch cmake_get_src_addr.patch fix_windows_snprintf.patch

doc/bips.md

@@ -30,6 +30,7 @@ BIPs that are implemented by Bitcoin Core:
 * [`BIP 84`](https://github.com/bitcoin/bips/blob/master/bip-0084.mediawiki): The experimental descriptor wallets introduced in **v0.21.0** by default use the Hierarchical Deterministic Wallet derivation proposed by BIP 84. ([PR #16528](https://github.com/bitcoin/bitcoin/pull/16528))
 * [`BIP 86`](https://github.com/bitcoin/bips/blob/master/bip-0086.mediawiki): Descriptor wallets by default use the Hierarchical Deterministic Wallet derivation proposed by BIP 86 since **v23.0** ([PR #22364](https://github.com/bitcoin/bitcoin/pull/22364)).
 * [`BIP 90`](https://github.com/bitcoin/bips/blob/master/bip-0090.mediawiki): Trigger mechanism for activation of BIPs 34, 65, and 66 has been simplified to block height checks since **v0.14.0** ([PR #8391](https://github.com/bitcoin/bitcoin/pull/8391)).
+* [`BIP 94`](https://github.com/bitcoin/bips/blob/master/bip-0094.mediawiki): Testnet 4 (`-testnet4`) supported as of **v28.0** ([PR #29775](https://github.com/bitcoin/bitcoin/pull/29775)).
 * [`BIP 111`](https://github.com/bitcoin/bips/blob/master/bip-0111.mediawiki): `NODE_BLOOM` service bit added, and enforced for all peer versions as of **v0.13.0** ([PR #6579](https://github.com/bitcoin/bitcoin/pull/6579) and [PR #6641](https://github.com/bitcoin/bitcoin/pull/6641)).
 * [`BIP 112`](https://github.com/bitcoin/bips/blob/master/bip-0112.mediawiki): The CHECKSEQUENCEVERIFY opcode has been implemented since **v0.12.1** ([PR #7524](https://github.com/bitcoin/bitcoin/pull/7524)), and has been *buried* since **v0.19.0** ([PR #16060](https://github.com/bitcoin/bitcoin/pull/16060)).
 * [`BIP 113`](https://github.com/bitcoin/bips/blob/master/bip-0113.mediawiki): Median time past lock-time calculations have been implemented since **v0.12.1** ([PR #6566](https://github.com/bitcoin/bitcoin/pull/6566)), and has been *buried* since **v0.19.0** ([PR #16060](https://github.com/bitcoin/bitcoin/pull/16060)).

doc/build-windows.md

@@ -13,6 +13,9 @@ Other options which may work, but which have not been extensively tested are (pl
 
 * On Windows, using a POSIX compatibility layer application such as [cygwin](https://www.cygwin.com/) or [msys2](https://www.msys2.org/).
 
+The instructions below work on Ubuntu and Debian. Make sure the distribution's `g++-mingw-w64-x86-64-posix`
+package meets the minimum required `g++` version specified in [dependencies.md](dependencies.md).
+
 Installing Windows Subsystem for Linux
 ---------------------------------------
 
@@ -66,7 +69,6 @@ is to temporarily disable WSL support for Win32 applications.
 
 Build using:
 
-    PATH=$(echo "$PATH" | sed -e 's/:\/mnt.*//g') # strip out problematic Windows %PATH% imported var
     sudo bash -c "echo 0 > /proc/sys/fs/binfmt_misc/status" # Disable WSL support for Win32 applications.
     cd depends
     make HOST=x86_64-w64-mingw32

doc/productivity.md

@@ -33,7 +33,7 @@ The easiest way to faster compile times is to cache compiles. `ccache` is a way
 
 Install `ccache` through your distribution's package manager, and run `./configure` with your normal flags to pick it up.
 
-To use ccache for all your C/C++ projects, follow the symlinks method [here](https://ccache.samba.org/manual/latest.html#_run_modes) to set it up.
+To use ccache for all your C/C++ projects, follow the symlinks method [here](https://ccache.dev/manual/latest.html#_run_modes) to set it up.
 
 To get the most out of ccache, put something like this in `~/.ccache/ccache.conf`:
 

src/Makefile.test.include

@@ -305,6 +305,7 @@ test_fuzz_fuzz_SOURCES = \
  test/fuzz/bitset.cpp \
  test/fuzz/block.cpp \
  test/fuzz/block_header.cpp \
+ test/fuzz/block_index.cpp \
  test/fuzz/blockfilter.cpp \
  test/fuzz/bloom_filter.cpp \
  test/fuzz/buffered_file.cpp \

src/bench/wallet_create.cpp

@@ -42,7 +42,7 @@ static void WalletCreate(benchmark::Bench& bench, bool encrypted)
 
         // Release wallet
         RemoveWallet(context, wallet, /*load_on_start=*/ std::nullopt);
-        UnloadWallet(std::move(wallet));
+        WaitForDeleteWallet(std::move(wallet));
         fs::remove_all(wallet_path);
     });
 }

src/chainparamsbase.cpp

@@ -13,7 +13,7 @@
 
 void SetupChainParamsBaseOptions(ArgsManager& argsman)
 {
-    argsman.AddArg("-chain=<chain>", "Use the chain <chain> (default: main). Allowed values: main, test, signet, regtest", ArgsManager::ALLOW_ANY, OptionsCategory::CHAINPARAMS);
+    argsman.AddArg("-chain=<chain>", "Use the chain <chain> (default: main). Allowed values: " LIST_CHAIN_NAMES, ArgsManager::ALLOW_ANY, OptionsCategory::CHAINPARAMS);
     argsman.AddArg("-regtest", "Enter regression test mode, which uses a special chain in which blocks can be solved instantly. "
                  "This is intended for regression testing tools and app development. Equivalent to -chain=regtest.", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CHAINPARAMS);
     argsman.AddArg("-testactivationheight=name@height.", "Set the activation height of 'name' (bip16, segwit, bip34, dersig, cltv, csv). (regtest-only)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);

src/chainparamsbase.h

@@ -7,6 +7,7 @@
 
 #include <util/chaintype.h>
 
+#include <cstdint>
 #include <memory>
 #include <string>
 
@@ -52,4 +53,7 @@ const CBaseChainParams& BaseParams();
 /** Sets the params returned by Params() to those for the given chain. */
 void SelectBaseParams(const ChainType chain);
 
+/** List of possible chain / network names  */
+#define LIST_CHAIN_NAMES "main, test, testnet4, signet, regtest"
+
 #endif // BITCOIN_CHAINPARAMSBASE_H

src/coins.cpp

@@ -53,19 +53,18 @@ size_t CCoinsViewCache::DynamicMemoryUsage() const {
 }
 
 CCoinsMap::iterator CCoinsViewCache::FetchCoin(const COutPoint &outpoint) const {
-    CCoinsMap::iterator it = cacheCoins.find(outpoint);
-    if (it != cacheCoins.end())
-        return it;
-    Coin tmp;
-    if (!base->GetCoin(outpoint, tmp))
-        return cacheCoins.end();
-    CCoinsMap::iterator ret = cacheCoins.emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), std::forward_as_tuple(std::move(tmp))).first;
-    if (ret->second.coin.IsSpent()) {
-        // The parent only has an empty entry for this outpoint; we can consider our
-        // version as fresh.
-        ret->second.AddFlags(CCoinsCacheEntry::FRESH, *ret, m_sentinel);
+    const auto [ret, inserted] = cacheCoins.try_emplace(outpoint);
+    if (inserted) {
+        if (!base->GetCoin(outpoint, ret->second.coin)) {
+            cacheCoins.erase(ret);
+            return cacheCoins.end();
+        }
+        if (ret->second.coin.IsSpent()) {
+            // The parent only has an empty entry for this outpoint; we can consider our version as fresh.
+            ret->second.AddFlags(CCoinsCacheEntry::FRESH, *ret, m_sentinel);
+        }
+        cachedCoinsUsage += ret->second.coin.DynamicMemoryUsage();
     }
-    cachedCoinsUsage += ret->second.coin.DynamicMemoryUsage();
     return ret;
 }
 

src/interfaces/wallet.h

@@ -343,8 +343,11 @@ class WalletLoader : public ChainClient
     //! Migrate a wallet
     virtual util::Result<WalletMigrationResult> migrateWallet(const std::string& name, const SecureString& passphrase) = 0;
 
+    //! Returns true if wallet stores encryption keys
+    virtual bool isEncrypted(const std::string& wallet_name) = 0;
+
     //! Return available wallets in wallet directory.
-    virtual std::vector<std::string> listWalletDir() = 0;
+    virtual std::vector<std::pair<std::string, std::string>> listWalletDir() = 0;
 
     //! Return interfaces for accessing wallets (if any).
     virtual std::vector<std::unique_ptr<Wallet>> getWallets() = 0;

src/kernel/chainparams.cpp

@@ -219,7 +219,12 @@ class CMainParams : public CChainParams {
         };
 
         m_assumeutxo_data = {
-            // TODO to be specified in a future patch.
+            {
+                .height = 840'000,
+                .hash_serialized = AssumeutxoHash{uint256{"a2a5521b1b5ab65f67818e5e8eccabb7171a517f9e2382208f77687310768f96"}},
+                .m_chain_tx_count = 991032194,
+                .blockhash = consteval_ctor(uint256{"0000000000000000000320283a032748cef8227873ff4872689bf23f1cda83a5"}),
+            }
         };
 
         chainTxData = ChainTxData{
@@ -823,13 +828,16 @@ std::optional<ChainType> GetNetworkForMagic(const MessageStartChars& message)
 {
     const auto mainnet_msg = CChainParams::Main()->MessageStart();
     const auto testnet_msg = CChainParams::TestNet()->MessageStart();
+    const auto testnet4_msg = CChainParams::TestNet4()->MessageStart();
     const auto regtest_msg = CChainParams::RegTest({})->MessageStart();
     const auto signet_msg = CChainParams::SigNet({})->MessageStart();
 
     if (std::equal(message.begin(), message.end(), mainnet_msg.data())) {
         return ChainType::MAIN;
     } else if (std::equal(message.begin(), message.end(), testnet_msg.data())) {
         return ChainType::TESTNET;
+    } else if (std::equal(message.begin(), message.end(), testnet4_msg.data())) {
+        return ChainType::TESTNET4;
     } else if (std::equal(message.begin(), message.end(), regtest_msg.data())) {
         return ChainType::REGTEST;
     } else if (std::equal(message.begin(), message.end(), signet_msg.data())) {

src/net.cpp

@@ -576,16 +576,14 @@ CService CNode::GetAddrLocal() const
 {
     AssertLockNotHeld(m_addr_local_mutex);
     LOCK(m_addr_local_mutex);
-    return addrLocal;
+    return m_addr_local;
 }
 
 void CNode::SetAddrLocal(const CService& addrLocalIn) {
     AssertLockNotHeld(m_addr_local_mutex);
     LOCK(m_addr_local_mutex);
-    if (addrLocal.IsValid()) {
-        LogError("Addr local already set for node: %i. Refusing to change from %s to %s\n", id, addrLocal.ToStringAddrPort(), addrLocalIn.ToStringAddrPort());
-    } else {
-        addrLocal = addrLocalIn;
+    if (Assume(!m_addr_local.IsValid())) { // Addr local can only be set once during version msg processing
+        m_addr_local = addrLocalIn;
     }
 }
 

src/net.h

@@ -969,7 +969,7 @@ class CNode
     size_t m_msg_process_queue_size GUARDED_BY(m_msg_process_queue_mutex){0};
 
     // Our address, as reported by the peer
-    CService addrLocal GUARDED_BY(m_addr_local_mutex);
+    CService m_addr_local GUARDED_BY(m_addr_local_mutex);
     mutable Mutex m_addr_local_mutex;
 
     mapMsgTypeSize mapSendBytesPerMsgType GUARDED_BY(cs_vSend);

src/node/interface_ui.h

@@ -6,9 +6,10 @@
 #ifndef BITCOIN_NODE_INTERFACE_UI_H
 #define BITCOIN_NODE_INTERFACE_UI_H
 
+#include <cstdint>
 #include <functional>
-#include <memory>
 #include <string>
+#include <vector>
 
 class CBlockIndex;
 enum class SynchronizationState;