Merge branch 'auxpow'

.github/workflows/ci.yml

@@ -40,7 +40,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Clang version
         run: clang --version
@@ -90,7 +90,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Fix Visual Studio installation
         # See: https://github.com/actions/runner-images/issues/7832#issuecomment-1617585694.

ci/test/00_setup_env_native_asan.sh

@@ -22,4 +22,5 @@ export NO_DEPENDS=1
 export GOAL="install"
 export BITCOIN_CONFIG="--enable-c++20 --enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 \
 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' \
---with-sanitizers=address,float-divide-by-zero,integer,undefined CC=clang-16 CXX=clang++-16"
+--with-sanitizers=address,float-divide-by-zero,integer,undefined \
+CC='clang-16 -ftrivial-auto-var-init=pattern' CXX='clang++-16 -ftrivial-auto-var-init=pattern'"

ci/test/06_script_b.sh

@@ -29,6 +29,29 @@ df -h
 # Tests that run natively guess the host
 export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")}
 
+(
+  # compact->outputs[i].file_size is uninitialized memory, so reading it is UB.
+  # The statistic bytes_written is only used for logging, which is disabled in
+  # CI, so as a temporary minimal fix to work around UB and CI failures, leave
+  # bytes_written unmodified.
+  # See https://github.com/bitcoin/bitcoin/pull/28359#issuecomment-1698694748
+  echo 'diff --git a/src/leveldb/db/db_impl.cc b/src/leveldb/db/db_impl.cc
+index 65e31724bc..f61b471953 100644
+--- a/src/leveldb/db/db_impl.cc
++++ b/src/leveldb/db/db_impl.cc
+@@ -1028,9 +1028,6 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
+       stats.bytes_read += compact->compaction->input(which, i)->file_size;
+     }
+   }
+-  for (size_t i = 0; i < compact->outputs.size(); i++) {
+-    stats.bytes_written += compact->outputs[i].file_size;
+-  }
+
+   mutex_.Lock();
+   stats_[compact->compaction->level() + 1].Add(stats);' | patch -p1
+  git diff
+)
+
 if [ "$RUN_FUZZ_TESTS" = "true" ]; then
   export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/
   if [ ! -d "$DIR_FUZZ_IN" ]; then

configure.ac

@@ -725,6 +725,10 @@ case $host in
 
      dnl We require Windows 7 (NT 6.1) or later
      AX_CHECK_LINK_FLAG([-Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1"], [], [$LDFLAG_WERROR])
+
+     dnl Avoid the use of aligned vector instructions when building for Windows.
+     dnl See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54412.
+     AX_CHECK_COMPILE_FLAG([-Wa,-muse-unaligned-vector-move], [CORE_CXXFLAGS="$CORE_CXXFLAGS -Wa,-muse-unaligned-vector-move"], [], [$CXXFLAG_WERROR])
      ;;
   *darwin*)
      TARGET_OS=darwin

contrib/guix/INSTALL.md

@@ -578,7 +578,7 @@ sudo --login guix pull --commit=<particular-commit>
 ```
 
 `guix pull` is quite a long process (especially if you're using
-`--no-substitute`). If you encounter build problems, please refer to the
+`--no-substitutes`). If you encounter build problems, please refer to the
 [troubleshooting section](#troubleshooting).
 
 Note that running a bare `guix pull` with no commit or branch specified will
@@ -616,7 +616,7 @@ systemctl enable guix-daemon
 systemctl start guix-daemon
 ```
 
-Remember to set `--no-substitute` in `$libdir/systemd/system/guix-daemon.service` and other customizations if you used them for `guix-daemon-original.service`.
+Remember to set `--no-substitutes` in `$libdir/systemd/system/guix-daemon.service` and other customizations if you used them for `guix-daemon-original.service`.
 
 ##### If you installed Guix via the Debian/Ubuntu distribution packages
 

depends/packages/native_cctools.mk

@@ -1,11 +1,10 @@
 package=native_cctools
-$(package)_version=2ef2e931cf641547eb8a68cfebde61003587c9fd
+$(package)_version=c74fafe86076713cb8e6f937af43b6df6da1f42d
 $(package)_download_path=https://github.com/tpoechtrager/cctools-port/archive
 $(package)_file_name=$($(package)_version).tar.gz
-$(package)_sha256_hash=6b73269efdf5c58a070e7357b66ee760501388549d6a12b423723f45888b074b
+$(package)_sha256_hash=e2c1588d505a69c32e079f4e616e0f117d5478429040e394f624f43f2796e6bc
 $(package)_build_subdir=cctools
 $(package)_dependencies=native_libtapi
-$(package)_patches=no_fixup_chains.patch
 
 define $(package)_set_vars
   $(package)_config_opts=--target=$(host) --enable-lto-support
@@ -18,15 +17,9 @@ endef
 ifneq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 define $(package)_preprocess_cmds
   mkdir -p $($(package)_staging_prefix_dir)/lib && \
-  cp $(llvm_lib_dir)/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \
-  cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub cctools && \
-  patch -p1 < $($(package)_patch_dir)/no_fixup_chains.patch
+  cp $(llvm_lib_dir)/libLTO.so $($(package)_staging_prefix_dir)/lib/
 endef
 else
-define $(package)_preprocess_cmds
-  cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub cctools && \
-  patch -p1 < $($(package)_patch_dir)/no_fixup_chains.patch
-endef
 endif
 
 define $(package)_config_cmds

depends/packages/native_libtapi.mk

@@ -1,13 +1,18 @@
 package=native_libtapi
-$(package)_version=664b8414f89612f2dfd35a9b679c345aa5389026
+$(package)_version=eb33a59f2e30ff9724dc1ea8bee8b5229b0557c9
 $(package)_download_path=https://github.com/tpoechtrager/apple-libtapi/archive
 $(package)_file_name=$($(package)_version).tar.gz
-$(package)_sha256_hash=62e419c12d1c9fad67cc1cd523132bc00db050998337c734c15bc8d73cc02b61
+$(package)_sha256_hash=d4d46c64622f13d6938cecf989046d9561011bb59e8ee835f8f39825d67f578f
+$(package)_patches=disable_zlib.patch
 
 ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 $(package)_dependencies=native_clang
 endif
 
+define $(package)_preprocess_cmds
+  patch -p1 < $($(package)_patch_dir)/disable_zlib.patch
+endef
+
 define $(package)_build_cmds
   CC=$(clang_prog) CXX=$(clangxx_prog) INSTALLPREFIX=$($(package)_staging_prefix_dir) ./build.sh
 endef

depends/patches/native_libtapi/disable_zlib.patch

@@ -0,0 +1,17 @@
+build: disable zlib
+
+This isn't needed, and causes issues when clang-tblgen
+is built, but trys to reach for a system libz.so.
+
+diff --git a/build.sh b/build.sh
+index e25d2f732..ec8422621 100755
+--- a/build.sh
++++ b/build.sh
+@@ -66,6 +66,7 @@ cmake ../src/llvm \
+  -DCMAKE_INSTALL_PREFIX=$INSTALLPREFIX \
+  -DTAPI_REPOSITORY_STRING=$TAPI_VERSION \
+  -DTAPI_FULL_VERSION=$TAPI_VERSION \
++ -DLLVM_ENABLE_ZLIB=OFF \
+  $CMAKE_EXTRA_ARGS
+
+ echo ""

src/addrdb.cpp

@@ -47,16 +47,16 @@ bool SerializeDB(Stream& stream, const Data& data)
 }
 
 template <typename Data>
-bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data& data, int version)
+bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data& data)
 {
     // Generate random temporary filename
     const uint16_t randv{GetRand<uint16_t>()};
     std::string tmpfn = strprintf("%s.%04x", prefix, randv);
 
-    // open temp output file, and associate with CAutoFile
+    // open temp output file
     fs::path pathTmp = gArgs.GetDataDirNet() / fs::u8path(tmpfn);
     FILE *file = fsbridge::fopen(pathTmp, "wb");
-    CAutoFile fileout(file, SER_DISK, version);
+    AutoFile fileout{file};
     if (fileout.IsNull()) {
         fileout.fclose();
         remove(pathTmp);
@@ -86,9 +86,9 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data
 }
 
 template <typename Stream, typename Data>
-void DeserializeDB(Stream& stream, Data& data, bool fCheckSum = true)
+void DeserializeDB(Stream& stream, Data&& data, bool fCheckSum = true)
 {
-    CHashVerifier<Stream> verifier(&stream);
+    HashVerifier verifier{stream};
     // de-serialize file header (network specific magic number) and ..
     unsigned char pchMsgTmp[4];
     verifier >> pchMsgTmp;
@@ -111,11 +111,10 @@ void DeserializeDB(Stream& stream, Data& data, bool fCheckSum = true)
 }
 
 template <typename Data>
-void DeserializeFileDB(const fs::path& path, Data& data, int version)
+void DeserializeFileDB(const fs::path& path, Data&& data)
 {
-    // open input file, and associate with CAutoFile
     FILE* file = fsbridge::fopen(path, "rb");
-    CAutoFile filein(file, SER_DISK, version);
+    AutoFile filein{file};
     if (filein.IsNull()) {
         throw DbNotFoundError{};
     }
@@ -175,10 +174,10 @@ bool CBanDB::Read(banmap_t& banSet)
 bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr)
 {
     const auto pathAddr = args.GetDataDirNet() / "peers.dat";
-    return SerializeFileDB("peers", pathAddr, addr, CLIENT_VERSION);
+    return SerializeFileDB("peers", pathAddr, addr);
 }
 
-void ReadFromStream(AddrMan& addr, CDataStream& ssPeers)
+void ReadFromStream(AddrMan& addr, DataStream& ssPeers)
 {
     DeserializeDB(ssPeers, addr, false);
 }
@@ -191,7 +190,7 @@ util::Result<std::unique_ptr<AddrMan>> LoadAddrman(const NetGroupManager& netgro
     const auto start{SteadyClock::now()};
     const auto path_addr{args.GetDataDirNet() / "peers.dat"};
     try {
-        DeserializeFileDB(path_addr, *addrman, CLIENT_VERSION);
+        DeserializeFileDB(path_addr, *addrman);
         LogPrintf("Loaded %i addresses from peers.dat  %dms\n", addrman->Size(), Ticks<std::chrono::milliseconds>(SteadyClock::now() - start));
     } catch (const DbNotFoundError&) {
         // Addrman can be in an inconsistent state after failure, reset it
@@ -217,14 +216,14 @@ util::Result<std::unique_ptr<AddrMan>> LoadAddrman(const NetGroupManager& netgro
 void DumpAnchors(const fs::path& anchors_db_path, const std::vector<CAddress>& anchors)
 {
     LOG_TIME_SECONDS(strprintf("Flush %d outbound block-relay-only peer addresses to anchors.dat", anchors.size()));
-    SerializeFileDB("anchors", anchors_db_path, anchors, CLIENT_VERSION | ADDRV2_FORMAT);
+    SerializeFileDB("anchors", anchors_db_path, WithParams(CAddress::V2_DISK, anchors));
 }
 
 std::vector<CAddress> ReadAnchors(const fs::path& anchors_db_path)
 {
     std::vector<CAddress> anchors;
     try {
-        DeserializeFileDB(anchors_db_path, anchors, CLIENT_VERSION | ADDRV2_FORMAT);
+        DeserializeFileDB(anchors_db_path, WithParams(CAddress::V2_DISK, anchors));
         LogPrintf("Loaded %i addresses from %s\n", anchors.size(), fs::quoted(fs::PathToString(anchors_db_path.filename())));
     } catch (const std::exception&) {
         anchors.clear();

src/addrdb.h

@@ -16,12 +16,13 @@
 class ArgsManager;
 class AddrMan;
 class CAddress;
-class CDataStream;
+class DataStream;
 class NetGroupManager;
 
-bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr);
 /** Only used by tests. */
-void ReadFromStream(AddrMan& addr, CDataStream& ssPeers);
+void ReadFromStream(AddrMan& addr, DataStream& ssPeers);
+
+bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr);
 
 /** Access to the banlist database (banlist.json) */
 class CBanDB

src/addrman.cpp

@@ -171,8 +171,7 @@ void AddrManImpl::Serialize(Stream& s_) const
      */
 
     // Always serialize in the latest version (FILE_FORMAT).
-
-    OverrideStream<Stream> s(&s_, s_.GetType(), s_.GetVersion() | ADDRV2_FORMAT);
+    ParamsStream s{CAddress::V2_DISK, s_};
 
     s << static_cast<uint8_t>(FILE_FORMAT);
 
@@ -236,14 +235,8 @@ void AddrManImpl::Unserialize(Stream& s_)
     Format format;
     s_ >> Using<CustomUintFormatter<1>>(format);
 
-    int stream_version = s_.GetVersion();
-    if (format >= Format::V3_BIP155) {
-        // Add ADDRV2_FORMAT to the version so that the CNetAddr and CAddress
-        // unserialize methods know that an address in addrv2 format is coming.
-        stream_version |= ADDRV2_FORMAT;
-    }
-
-    OverrideStream<Stream> s(&s_, s_.GetType(), stream_version);
+    const auto ser_params = (format >= Format::V3_BIP155 ? CAddress::V2_DISK : CAddress::V1_DISK);
+    ParamsStream s{ser_params, s_};
 
     uint8_t compat;
     s >> compat;
@@ -1249,12 +1242,12 @@ void AddrMan::Unserialize(Stream& s_)
 }
 
 // explicit instantiation
-template void AddrMan::Serialize(HashedSourceWriter<CAutoFile>& s) const;
-template void AddrMan::Serialize(CDataStream& s) const;
-template void AddrMan::Unserialize(CAutoFile& s);
-template void AddrMan::Unserialize(CHashVerifier<CAutoFile>& s);
-template void AddrMan::Unserialize(CDataStream& s);
-template void AddrMan::Unserialize(CHashVerifier<CDataStream>& s);
+template void AddrMan::Serialize(HashedSourceWriter<AutoFile>&) const;
+template void AddrMan::Serialize(DataStream&) const;
+template void AddrMan::Unserialize(AutoFile&);
+template void AddrMan::Unserialize(HashVerifier<AutoFile>&);
+template void AddrMan::Unserialize(DataStream&);
+template void AddrMan::Unserialize(HashVerifier<DataStream>&);
 
 size_t AddrMan::Size(std::optional<Network> net, std::optional<bool> in_new) const
 {

src/addrman_impl.h

@@ -65,8 +65,7 @@ class AddrInfo : public CAddress
 
     SERIALIZE_METHODS(AddrInfo, obj)
     {
-        READWRITEAS(CAddress, obj);
-        READWRITE(obj.source, Using<ChronoFormatter<int64_t>>(obj.m_last_success), obj.nAttempts);
+        READWRITE(AsBase<CAddress>(obj), obj.source, Using<ChronoFormatter<int64_t>>(obj.m_last_success), obj.nAttempts);
     }
 
     AddrInfo(const CAddress &addrIn, const CNetAddr &addrSource) : CAddress(addrIn), source(addrSource)

src/chain.h

@@ -402,6 +402,14 @@ const CBlockIndex* LastCommonAncestor(const CBlockIndex* pa, const CBlockIndex*
 /** Used to marshal pointers into hashes for db storage. */
 class CDiskBlockIndex : public CBlockIndex
 {
+    /** Historically CBlockLocator's version field has been written to disk
+     * streams as the client version, but the value has never been used.
+     *
+     * Hard-code to the highest client version ever written.
+     * SerParams can be used if the field requires any meaning in the future.
+     **/
+    static constexpr int DUMMY_VERSION = 259900;
+
 public:
     uint256 hashPrev;
 
@@ -418,8 +426,8 @@ class CDiskBlockIndex : public CBlockIndex
     SERIALIZE_METHODS(CDiskBlockIndex, obj)
     {
         LOCK(::cs_main);
-        int _nVersion = s.GetVersion();
-        if (!(s.GetType() & SER_GETHASH)) READWRITE(VARINT_MODE(_nVersion, VarIntMode::NONNEGATIVE_SIGNED));
+        int _nVersion = DUMMY_VERSION;
+        READWRITE(VARINT_MODE(_nVersion, VarIntMode::NONNEGATIVE_SIGNED));
 
         READWRITE(VARINT_MODE(obj.nHeight, VarIntMode::NONNEGATIVE_SIGNED));
         READWRITE(VARINT(obj.nStatus));

src/dbwrapper.h

@@ -6,7 +6,6 @@
 #define BITCOIN_DBWRAPPER_H
 
 #include <attributes.h>
-#include <clientversion.h>
 #include <serialize.h>
 #include <span.h>
 #include <streams.h>
@@ -180,7 +179,7 @@ class CDBIterator
 
     template<typename V> bool GetValue(V& value) {
         try {
-            CDataStream ssValue{GetValueImpl(), SER_DISK, CLIENT_VERSION};
+            DataStream ssValue{GetValueImpl()};
             ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
             ssValue >> value;
         } catch (const std::exception&) {
@@ -242,7 +241,7 @@ class CDBWrapper
             return false;
         }
         try {
-            CDataStream ssValue{MakeByteSpan(*strValue), SER_DISK, CLIENT_VERSION};
+            DataStream ssValue{MakeByteSpan(*strValue)};
             ssValue.Xor(obfuscate_key);
             ssValue >> value;
         } catch (const std::exception&) {