From 8f43302a0a1bb79129933e4cc174bf8d8d59ec15 Mon Sep 17 00:00:00 2001 From: fanquake Date: Tue, 12 Sep 2023 14:38:06 +0100 Subject: [PATCH 01/29] build: remove explicit libssp linking from Windows build --- configure.ac | 6 ------ 1 file changed, 6 deletions(-) diff --git a/configure.ac b/configure.ac index 6add570d00..147a514686 100644 --- a/configure.ac +++ b/configure.ac @@ -988,12 +988,6 @@ if test "$use_hardening" != "no"; then AX_CHECK_LINK_FLAG([-Wl,-z,now], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,now"], [], [$LDFLAG_WERROR]) AX_CHECK_LINK_FLAG([-Wl,-z,separate-code], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,separate-code"], [], [$LDFLAG_WERROR]) AX_CHECK_LINK_FLAG([-fPIE -pie], [PIE_FLAGS="-fPIE"; HARDENED_LDFLAGS="$HARDENED_LDFLAGS -pie"], [], [$CXXFLAG_WERROR]) - - case $host in - *mingw*) - AC_CHECK_LIB([ssp], [main], [], [AC_MSG_ERROR([libssp missing])]) - ;; - esac fi dnl These flags are specific to ld64, and may cause issues with other linkers. From 95d55b96c2cfd2a0d5a246d4a9eff9d0744ba223 Mon Sep 17 00:00:00 2001 From: fanquake Date: Tue, 12 Sep 2023 13:04:51 +0100 Subject: [PATCH 02/29] guix: remove ssp workaround from Windows GCC --- contrib/guix/manifest.scm | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index ceb178155d..038f918d94 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -461,12 +461,7 @@ inspecting signatures in Mach-O binaries.") `(append ,flags ;; https://gcc.gnu.org/install/configure.html (list "--enable-threads=posix", - building-on))) - ((#:make-flags flags) - ;; Uses the SSP functions from glibc instead of from libssp.so. - ;; Our 'symbol-check' script will complain if we link against libssp.so, - ;; and thus will ensure that this works properly. - `(cons "gcc_cv_libc_provides_ssp=yes" ,flags)))))) + building-on))))))) (define-public linux-base-gcc (package From f95af98128f17002bf137a48441167020f3ef9bb Mon Sep 17 00:00:00 2001 From: fanquake Date: Tue, 12 Sep 2023 14:39:20 +0100 Subject: [PATCH 03/29] guix: default ssp for Windows GCC --- contrib/guix/manifest.scm | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 038f918d94..3795e6d049 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -461,6 +461,7 @@ inspecting signatures in Mach-O binaries.") `(append ,flags ;; https://gcc.gnu.org/install/configure.html (list "--enable-threads=posix", + "--enable-default-ssp=yes", building-on))))))) (define-public linux-base-gcc From faff3e3b4604519375e122c103b156ec13eef80f Mon Sep 17 00:00:00 2001 From: MarcoFalke <*~=`'#}+{/-|&$^_@721217.xyz> Date: Mon, 13 Nov 2023 16:54:20 +0100 Subject: [PATCH 04/29] lint: Report all lint errors instead of early exit --- ci/lint/06_script.sh | 9 ----- test/lint/test_runner/src/main.rs | 61 +++++++++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 12 deletions(-) diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index af7a517930..318b2bb819 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -23,16 +23,7 @@ else fi export COMMIT_RANGE -# This only checks that the trees are pure subtrees, it is not doing a full -# check with -r to not have to fetch all the remotes. -test/lint/git-subtree-check.sh src/crypto/ctaes -test/lint/git-subtree-check.sh src/secp256k1 -test/lint/git-subtree-check.sh src/minisketch -test/lint/git-subtree-check.sh src/leveldb -test/lint/git-subtree-check.sh src/crc32c RUST_BACKTRACE=1 "${LINT_RUNNER_PATH}/test_runner" -test/lint/check-doc.py -test/lint/all-lint.py if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ "$CIRRUS_PR" = "" ] ; then # Sanity check only the last few commits to get notified of missing sigs, diff --git a/test/lint/test_runner/src/main.rs b/test/lint/test_runner/src/main.rs index b7ec9ee3b2..ce94c3b628 100644 --- a/test/lint/test_runner/src/main.rs +++ b/test/lint/test_runner/src/main.rs @@ -7,7 +7,9 @@ use std::path::PathBuf; use std::process::Command; use std::process::ExitCode; -use String as LintError; +type LintError = String; +type LintResult = Result<(), LintError>; +type LintFn = fn() -> LintResult; /// Return the git command fn git() -> Command { @@ -31,7 +33,31 @@ fn get_git_root() -> String { check_output(git().args(["rev-parse", "--show-toplevel"])).unwrap() } -fn lint_std_filesystem() -> Result<(), LintError> { +fn lint_subtree() -> LintResult { + // This only checks that the trees are pure subtrees, it is not doing a full + // check with -r to not have to fetch all the remotes. + let mut good = true; + for subtree in [ + "src/crypto/ctaes", + "src/secp256k1", + "src/minisketch", + "src/leveldb", + "src/crc32c", + ] { + good &= Command::new("test/lint/git-subtree-check.sh") + .arg(subtree) + .status() + .expect("command_error") + .success(); + } + if good { + Ok(()) + } else { + Err("".to_string()) + } +} + +fn lint_std_filesystem() -> LintResult { let found = git() .args([ "grep", @@ -55,8 +81,37 @@ fs:: namespace, which has unsafe filesystem functions marked as deleted. } } +fn lint_doc() -> LintResult { + if Command::new("test/lint/check-doc.py") + .status() + .expect("command error") + .success() + { + Ok(()) + } else { + Err("".to_string()) + } +} + +fn lint_all() -> LintResult { + if Command::new("test/lint/all-lint.py") + .status() + .expect("command error") + .success() + { + Ok(()) + } else { + Err("".to_string()) + } +} + fn main() -> ExitCode { - let test_list = [("std::filesystem check", lint_std_filesystem)]; + let test_list: Vec<(&str, LintFn)> = vec![ + ("subtree check", lint_subtree), + ("std::filesystem check", lint_std_filesystem), + ("-help=1 documentation check", lint_doc), + ("all-lint.py script", lint_all), + ]; let git_root = PathBuf::from(get_git_root()); From fa01f884d3ac128f09bfa57ac2648a19a19d854e Mon Sep 17 00:00:00 2001 From: MarcoFalke <*~=`'#}+{/-|&$^_@721217.xyz> Date: Mon, 13 Nov 2023 18:10:55 +0100 Subject: [PATCH 05/29] ci: Add missing COPY for ./test/lint/test_runner --- ci/lint_imagefile | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/lint_imagefile b/ci/lint_imagefile index 4136a9bfde..d32b35b19d 100644 --- a/ci/lint_imagefile +++ b/ci/lint_imagefile @@ -12,6 +12,7 @@ ENV LC_ALL=C.UTF-8 COPY ./.python-version /.python-version COPY ./ci/lint/container-entrypoint.sh /entrypoint.sh COPY ./ci/lint/04_install.sh /install.sh +COPY ./test/lint/test_runner /test/lint/test_runner RUN /install.sh && \ echo 'alias lint="./ci/lint/06_script.sh"' >> ~/.bashrc && \ From 007d6f0e85bc329040bb405ef6016339518caa66 Mon Sep 17 00:00:00 2001 From: Sebastian Falbesoner Date: Thu, 16 Nov 2023 03:47:26 +0100 Subject: [PATCH 06/29] test: fix `AddNode` unit test failure on OpenBSD --- src/test/net_peer_connection_tests.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/net_peer_connection_tests.cpp b/src/test/net_peer_connection_tests.cpp index 3d3f296d82..a9e9858d95 100644 --- a/src/test/net_peer_connection_tests.cpp +++ b/src/test/net_peer_connection_tests.cpp @@ -116,7 +116,10 @@ BOOST_AUTO_TEST_CASE(test_addnode_getaddednodeinfo_and_connection_detection) BOOST_TEST_MESSAGE("\nCall AddNode() with 2 addrs resolving to existing localhost addnode entry; neither should be added"); BOOST_CHECK(!connman->AddNode({/*m_added_node=*/"127.0.0.1", /*m_use_v2transport=*/true})); + // OpenBSD doesn't support the IPv4 shorthand notation with omitted zero-bytes. +#if !defined(__OpenBSD__) BOOST_CHECK(!connman->AddNode({/*m_added_node=*/"127.1", /*m_use_v2transport=*/true})); +#endif BOOST_TEST_MESSAGE("\nExpect GetAddedNodeInfo to return expected number of peers with `include_connected` true/false"); BOOST_CHECK_EQUAL(connman->GetAddedNodeInfo(/*include_connected=*/true).size(), nodes.size()); From cc627169206fe902157806d88fcaf2b05701c38d Mon Sep 17 00:00:00 2001 From: Jon Atack Date: Thu, 16 Nov 2023 09:56:03 -0600 Subject: [PATCH 07/29] p2p: do not make automatic outbound connections to addnode peers to allocate our limited outbound slots correctly, and to ensure addnode connections benefit from their intended protections. Our addnode logic usually connects the addnode peers before the automatic outbound logic does, but not always, as a connection race can occur. If an addnode peer disconnects us and if it was the only one from its network, there can be a race between reconnecting to it with the addnode thread, and it being picked as automatic network-specific outbound peer. Or our internet connection or router, or the addnode peer, could be temporarily offline, and then return online during the automatic outbound thread. Or we could add a new manual peer using the addnode RPC at that time. The race can be more apparent when our node doesn't know many peers, or with networks like cjdns that currently have few bitcoin peers. When an addnode peer is connected as an automatic outbound peer and is the only connection we have to a network, it can be protected by our new outbound eviction logic and persist in the "wrong role". Examples on mainnet using logging added in the same pull request: 2023-08-12T14:51:05.681743Z [opencon] [net.cpp:1949] [ThreadOpenConnections] [net:debug] Not making automatic network-specific outbound-full-relay connection to i2p peer selected for manual (addnode) connection: [geh...odq.b32.i2p]:0 2023-08-13T03:59:28.050853Z [opencon] [net.cpp:1949] [ThreadOpenConnections] [net:debug] Not making automatic block-relay-only connection to onion peer selected for manual (addnode) connection: kpg...aid.onion:8333 2023-08-13T16:21:26.979052Z [opencon] [net.cpp:1949] [ThreadOpenConnections] [net:debug] Not making automatic network-specific outbound-full-relay connection to cjdns peer selected for manual (addnode) connection: [fcc...8ce]:8333 2023-08-14T20:43:53.401271Z [opencon] [net.cpp:1949] [ThreadOpenConnections] [net:debug] Not making automatic network-specific outbound-full-relay connection to cjdns peer selected for manual (addnode) connection: [fc7...59e]:8333 2023-08-15T00:10:01.894147Z [opencon] [net.cpp:1949] [ThreadOpenConnections] [net:debug] Not making automatic feeler connection to i2p peer selected for manual (addnode) connection: geh...odq.b32.i2p:8333 Finally, there does not seem to be a reason to make block-relay or short-lived feeler connections to addnode peers, as the addnode logic will ensure we connect to them if they are up, within the addnode connection limit. Fix these issues by checking if the address is an addnode peer in our automatic outbound connection logic. --- src/net.cpp | 22 ++++++++++++++++++++++ src/net.h | 1 + 2 files changed, 23 insertions(+) diff --git a/src/net.cpp b/src/net.cpp index a2f80cbcf7..49b84597bc 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -2715,6 +2715,17 @@ void CConnman::ThreadOpenConnections(const std::vector connect) continue; } + // Do not make automatic outbound connections to addnode peers, to + // not use our limited outbound slots for them and to ensure + // addnode connections benefit from their intended protections. + if (AddedNodesContain(addr)) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "Not making automatic %s%s connection to %s peer selected for manual (addnode) connection%s\n", + preferred_net.has_value() ? "network-specific " : "", + ConnectionTypeAsString(conn_type), GetNetworkName(addr.GetNetwork()), + fLogIPs ? strprintf(": %s", addr.ToStringAddrPort()) : ""); + continue; + } + addrConnect = addr; break; } @@ -3454,6 +3465,17 @@ bool CConnman::RemoveAddedNode(const std::string& strNode) return false; } +bool CConnman::AddedNodesContain(const CAddress& addr) const +{ + AssertLockNotHeld(m_added_nodes_mutex); + const std::string addr_str{addr.ToStringAddr()}; + const std::string addr_port_str{addr.ToStringAddrPort()}; + LOCK(m_added_nodes_mutex); + return (m_added_node_params.size() < 24 // bound the query to a reasonable limit + && std::any_of(m_added_node_params.cbegin(), m_added_node_params.cend(), + [&](const auto& p) { return p.m_added_node == addr_str || p.m_added_node == addr_port_str; })); +} + size_t CConnman::GetNodeCount(ConnectionDirection flags) const { LOCK(m_nodes_mutex); diff --git a/src/net.h b/src/net.h index 16312cf72d..3b38f19350 100644 --- a/src/net.h +++ b/src/net.h @@ -1184,6 +1184,7 @@ class CConnman bool AddNode(const AddedNodeParams& add) EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); bool RemoveAddedNode(const std::string& node) EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); + bool AddedNodesContain(const CAddress& addr) const EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); std::vector GetAddedNodeInfo(bool include_connected) const EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); /** From 5e7cc4144bb09cfde48ba5bc055a2da3ca4252d1 Mon Sep 17 00:00:00 2001 From: Jon Atack Date: Thu, 16 Nov 2023 10:35:08 -0600 Subject: [PATCH 08/29] test: add unit test for CConnman::AddedNodesContain() --- src/test/net_peer_connection_tests.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/test/net_peer_connection_tests.cpp b/src/test/net_peer_connection_tests.cpp index 3d3f296d82..f1662fa00f 100644 --- a/src/test/net_peer_connection_tests.cpp +++ b/src/test/net_peer_connection_tests.cpp @@ -122,6 +122,13 @@ BOOST_AUTO_TEST_CASE(test_addnode_getaddednodeinfo_and_connection_detection) BOOST_CHECK_EQUAL(connman->GetAddedNodeInfo(/*include_connected=*/true).size(), nodes.size()); BOOST_CHECK(connman->GetAddedNodeInfo(/*include_connected=*/false).empty()); + // Test AddedNodesContain() + for (auto node : connman->TestNodes()) { + BOOST_CHECK(connman->AddedNodesContain(node->addr)); + } + AddPeer(id, nodes, *peerman, *connman, ConnectionType::OUTBOUND_FULL_RELAY); + BOOST_CHECK(!connman->AddedNodesContain(nodes.back()->addr)); + BOOST_TEST_MESSAGE("\nPrint GetAddedNodeInfo contents:"); for (const auto& info : connman->GetAddedNodeInfo(/*include_connected=*/true)) { BOOST_TEST_MESSAGE(strprintf("\nadded node: %s", info.m_params.m_added_node)); From e63f64307929ad398a23ecfaabc3664270883155 Mon Sep 17 00:00:00 2001 From: Anthony Towns Date: Wed, 15 Nov 2023 13:07:14 +1000 Subject: [PATCH 09/29] streams: Base BufferedFile on AutoFile instead of CAutoFile --- src/streams.h | 8 +++----- src/test/fuzz/buffered_file.cpp | 4 +--- src/test/streams_tests.cpp | 3 --- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/src/streams.h b/src/streams.h index a3f8028da7..e7779d5bbd 100644 --- a/src/streams.h +++ b/src/streams.h @@ -529,7 +529,7 @@ class CAutoFile : public AutoFile } }; -/** Wrapper around a CAutoFile& that implements a ring buffer to +/** Wrapper around an AutoFile& that implements a ring buffer to * deserialize from. It guarantees the ability to rewind a given number of bytes. * * Will automatically close the file when it goes out of scope if not null. @@ -538,7 +538,7 @@ class CAutoFile : public AutoFile class BufferedFile { private: - CAutoFile& m_src; + AutoFile& m_src; uint64_t nSrcPos{0}; //!< how many bytes have been read from source uint64_t m_read_pos{0}; //!< how many bytes have been read from this uint64_t nReadLimit; //!< up to which position we're allowed to read @@ -585,15 +585,13 @@ class BufferedFile } public: - BufferedFile(CAutoFile& file, uint64_t nBufSize, uint64_t nRewindIn) + BufferedFile(AutoFile& file, uint64_t nBufSize, uint64_t nRewindIn) : m_src{file}, nReadLimit{std::numeric_limits::max()}, nRewind{nRewindIn}, vchBuf(nBufSize, std::byte{0}) { if (nRewindIn >= nBufSize) throw std::ios_base::failure("Rewind limit must be less than buffer size"); } - int GetVersion() const { return m_src.GetVersion(); } - //! check whether we're at the end of the source file bool eof() const { return m_read_pos == nSrcPos && m_src.feof(); diff --git a/src/test/fuzz/buffered_file.cpp b/src/test/fuzz/buffered_file.cpp index 813af63738..e30c19b265 100644 --- a/src/test/fuzz/buffered_file.cpp +++ b/src/test/fuzz/buffered_file.cpp @@ -20,9 +20,8 @@ FUZZ_TARGET(buffered_file) FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; FuzzedFileProvider fuzzed_file_provider{fuzzed_data_provider}; std::optional opt_buffered_file; - CAutoFile fuzzed_file{ + AutoFile fuzzed_file{ fuzzed_file_provider.open(), - 0, ConsumeRandomLengthByteVector(fuzzed_data_provider), }; try { @@ -65,6 +64,5 @@ FUZZ_TARGET(buffered_file) }); } opt_buffered_file->GetPos(); - opt_buffered_file->GetVersion(); } } diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index f03f7c1da2..38fdc7a6e9 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -271,9 +271,6 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) BufferedFile bf{file, 25, 10}; BOOST_CHECK(!bf.eof()); - // This member has no functional effect. - BOOST_CHECK_EQUAL(bf.GetVersion(), 333); - uint8_t i; bf >> i; BOOST_CHECK_EQUAL(i, 0); From c72ddf04db95a94e91939d46d13ab6a46fefb4be Mon Sep 17 00:00:00 2001 From: Anthony Towns Date: Fri, 17 Nov 2023 23:58:43 +1000 Subject: [PATCH 10/29] streams: Remove unused CAutoFile::GetVersion --- src/streams.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/streams.h b/src/streams.h index e7779d5bbd..794a0dd6b7 100644 --- a/src/streams.h +++ b/src/streams.h @@ -507,12 +507,8 @@ class AutoFile class CAutoFile : public AutoFile { -private: - const int nVersion; - public: - explicit CAutoFile(std::FILE* file, int version, std::vector data_xor = {}) : AutoFile{file, std::move(data_xor)}, nVersion{version} {} - int GetVersion() const { return nVersion; } + explicit CAutoFile(std::FILE* file, int /*unused*/, std::vector data_xor = {}) : AutoFile{file, std::move(data_xor)} {} template CAutoFile& operator<<(const T& obj) From 91d5bd8ac9a28725c735f8e6900bc85673bb190a Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Fri, 17 Nov 2023 15:15:06 +0000 Subject: [PATCH 11/29] ci: Avoid toolset ambiguity that MSVC can't handle This change is required to work with the new windows-2022 image version 20231115 properly. --- .github/workflows/ci.yml | 49 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1cd40be541..8b4de792ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,6 +136,55 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Fix Visual Studio installation + # Avoid toolset ambiguity that MSVC can't handle. + run: | + Set-Location "C:\Program Files (x86)\Microsoft Visual Studio\Installer\" + $InstallPath = "C:\Program Files\Microsoft Visual Studio\2022\Enterprise" + $componentsToRemove= @( + "Microsoft.VisualStudio.Component.VC.14.37.17.7.ARM.Spectre" + "Microsoft.VisualStudio.Component.VC.14.37.17.7.ARM" + "Microsoft.VisualStudio.Component.VC.14.37.17.7.ARM64.Spectre" + "Microsoft.VisualStudio.Component.VC.14.37.17.7.ARM64" + "Microsoft.VisualStudio.Component.VC.14.37.17.7.ATL.ARM.Spectre" + "Microsoft.VisualStudio.Component.VC.14.37.17.7.ATL.ARM" + "Microsoft.VisualStudio.Component.VC.14.37.17.7.ATL.ARM64.Spectre" + "Microsoft.VisualStudio.Component.VC.14.37.17.7.ATL.ARM64" + "Microsoft.VisualStudio.Component.VC.14.37.17.7.ATL.Spectre" + "Microsoft.VisualStudio.Component.VC.14.37.17.7.ATL" + "Microsoft.VisualStudio.Component.VC.14.37.17.7.MFC.ARM.Spectre" + "Microsoft.VisualStudio.Component.VC.14.37.17.7.MFC.ARM" + "Microsoft.VisualStudio.Component.VC.14.37.17.7.MFC.ARM64.Spectre" + "Microsoft.VisualStudio.Component.VC.14.37.17.7.MFC.ARM64" + "Microsoft.VisualStudio.Component.VC.14.37.17.7.MFC.Spectre" + "Microsoft.VisualStudio.Component.VC.14.37.17.7.MFC" + "Microsoft.VisualStudio.Component.VC.14.37.17.7.x86.x64.Spectre" + "Microsoft.VisualStudio.Component.VC.14.37.17.7.x86.x64" + "Microsoft.VisualStudio.Component.VC.14.38.17.8.ARM" + "Microsoft.VisualStudio.Component.VC.14.38.17.8.ARM.Spectre" + "Microsoft.VisualStudio.Component.VC.14.38.17.8.ARM64" + "Microsoft.VisualStudio.Component.VC.14.38.17.8.ARM64.Spectre" + "Microsoft.VisualStudio.Component.VC.14.38.17.8.ATL" + "Microsoft.VisualStudio.Component.VC.14.38.17.8.ATL.ARM" + "Microsoft.VisualStudio.Component.VC.14.38.17.8.ATL.ARM.Spectre" + "Microsoft.VisualStudio.Component.VC.14.38.17.8.ATL.ARM64" + "Microsoft.VisualStudio.Component.VC.14.38.17.8.ATL.ARM64.Spectre" + "Microsoft.VisualStudio.Component.VC.14.38.17.8.ATL.Spectre" + "Microsoft.VisualStudio.Component.VC.14.38.17.8.MFC" + "Microsoft.VisualStudio.Component.VC.14.38.17.8.MFC.ARM" + "Microsoft.VisualStudio.Component.VC.14.38.17.8.MFC.ARM.Spectre" + "Microsoft.VisualStudio.Component.VC.14.38.17.8.MFC.ARM64" + "Microsoft.VisualStudio.Component.VC.14.38.17.8.MFC.ARM64.Spectre" + "Microsoft.VisualStudio.Component.VC.14.38.17.8.MFC.Spectre" + "Microsoft.VisualStudio.Component.VC.14.38.17.8.x86.x64" + "Microsoft.VisualStudio.Component.VC.14.38.17.8.x86.x64.Spectre" + ) + [string]$workloadArgs = $componentsToRemove | ForEach-Object {" --remove " + $_} + $Arguments = ('/c', "vs_installer.exe", 'modify', '--installPath', "`"$InstallPath`"",$workloadArgs, '--quiet', '--norestart', '--nocache') + # should be run twice + $process = Start-Process -FilePath cmd.exe -ArgumentList $Arguments -Wait -PassThru -WindowStyle Hidden + $process = Start-Process -FilePath cmd.exe -ArgumentList $Arguments -Wait -PassThru -WindowStyle Hidden + - name: Configure Developer Command Prompt for Microsoft Visual C++ # Using microsoft/setup-msbuild is not enough. uses: ilammy/msvc-dev-cmd@v1 From bbd4646a2ef514c31570ca9d1475fc9ddb35bdfd Mon Sep 17 00:00:00 2001 From: Anthony Towns Date: Wed, 15 Nov 2023 13:07:28 +1000 Subject: [PATCH 12/29] blockstorage: switch from CAutoFile to AutoFile Also bump includes per suggestions from iwyu. --- src/index/txindex.cpp | 2 +- src/node/blockstorage.cpp | 35 ++++++++++++++-------- src/node/blockstorage.h | 16 +++++----- src/test/fuzz/load_external_block_file.cpp | 2 +- src/validation.cpp | 2 +- src/validation.h | 2 +- 6 files changed, 34 insertions(+), 25 deletions(-) diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 5d37fd0d8f..4983926e68 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -79,7 +79,7 @@ bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRe return false; } - CAutoFile file{m_chainstate->m_blockman.OpenBlockFile(postx, true)}; + AutoFile file{m_chainstate->m_blockman.OpenBlockFile(postx, true)}; if (file.IsNull()) { return error("%s: OpenBlockFile failed", __func__); } diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index c066d1c939..ebc08d7567 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -4,23 +4,32 @@ #include +#include #include -#include +#include #include #include #include #include -#include +#include #include #include +#include #include #include +#include +#include #include +#include #include +#include #include #include +#include +#include #include #include +#include #include #include #include @@ -656,7 +665,7 @@ CBlockFileInfo* BlockManager::GetBlockFileInfo(size_t n) bool BlockManager::UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock) const { // Open history file to append - CAutoFile fileout{OpenUndoFile(pos)}; + AutoFile fileout{OpenUndoFile(pos)}; if (fileout.IsNull()) { return error("%s: OpenUndoFile failed", __func__); } @@ -691,7 +700,7 @@ bool BlockManager::UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& in } // Open history file to read - CAutoFile filein{OpenUndoFile(pos, true)}; + AutoFile filein{OpenUndoFile(pos, true)}; if (filein.IsNull()) { return error("%s: OpenUndoFile failed", __func__); } @@ -810,15 +819,15 @@ FlatFileSeq BlockManager::UndoFileSeq() const return FlatFileSeq(m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE); } -CAutoFile BlockManager::OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) const +AutoFile BlockManager::OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) const { - return CAutoFile{BlockFileSeq().Open(pos, fReadOnly), CLIENT_VERSION}; + return AutoFile{BlockFileSeq().Open(pos, fReadOnly)}; } /** Open an undo file (rev?????.dat) */ -CAutoFile BlockManager::OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) const +AutoFile BlockManager::OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) const { - return CAutoFile{UndoFileSeq().Open(pos, fReadOnly), CLIENT_VERSION}; + return AutoFile{UndoFileSeq().Open(pos, fReadOnly)}; } fs::path BlockManager::GetBlockPosFilename(const FlatFilePos& pos) const @@ -950,7 +959,7 @@ bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFileP bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const { // Open history file to append - CAutoFile fileout{OpenBlockFile(pos)}; + AutoFile fileout{OpenBlockFile(pos)}; if (fileout.IsNull()) { return error("WriteBlockToDisk: OpenBlockFile failed"); } @@ -1016,7 +1025,7 @@ bool BlockManager::ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos) cons block.SetNull(); // Open history file to read - CAutoFile filein{OpenBlockFile(pos, true)}; + AutoFile filein{OpenBlockFile(pos, true)}; if (filein.IsNull()) { return error("ReadBlockFromDisk: OpenBlockFile failed for %s", pos.ToString()); } @@ -1059,7 +1068,7 @@ bool BlockManager::ReadRawBlockFromDisk(std::vector& block, const FlatF { FlatFilePos hpos = pos; hpos.nPos -= 8; // Seek back 8 bytes for meta header - CAutoFile filein{OpenBlockFile(hpos, true)}; + AutoFile filein{OpenBlockFile(hpos, true)}; if (filein.IsNull()) { return error("%s: OpenBlockFile failed for %s", __func__, pos.ToString()); } @@ -1151,7 +1160,7 @@ void ImportBlocks(ChainstateManager& chainman, std::vector vImportFile if (!fs::exists(chainman.m_blockman.GetBlockPosFilename(pos))) { break; // No block files left to reindex } - CAutoFile file{chainman.m_blockman.OpenBlockFile(pos, true)}; + AutoFile file{chainman.m_blockman.OpenBlockFile(pos, true)}; if (file.IsNull()) { break; // This error is logged in OpenBlockFile } @@ -1172,7 +1181,7 @@ void ImportBlocks(ChainstateManager& chainman, std::vector vImportFile // -loadblock= for (const fs::path& path : vImportFiles) { - CAutoFile file{fsbridge::fopen(path, "rb"), CLIENT_VERSION}; + AutoFile file{fsbridge::fopen(path, "rb")}; if (!file.IsNull()) { LogPrintf("Importing blocks file %s...\n", fs::PathToString(path)); chainman.LoadExternalBlockFile(file); diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index ba44d31581..d540406ae5 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -8,21 +8,26 @@ #include #include #include +#include #include -#include #include #include #include +#include +#include #include +#include #include #include +#include #include #include #include #include #include #include +#include #include #include #include @@ -30,14 +35,9 @@ #include class BlockValidationState; -class CAutoFile; -class CBlock; class CBlockUndo; -class CChainParams; class Chainstate; class ChainstateManager; -struct CCheckpointData; -struct FlatFilePos; namespace Consensus { struct Params; } @@ -162,7 +162,7 @@ class BlockManager FlatFileSeq BlockFileSeq() const; FlatFileSeq UndoFileSeq() const; - CAutoFile OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false) const; + AutoFile OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false) const; bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const; bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock) const; @@ -350,7 +350,7 @@ class BlockManager void UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** Open a block file (blk?????.dat) */ - CAutoFile OpenBlockFile(const FlatFilePos& pos, bool fReadOnly = false) const; + AutoFile OpenBlockFile(const FlatFilePos& pos, bool fReadOnly = false) const; /** Translation to a filesystem path */ fs::path GetBlockPosFilename(const FlatFilePos& pos) const; diff --git a/src/test/fuzz/load_external_block_file.cpp b/src/test/fuzz/load_external_block_file.cpp index ae4f5d089b..6460261f0f 100644 --- a/src/test/fuzz/load_external_block_file.cpp +++ b/src/test/fuzz/load_external_block_file.cpp @@ -28,7 +28,7 @@ FUZZ_TARGET(load_external_block_file, .init = initialize_load_external_block_fil { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; FuzzedFileProvider fuzzed_file_provider{fuzzed_data_provider}; - CAutoFile fuzzed_block_file{fuzzed_file_provider.open(), CLIENT_VERSION}; + AutoFile fuzzed_block_file{fuzzed_file_provider.open()}; if (fuzzed_block_file.IsNull()) { return; } diff --git a/src/validation.cpp b/src/validation.cpp index ed72a7c97a..728a049f89 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -4649,7 +4649,7 @@ bool Chainstate::LoadGenesisBlock() } void ChainstateManager::LoadExternalBlockFile( - CAutoFile& file_in, + AutoFile& file_in, FlatFilePos* dbp, std::multimap* blocks_with_unknown_parent) { diff --git a/src/validation.h b/src/validation.h index e669ec46d5..7473bcbc3b 100644 --- a/src/validation.h +++ b/src/validation.h @@ -1137,7 +1137,7 @@ class ChainstateManager * (only used for reindex) * */ void LoadExternalBlockFile( - CAutoFile& file_in, + AutoFile& file_in, FlatFilePos* dbp = nullptr, std::multimap* blocks_with_unknown_parent = nullptr); From cde9a4b137e93f1548dffac9b396ca0b472b6187 Mon Sep 17 00:00:00 2001 From: Anthony Towns Date: Wed, 15 Nov 2023 13:10:13 +1000 Subject: [PATCH 13/29] refactor: switch from CAutoFile to AutoFile --- src/bench/load_external.cpp | 2 +- src/bench/streams_findbyte.cpp | 2 +- src/test/streams_tests.cpp | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/bench/load_external.cpp b/src/bench/load_external.cpp index 3b100d97b0..fba1233901 100644 --- a/src/bench/load_external.cpp +++ b/src/bench/load_external.cpp @@ -55,7 +55,7 @@ static void LoadExternalBlockFile(benchmark::Bench& bench) bench.run([&] { // "rb" is "binary, O_RDONLY", positioned to the start of the file. // The file will be closed by LoadExternalBlockFile(). - CAutoFile file{fsbridge::fopen(blkfile, "rb"), CLIENT_VERSION}; + AutoFile file{fsbridge::fopen(blkfile, "rb")}; testing_setup->m_node.chainman->LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent); }); fs::remove(blkfile); diff --git a/src/bench/streams_findbyte.cpp b/src/bench/streams_findbyte.cpp index 4aaccb2af8..5098262e9a 100644 --- a/src/bench/streams_findbyte.cpp +++ b/src/bench/streams_findbyte.cpp @@ -14,7 +14,7 @@ static void FindByte(benchmark::Bench& bench) { // Setup - CAutoFile file{fsbridge::fopen("streams_tmp", "w+b"), 0}; + AutoFile file{fsbridge::fopen("streams_tmp", "w+b")}; const size_t file_size = 200; uint8_t data[file_size] = {0}; data[file_size-1] = 1; diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index 38fdc7a6e9..d8478ecf5f 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -249,7 +249,7 @@ BOOST_AUTO_TEST_CASE(streams_serializedata_xor) BOOST_AUTO_TEST_CASE(streams_buffered_file) { fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp"; - CAutoFile file{fsbridge::fopen(streams_test_filename, "w+b"), 333}; + AutoFile file{fsbridge::fopen(streams_test_filename, "w+b")}; // The value at each offset is the offset. for (uint8_t j = 0; j < 40; ++j) { @@ -380,7 +380,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) BOOST_AUTO_TEST_CASE(streams_buffered_file_skip) { fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp"; - CAutoFile file{fsbridge::fopen(streams_test_filename, "w+b"), 333}; + AutoFile file{fsbridge::fopen(streams_test_filename, "w+b")}; // The value at each offset is the byte offset (e.g. byte 1 in the file has the value 0x01). for (uint8_t j = 0; j < 40; ++j) { file << j; @@ -433,7 +433,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp"; for (int rep = 0; rep < 50; ++rep) { - CAutoFile file{fsbridge::fopen(streams_test_filename, "w+b"), 333}; + AutoFile file{fsbridge::fopen(streams_test_filename, "w+b")}; size_t fileSize = InsecureRandRange(256); for (uint8_t i = 0; i < fileSize; ++i) { file << i; From 4eb2a9ea4b6262bec0bc7c20cb3e684ea75caf42 Mon Sep 17 00:00:00 2001 From: Anthony Towns Date: Fri, 17 Nov 2023 23:57:35 +1000 Subject: [PATCH 14/29] streams: Drop unused CAutoFile --- src/streams.h | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/src/streams.h b/src/streams.h index 794a0dd6b7..068e2e8efd 100644 --- a/src/streams.h +++ b/src/streams.h @@ -505,26 +505,6 @@ class AutoFile } }; -class CAutoFile : public AutoFile -{ -public: - explicit CAutoFile(std::FILE* file, int /*unused*/, std::vector data_xor = {}) : AutoFile{file, std::move(data_xor)} {} - - template - CAutoFile& operator<<(const T& obj) - { - ::Serialize(*this, obj); - return (*this); - } - - template - CAutoFile& operator>>(T&& obj) - { - ::Unserialize(*this, obj); - return (*this); - } -}; - /** Wrapper around an AutoFile& that implements a ring buffer to * deserialize from. It guarantees the ability to rewind a given number of bytes. * From 21bfee0720ba72935d4f9fc4c2a2887ae5b54092 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Fri, 17 Nov 2023 15:27:19 -0500 Subject: [PATCH 15/29] depends: bump libmultiprocess to fix capnproto deprecation warnings This incorporates PR https://github.com/chaincodelabs/libmultiprocess/pull/88 "Fix current deprecation warnings as of capnproto-1.0.1" and reverts the NO_WERROR CI workaround added in https://github.com/bitcoin/bitcoin/pull/28735 --- ci/test/00_setup_env_i686_multiprocess.sh | 1 - depends/packages/native_libmultiprocess.mk | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ci/test/00_setup_env_i686_multiprocess.sh b/ci/test/00_setup_env_i686_multiprocess.sh index 1befbc6164..647f4103e2 100755 --- a/ci/test/00_setup_env_i686_multiprocess.sh +++ b/ci/test/00_setup_env_i686_multiprocess.sh @@ -15,4 +15,3 @@ export GOAL="install" export BITCOIN_CONFIG="--enable-debug CC='clang -m32' CXX='clang++ -m32' \ LDFLAGS='--rtlib=compiler-rt -lgcc_s' CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE'" export TEST_RUNNER_ENV="BITCOIND=bitcoin-node" -export NO_WERROR=1 # Temporary workaround to avoid -Wdeprecated-declarations from KJ diff --git a/depends/packages/native_libmultiprocess.mk b/depends/packages/native_libmultiprocess.mk index c91ffb0334..946e885354 100644 --- a/depends/packages/native_libmultiprocess.mk +++ b/depends/packages/native_libmultiprocess.mk @@ -1,8 +1,8 @@ package=native_libmultiprocess -$(package)_version=61d5a0e661f20a4928fbf868ec9c3c6f17883cc7 +$(package)_version=414542f81e0997354b45b8ade13ca144a3e35ff1 $(package)_download_path=https://github.com/chaincodelabs/libmultiprocess/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=5cfda224cc2ce913f2493f843317e0fca3184f6d7c1434c9754b2e7dca440ab5 +$(package)_sha256_hash=8542dbaf8c4fce8fd7af6929f5dc9b34dffa51c43e9ee360e93ee0f34b180bc2 $(package)_dependencies=native_capnp define $(package)_config_cmds From ce881bf9fcb7c30bb1fafd6ce38844f4f829452a Mon Sep 17 00:00:00 2001 From: Martin Leitner-Ankerl Date: Sun, 19 Nov 2023 15:57:03 +0100 Subject: [PATCH 16/29] pool: make sure PoolAllocator uses the correct alignment This changes the PoolAllocator to default the alignment to the given type. This makes the code simpler, and most importantly fixes a bug on ARM 32bit that caused OOM: The class CTxOut has a member CAmount which is an int64_t and on ARM 32bit int64_t are 8 byte aligned which is larger than the pointer alignment of 4 bytes. So for CCoinsMap to be able to use the pool, we need to use the alignment of the member instead of just alignof(void*). --- src/bench/pool.cpp | 3 +-- src/coins.h | 3 +-- src/support/allocators/pool.h | 2 +- src/test/pool_tests.cpp | 3 +-- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/bench/pool.cpp b/src/bench/pool.cpp index b3e54d85a2..b2a5f8debf 100644 --- a/src/bench/pool.cpp +++ b/src/bench/pool.cpp @@ -37,8 +37,7 @@ static void PoolAllocator_StdUnorderedMapWithPoolResource(benchmark::Bench& benc std::hash, std::equal_to, PoolAllocator, - sizeof(std::pair) + 4 * sizeof(void*), - alignof(void*)>>; + sizeof(std::pair) + 4 * sizeof(void*)>>; // make sure the resource supports large enough pools to hold the node. We do this by adding the size of a few pointers to it. auto pool_resource = Map::allocator_type::ResourceType(); diff --git a/src/coins.h b/src/coins.h index a6cbb03133..bbf9e3895f 100644 --- a/src/coins.h +++ b/src/coins.h @@ -145,8 +145,7 @@ using CCoinsMap = std::unordered_map, PoolAllocator, - sizeof(std::pair) + sizeof(void*) * 4, - alignof(void*)>>; + sizeof(std::pair) + sizeof(void*) * 4>>; using CCoinsMapMemoryResource = CCoinsMap::allocator_type::ResourceType; diff --git a/src/support/allocators/pool.h b/src/support/allocators/pool.h index c8e70ebacf..873e260b65 100644 --- a/src/support/allocators/pool.h +++ b/src/support/allocators/pool.h @@ -272,7 +272,7 @@ class PoolResource final /** * Forwards all allocations/deallocations to the PoolResource. */ -template +template class PoolAllocator { PoolResource* m_resource; diff --git a/src/test/pool_tests.cpp b/src/test/pool_tests.cpp index 8a07e09a44..2663e088eb 100644 --- a/src/test/pool_tests.cpp +++ b/src/test/pool_tests.cpp @@ -163,8 +163,7 @@ BOOST_AUTO_TEST_CASE(memusage_test) std::hash, std::equal_to, PoolAllocator, - sizeof(std::pair) + sizeof(void*) * 4, - alignof(void*)>>; + sizeof(std::pair) + sizeof(void*) * 4>>; auto resource = Map::allocator_type::ResourceType(1024); PoolResourceTester::CheckAllDataAccountedFor(resource); From 228d6a2969e4fcee573c9df7aad31550eab9c8d4 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Mon, 20 Nov 2023 13:37:44 +0000 Subject: [PATCH 17/29] build: Fix regression in "ARMv8 CRC32 intrinsics" test The `vmull_p64` is a part of the Crypto extensions from the ACLE. They are optional extensions, so they get enabled with a `+crypto` for architecture flags. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 20992380d2..8305e5bffa 100644 --- a/configure.ac +++ b/configure.ac @@ -579,7 +579,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ CXXFLAGS="$TEMP_CXXFLAGS" # ARM -AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc], [ARM_CRC_CXXFLAGS="-march=armv8-a+crc"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc+crypto], [ARM_CRC_CXXFLAGS="-march=armv8-a+crc+crypto"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-march=armv8-a+crypto], [ARM_SHANI_CXXFLAGS="-march=armv8-a+crypto"], [], [$CXXFLAG_WERROR]) TEMP_CXXFLAGS="$CXXFLAGS" From d5b4c0b69e543de51bb37d602d488ee0949ba185 Mon Sep 17 00:00:00 2001 From: Martin Leitner-Ankerl Date: Sun, 19 Nov 2023 18:43:15 +0100 Subject: [PATCH 18/29] pool: change memusage_test to use int64_t, add allocation check If alignment of the PoolAllocator would be insufficient, then the test would fail. This also catches the issue with ARM 32bit, where int64_t is aligned to 8 bytes but void* is aligned to 4 bytes. The test adds a check to ensure the pool has allocated a minimum number of chunks --- src/test/pool_tests.cpp | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/test/pool_tests.cpp b/src/test/pool_tests.cpp index 2663e088eb..5ad4afa3a1 100644 --- a/src/test/pool_tests.cpp +++ b/src/test/pool_tests.cpp @@ -156,20 +156,20 @@ BOOST_AUTO_TEST_CASE(random_allocations) BOOST_AUTO_TEST_CASE(memusage_test) { - auto std_map = std::unordered_map{}; - - using Map = std::unordered_map, - std::equal_to, - PoolAllocator, - sizeof(std::pair) + sizeof(void*) * 4>>; + auto std_map = std::unordered_map{}; + + using Map = std::unordered_map, + std::equal_to, + PoolAllocator, + sizeof(std::pair) + sizeof(void*) * 4>>; auto resource = Map::allocator_type::ResourceType(1024); PoolResourceTester::CheckAllDataAccountedFor(resource); { - auto resource_map = Map{0, std::hash{}, std::equal_to{}, &resource}; + auto resource_map = Map{0, std::hash{}, std::equal_to{}, &resource}; // can't have the same resource usage BOOST_TEST(memusage::DynamicUsage(std_map) != memusage::DynamicUsage(resource_map)); @@ -181,6 +181,11 @@ BOOST_AUTO_TEST_CASE(memusage_test) // Eventually the resource_map should have a much lower memory usage because it has less malloc overhead BOOST_TEST(memusage::DynamicUsage(resource_map) <= memusage::DynamicUsage(std_map) * 90 / 100); + + // Make sure the pool is actually used by the nodes + auto max_nodes_per_chunk = resource.ChunkSizeBytes() / sizeof(Map::value_type); + auto min_num_allocated_chunks = resource_map.size() / max_nodes_per_chunk + 1; + BOOST_TEST(resource.NumAllocatedChunks() >= min_num_allocated_chunks); } PoolResourceTester::CheckAllDataAccountedFor(resource); From 2e1833ca1341ab4dc92508a59181aa6c7c38db88 Mon Sep 17 00:00:00 2001 From: brunoerg Date: Wed, 4 Oct 2023 16:28:06 -0300 Subject: [PATCH 19/29] fuzz: move `MockedDescriptorConverter` to `fuzz/util` --- src/Makefile.test_fuzz.include | 2 + src/test/fuzz/descriptor_parse.cpp | 96 +----------------------------- src/test/fuzz/util/descriptor.cpp | 72 ++++++++++++++++++++++ src/test/fuzz/util/descriptor.h | 48 +++++++++++++++ 4 files changed, 123 insertions(+), 95 deletions(-) create mode 100644 src/test/fuzz/util/descriptor.cpp create mode 100644 src/test/fuzz/util/descriptor.h diff --git a/src/Makefile.test_fuzz.include b/src/Makefile.test_fuzz.include index aa9c052750..b4337991e4 100644 --- a/src/Makefile.test_fuzz.include +++ b/src/Makefile.test_fuzz.include @@ -11,6 +11,7 @@ TEST_FUZZ_H = \ test/fuzz/fuzz.h \ test/fuzz/FuzzedDataProvider.h \ test/fuzz/util.h \ + test/fuzz/util/descriptor.h \ test/fuzz/util/mempool.h \ test/fuzz/util/net.h @@ -19,6 +20,7 @@ libtest_fuzz_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libtest_fuzz_a_SOURCES = \ test/fuzz/fuzz.cpp \ test/fuzz/util.cpp \ + test/fuzz/util/descriptor.cpp \ test/fuzz/util/mempool.cpp \ test/fuzz/util/net.cpp \ $(TEST_FUZZ_H) diff --git a/src/test/fuzz/descriptor_parse.cpp b/src/test/fuzz/descriptor_parse.cpp index 57129a60b8..5474b38204 100644 --- a/src/test/fuzz/descriptor_parse.cpp +++ b/src/test/fuzz/descriptor_parse.cpp @@ -7,104 +7,10 @@ #include #include