diff --git a/depends/packages/openssl.mk b/depends/packages/openssl.mk index 37c2b076cce..388421891de 100644 --- a/depends/packages/openssl.mk +++ b/depends/packages/openssl.mk @@ -1,8 +1,8 @@ package=openssl -$(package)_version=1.1.1q +$(package)_version=1.1.1t $(package)_download_path=https://www.openssl.org/source $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=d7939ce614029cdff0b6c20f0e2e5703158a489a72b2507b8bd51bf8c8fd10ca +$(package)_sha256_hash=8dee9b24bdb1dcbf0c3d1e9b02fb8f6bf22165e807f45adeb7c9677536859d3b define $(package)_set_vars $(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" diff --git a/src/Makefile.am b/src/Makefile.am index b2faa6efe83..c77055a3235 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -471,6 +471,8 @@ crypto_libbitcoin_crypto_a_SOURCES = \ crypto/sha256.h \ crypto/sha512.cpp \ crypto/sha512.h \ + crypto/chacha20.h \ + crypto/chacha20.cpp \ crypto/haraka.h \ crypto/haraka_portable.h \ crypto/verus_hash.h \ @@ -590,6 +592,7 @@ libbitcoin_util_a_SOURCES = \ compat/glibc_sanity.cpp \ compat/glibcxx_sanity.cpp \ compat/strnlen.cpp \ + crypto/chacha20.cpp \ random.cpp \ rpc/protocol.cpp \ support/cleanse.cpp \ diff --git a/src/bloom.cpp b/src/bloom.cpp index 49dc5da695b..180a823abd9 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -1,4 +1,5 @@ // Copyright (c) 2012-2014 The Bitcoin Core developers +// Copyright (c) 2019-2023 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or https://www.opensource.org/licenses/mit-license.php . @@ -14,53 +15,38 @@ #include #include -#include +#include #define LN2SQUARED 0.4804530139182014246671025263266649717305529515945455 #define LN2 0.6931471805599453094172321214581765680755001343602552 -using namespace std; - -CBloomFilter::CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweakIn, unsigned char nFlagsIn) : +CBloomFilter::CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweakIn, unsigned char nFlagsIn) : /** * The ideal size for a bloom filter with a given number of elements and false positive rate is: * - nElements * log(fp rate) / ln(2)^2 * We ignore filter parameters which will create a bloom filter larger than the protocol limits */ - vData(min((unsigned int)(-1 / LN2SQUARED * nElements * log(nFPRate)), MAX_BLOOM_FILTER_SIZE * 8) / 8), + vData(std::min((unsigned int)(-1 / LN2SQUARED * nElements * log(nFPRate)), MAX_BLOOM_FILTER_SIZE * 8) / 8), /** * The ideal number of hash functions is filter size * ln(2) / number of elements * Again, we ignore filter parameters which will create a bloom filter with more hash functions than the protocol limits * See https://en.wikipedia.org/wiki/Bloom_filter for an explanation of these formulas */ - isFull(false), - isEmpty(false), - nHashFuncs(min((unsigned int)(vData.size() * 8 / nElements * LN2), MAX_HASH_FUNCS)), + nHashFuncs(std::min((unsigned int)(vData.size() * 8 / nElements * LN2), MAX_HASH_FUNCS)), nTweak(nTweakIn), nFlags(nFlagsIn) { } -// Private constructor used by CRollingBloomFilter -CBloomFilter::CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweakIn) : - vData((unsigned int)(-1 / LN2SQUARED * nElements * log(nFPRate)) / 8), - isFull(false), - isEmpty(true), - nHashFuncs((unsigned int)(vData.size() * 8 / nElements * LN2)), - nTweak(nTweakIn), - nFlags(BLOOM_UPDATE_NONE) -{ -} - inline unsigned int CBloomFilter::Hash(unsigned int nHashNum, const std::vector& vDataToHash) const { // 0xFBA4C795 chosen as it guarantees a reasonable bit difference between nHashNum values. return MurmurHash3(nHashNum * 0xFBA4C795 + nTweak, vDataToHash) % (vData.size() * 8); } -void CBloomFilter::insert(const vector& vKey) +void CBloomFilter::insert(const std::vector& vKey) { - if (isFull) + if (vData.empty()) // Avoid divide-by-zero (CVE-2013-5700) return; for (unsigned int i = 0; i < nHashFuncs; i++) { @@ -68,29 +54,26 @@ void CBloomFilter::insert(const vector& vKey) // Sets bit nIndex of vData vData[nIndex >> 3] |= (1 << (7 & nIndex)); } - isEmpty = false; } void CBloomFilter::insert(const COutPoint& outpoint) { CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << outpoint; - vector data(stream.begin(), stream.end()); + std::vector data(stream.begin(), stream.end()); insert(data); } void CBloomFilter::insert(const uint256& hash) { - vector data(hash.begin(), hash.end()); + std::vector data(hash.begin(), hash.end()); insert(data); } -bool CBloomFilter::contains(const vector& vKey) const +bool CBloomFilter::contains(const std::vector& vKey) const { - if (isFull) + if (vData.empty()) // Avoid divide-by-zero (CVE-2013-5700) return true; - if (isEmpty) - return false; for (unsigned int i = 0; i < nHashFuncs; i++) { unsigned int nIndex = Hash(i, vKey); @@ -105,29 +88,16 @@ bool CBloomFilter::contains(const COutPoint& outpoint) const { CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << outpoint; - vector data(stream.begin(), stream.end()); + std::vector data(stream.begin(), stream.end()); return contains(data); } bool CBloomFilter::contains(const uint256& hash) const { - vector data(hash.begin(), hash.end()); + std::vector data(hash.begin(), hash.end()); return contains(data); } -void CBloomFilter::clear() -{ - vData.assign(vData.size(),0); - isFull = false; - isEmpty = true; -} - -void CBloomFilter::reset(unsigned int nNewTweak) -{ - clear(); - nTweak = nNewTweak; -} - bool CBloomFilter::IsWithinSizeConstraints() const { return vData.size() <= MAX_BLOOM_FILTER_SIZE && nHashFuncs <= MAX_HASH_FUNCS; @@ -138,10 +108,8 @@ bool CBloomFilter::IsRelevantAndUpdate(const CTransaction& tx) bool fFound = false; // Match if the filter contains the hash of tx // for finding tx when they appear in a block - if (isFull) + if (vData.empty()) // zero-size = "match-all" filter return true; - if (isEmpty) - return false; const uint256& hash = tx.GetHash(); if (contains(hash)) fFound = true; @@ -151,10 +119,10 @@ bool CBloomFilter::IsRelevantAndUpdate(const CTransaction& tx) const CTxOut& txout = tx.vout[i]; // Match if the filter contains any arbitrary script data element in any scriptPubKey in tx // If this matches, also add the specific output that was matched. - // This means clients don't have to update the filter themselves when a new relevant tx + // This means clients don't have to update the filter themselves when a new relevant tx // is discovered in order to find spending transactions, which avoids round-tripping and race conditions. CScript::const_iterator pc = txout.scriptPubKey.begin(); - vector data; + std::vector data; while (pc < txout.scriptPubKey.end()) { opcodetype opcode; @@ -168,7 +136,7 @@ bool CBloomFilter::IsRelevantAndUpdate(const CTransaction& tx) else if ((nFlags & BLOOM_UPDATE_MASK) == BLOOM_UPDATE_P2PUBKEY_ONLY) { txnouttype type; - vector > vSolutions; + std::vector > vSolutions; if (Solver(txout.scriptPubKey, type, vSolutions) && (type == TX_PUBKEY || type == TX_MULTISIG)) insert(COutPoint(hash, i)); @@ -181,7 +149,7 @@ bool CBloomFilter::IsRelevantAndUpdate(const CTransaction& tx) if (fFound) return true; - BOOST_FOREACH(const CTxIn& txin, tx.vin) + for (const CTxIn& txin : tx.vin) { // Match if the filter contains an outpoint tx spends if (contains(txin.prevout)) @@ -189,7 +157,7 @@ bool CBloomFilter::IsRelevantAndUpdate(const CTransaction& tx) // Match if the filter contains any arbitrary script data element in any scriptSig in tx CScript::const_iterator pc = txin.scriptSig.begin(); - vector data; + std::vector data; while (pc < txin.scriptSig.end()) { opcodetype opcode; @@ -203,70 +171,116 @@ bool CBloomFilter::IsRelevantAndUpdate(const CTransaction& tx) return false; } -void CBloomFilter::UpdateEmptyFull() +CRollingBloomFilter::CRollingBloomFilter(const unsigned int nElements, const double fpRate) { - bool full = true; - bool empty = true; - for (unsigned int i = 0; i < vData.size(); i++) - { - full &= vData[i] == 0xff; - empty &= vData[i] == 0; - } - isFull = full; - isEmpty = empty; + double logFpRate = log(fpRate); + /* The optimal number of hash functions is log(fpRate) / log(0.5), but + * restrict it to the range 1-50. */ + nHashFuncs = std::max(1, std::min((int)round(logFpRate / log(0.5)), 50)); + /* In this rolling bloom filter, we'll store between 2 and 3 generations of nElements / 2 entries. */ + nEntriesPerGeneration = (nElements + 1) / 2; + uint32_t nMaxElements = nEntriesPerGeneration * 3; + /* The maximum fpRate = pow(1.0 - exp(-nHashFuncs * nMaxElements / nFilterBits), nHashFuncs) + * => pow(fpRate, 1.0 / nHashFuncs) = 1.0 - exp(-nHashFuncs * nMaxElements / nFilterBits) + * => 1.0 - pow(fpRate, 1.0 / nHashFuncs) = exp(-nHashFuncs * nMaxElements / nFilterBits) + * => log(1.0 - pow(fpRate, 1.0 / nHashFuncs)) = -nHashFuncs * nMaxElements / nFilterBits + * => nFilterBits = -nHashFuncs * nMaxElements / log(1.0 - pow(fpRate, 1.0 / nHashFuncs)) + * => nFilterBits = -nHashFuncs * nMaxElements / log(1.0 - exp(logFpRate / nHashFuncs)) + */ + nFilterBits = (uint32_t)ceil(-1.0 * nHashFuncs * nMaxElements / log(1.0 - exp(logFpRate / nHashFuncs))); } -CRollingBloomFilter::CRollingBloomFilter(unsigned int nElements, double fpRate) : - b1(nElements * 2, fpRate, 0), b2(nElements * 2, fpRate, 0) -{ - // Implemented using two bloom filters of 2 * nElements each. - // We fill them up, and clear them, staggered, every nElements - // inserted, so at least one always contains the last nElements - // inserted. - nInsertions = 0; - nBloomSize = nElements * 2; - - reset(); +/* Similar to CBloomFilter::Hash */ +static inline uint32_t RollingBloomHash(unsigned int nHashNum, uint32_t nTweak, const std::vector& vDataToHash) { + return MurmurHash3(nHashNum * 0xFBA4C795 + nTweak, vDataToHash); +} + + +// A replacement for x % n. This assumes that x and n are 32bit integers, and x is a uniformly random distributed 32bit value +// which should be the case for a good hash. +// See https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ +static inline uint32_t FastMod(uint32_t x, size_t n) { + return ((uint64_t)x * (uint64_t)n) >> 32; } void CRollingBloomFilter::insert(const std::vector& vKey) { - if (nInsertions == 0) { - b1.clear(); - } else if (nInsertions == nBloomSize / 2) { - b2.clear(); + if (data.empty()) { + initialize(); } - b1.insert(vKey); - b2.insert(vKey); - if (++nInsertions == nBloomSize) { - nInsertions = 0; + if (nEntriesThisGeneration == nEntriesPerGeneration) { + nEntriesThisGeneration = 0; + nGeneration++; + if (nGeneration == 4) { + nGeneration = 1; + } + uint64_t nGenerationMask1 = 0 - (uint64_t)(nGeneration & 1); + uint64_t nGenerationMask2 = 0 - (uint64_t)(nGeneration >> 1); + /* Wipe old entries that used this generation number. */ + for (uint32_t p = 0; p < data.size(); p += 2) { + uint64_t p1 = data[p], p2 = data[p + 1]; + uint64_t mask = (p1 ^ nGenerationMask1) | (p2 ^ nGenerationMask2); + data[p] = p1 & mask; + data[p + 1] = p2 & mask; + } + } + nEntriesThisGeneration++; + + for (int n = 0; n < nHashFuncs; n++) { + uint32_t h = RollingBloomHash(n, nTweak, vKey); + int bit = h & 0x3F; + /* FastMod works with the upper bits of h, so it is safe to ignore that the lower bits of h are already used for bit. */ + uint32_t pos = FastMod(h, data.size()); + /* The lowest bit of pos is ignored, and set to zero for the first bit, and to one for the second. */ + data[pos & ~1] = (data[pos & ~1] & ~(((uint64_t)1) << bit)) | ((uint64_t)(nGeneration & 1)) << bit; + data[pos | 1] = (data[pos | 1] & ~(((uint64_t)1) << bit)) | ((uint64_t)(nGeneration >> 1)) << bit; } } void CRollingBloomFilter::insert(const uint256& hash) { - vector data(hash.begin(), hash.end()); - insert(data); + std::vector vData(hash.begin(), hash.end()); + insert(vData); } bool CRollingBloomFilter::contains(const std::vector& vKey) const { - if (nInsertions < nBloomSize / 2) { - return b2.contains(vKey); + if (data.empty()) { + return false; + } + for (int n = 0; n < nHashFuncs; n++) { + uint32_t h = RollingBloomHash(n, nTweak, vKey); + int bit = h & 0x3F; + uint32_t pos = FastMod(h, data.size()); + /* If the relevant bit is not set in either data[pos & ~1] or data[pos | 1], the filter does not contain vKey */ + if (!(((data[pos & ~1] | data[pos | 1]) >> bit) & 1)) { + return false; + } } - return b1.contains(vKey); + return true; } bool CRollingBloomFilter::contains(const uint256& hash) const { - vector data(hash.begin(), hash.end()); - return contains(data); + std::vector vData(hash.begin(), hash.end()); + return contains(vData); +} + +void CRollingBloomFilter::reset() { + std::vector().swap(data); } -void CRollingBloomFilter::reset() +void CRollingBloomFilter::initialize() { - unsigned int nNewTweak = GetRand(std::numeric_limits::max()); - b1.reset(nNewTweak); - b2.reset(nNewTweak); - nInsertions = 0; + /* For each data element we need to store 2 bits. If both bits are 0, the + * bit is treated as unset. If the bits are (01), (10), or (11), the bit is + * treated as set in generation 1, 2, or 3 respectively. + * These bits are stored in separate integers: position P corresponds to bit + * (P & 63) of the integers data[(P >> 6) * 2] and data[(P >> 6) * 2 + 1]. */ + data.resize(((nFilterBits + 63) / 64) << 1); + + nTweak = GetRand(std::numeric_limits::max()); + nEntriesThisGeneration = 0; + nGeneration = 1; + std::fill(data.begin(), data.end(), 0); } diff --git a/src/bloom.h b/src/bloom.h index 00943610424..46a18fab7f9 100644 --- a/src/bloom.h +++ b/src/bloom.h @@ -1,4 +1,5 @@ // Copyright (c) 2012-2014 The Bitcoin Core developers +// Copyright (c) 2018-2023 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or https://www.opensource.org/licenses/mit-license.php . @@ -33,9 +34,9 @@ enum bloomflags /** * BloomFilter is a probabilistic filter which SPV clients provide * so that we can filter the transactions we send them. - * + * * This allows for significantly more efficient transaction and block downloads. - * + * * Because bloom filters are probabilistic, a SPV node can increase the false- * positive rate, making us send it transactions which aren't actually its, * allowing clients to trade more bandwidth for more privacy by obfuscating which @@ -45,18 +46,12 @@ class CBloomFilter { private: std::vector vData; - bool isFull; - bool isEmpty; unsigned int nHashFuncs; unsigned int nTweak; unsigned char nFlags; unsigned int Hash(unsigned int nHashNum, const std::vector& vDataToHash) const; - // Private constructor for CRollingBloomFilter, no restrictions on size - CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweak); - friend class CRollingBloomFilter; - public: /** * Creates a new bloom filter which will provide the given fp rate when filled with the given number of elements @@ -67,8 +62,8 @@ class CBloomFilter * It should generally always be a random value (and is largely only exposed for unit testing) * nFlags should be one of the BLOOM_UPDATE_* enums (not _MASK) */ - CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweak, unsigned char nFlagsIn); - CBloomFilter() : isFull(true), isEmpty(false), nHashFuncs(0), nTweak(0), nFlags(0) {} + CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweak, unsigned char nFlagsIn); + CBloomFilter() : nHashFuncs(0), nTweak(0), nFlags(0) {} ADD_SERIALIZE_METHODS; @@ -88,18 +83,12 @@ class CBloomFilter bool contains(const COutPoint& outpoint) const; bool contains(const uint256& hash) const; - void clear(); - void reset(unsigned int nNewTweak); - //! True if the size is <= MAX_BLOOM_FILTER_SIZE and the number of hash functions is <= MAX_HASH_FUNCS //! (catch a filter which was just deserialized which was too big) bool IsWithinSizeConstraints() const; //! Also adds any outputs which match the filter to the filter (to match their spending txes) bool IsRelevantAndUpdate(const CTransaction& tx); - - //! Checks for empty and full filters to avoid wasting cpu - void UpdateEmptyFull(); }; /** @@ -110,8 +99,22 @@ class CBloomFilter * reset() is provided, which also changes nTweak to decrease the impact of * false-positives. * - * contains(item) will always return true if item was one of the last N things + * contains(item) will always return true if item was one of the last N to 1.5*N * insert()'ed ... but may also return true for items that were not inserted. + * + * It needs around 1.8 bytes per element per factor 0.1 of false positive rate. + * For example, if we want 1000 elements, we'd need: + * - ~1800 bytes for a false positive rate of 0.1 + * - ~3600 bytes for a false positive rate of 0.01 + * - ~5400 bytes for a false positive rate of 0.001 + * + * If we make these simplifying assumptions: + * - logFpRate / log(0.5) doesn't get rounded or clamped in the nHashFuncs calculation + * - nElements is even, so that nEntriesPerGeneration == nElements / 2 + * + * Then we get a more accurate estimate for filter bytes: + * + * 3/(log(256)*log(2)) * log(1/fpRate) * nElements */ class CRollingBloomFilter { @@ -119,7 +122,7 @@ class CRollingBloomFilter // A random bloom filter calls GetRand() at creation time. // Don't create global CRollingBloomFilter objects, as they may be // constructed before the randomizer is properly initialized. - CRollingBloomFilter(unsigned int nElements, double nFPRate); + CRollingBloomFilter(const unsigned int nElements=5000, const double nFPRate=0.001); void insert(const std::vector& vKey); void insert(const uint256& hash); @@ -128,11 +131,19 @@ class CRollingBloomFilter void reset(); +protected: + bool is_data_empty() const { return data.empty(); } + private: - unsigned int nBloomSize; - unsigned int nInsertions; - CBloomFilter b1, b2; -}; + void initialize(); + uint32_t nFilterBits; + int nEntriesPerGeneration; + int nEntriesThisGeneration; + int nGeneration; + std::vector data; + unsigned int nTweak; + int nHashFuncs; +}; #endif // BITCOIN_BLOOM_H diff --git a/src/crypto/chacha20.cpp b/src/crypto/chacha20.cpp new file mode 100644 index 00000000000..c9d88fb7f94 --- /dev/null +++ b/src/crypto/chacha20.cpp @@ -0,0 +1,181 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Copyright (c) 2021-2023 The Zcash developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +// Based on the public domain implementation 'merged' by D. J. Bernstein +// See https://cr.yp.to/chacha.html. + +#include "crypto/common.h" +#include "crypto/chacha20.h" + +#include + +constexpr static inline uint32_t rotl32(uint32_t v, int c) { return (v << c) | (v >> (32 - c)); } + +#define QUARTERROUND(a,b,c,d) \ + a += b; d = rotl32(d ^ a, 16); \ + c += d; b = rotl32(b ^ c, 12); \ + a += b; d = rotl32(d ^ a, 8); \ + c += d; b = rotl32(b ^ c, 7); + +static const unsigned char sigma[] = "expand 32-byte k"; +static const unsigned char tau[] = "expand 16-byte k"; + +void ChaCha20::SetKey(const unsigned char* k, size_t keylen) +{ + const unsigned char *constants; + + input[4] = ReadLE32(k + 0); + input[5] = ReadLE32(k + 4); + input[6] = ReadLE32(k + 8); + input[7] = ReadLE32(k + 12); + if (keylen == 32) { /* recommended */ + k += 16; + constants = sigma; + } else { /* keylen == 16 */ + constants = tau; + } + input[8] = ReadLE32(k + 0); + input[9] = ReadLE32(k + 4); + input[10] = ReadLE32(k + 8); + input[11] = ReadLE32(k + 12); + input[0] = ReadLE32(constants + 0); + input[1] = ReadLE32(constants + 4); + input[2] = ReadLE32(constants + 8); + input[3] = ReadLE32(constants + 12); + input[12] = 0; + input[13] = 0; + input[14] = 0; + input[15] = 0; +} + +ChaCha20::ChaCha20() +{ + memset(input, 0, sizeof(input)); +} + +ChaCha20::ChaCha20(const unsigned char* k, size_t keylen) +{ + SetKey(k, keylen); +} + +void ChaCha20::SetIV(uint64_t iv) +{ + input[14] = iv; + input[15] = iv >> 32; +} + +void ChaCha20::Seek(uint64_t pos) +{ + input[12] = pos; + input[13] = pos >> 32; +} + +void ChaCha20::Output(unsigned char* c, size_t bytes) +{ + uint32_t x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15; + uint32_t j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15; + unsigned char *ctarget = NULL; + unsigned char tmp[64]; + unsigned int i; + + if (!bytes) return; + + j0 = input[0]; + j1 = input[1]; + j2 = input[2]; + j3 = input[3]; + j4 = input[4]; + j5 = input[5]; + j6 = input[6]; + j7 = input[7]; + j8 = input[8]; + j9 = input[9]; + j10 = input[10]; + j11 = input[11]; + j12 = input[12]; + j13 = input[13]; + j14 = input[14]; + j15 = input[15]; + + for (;;) { + if (bytes < 64) { + ctarget = c; + c = tmp; + } + x0 = j0; + x1 = j1; + x2 = j2; + x3 = j3; + x4 = j4; + x5 = j5; + x6 = j6; + x7 = j7; + x8 = j8; + x9 = j9; + x10 = j10; + x11 = j11; + x12 = j12; + x13 = j13; + x14 = j14; + x15 = j15; + for (i = 20;i > 0;i -= 2) { + QUARTERROUND( x0, x4, x8,x12) + QUARTERROUND( x1, x5, x9,x13) + QUARTERROUND( x2, x6,x10,x14) + QUARTERROUND( x3, x7,x11,x15) + QUARTERROUND( x0, x5,x10,x15) + QUARTERROUND( x1, x6,x11,x12) + QUARTERROUND( x2, x7, x8,x13) + QUARTERROUND( x3, x4, x9,x14) + } + x0 += j0; + x1 += j1; + x2 += j2; + x3 += j3; + x4 += j4; + x5 += j5; + x6 += j6; + x7 += j7; + x8 += j8; + x9 += j9; + x10 += j10; + x11 += j11; + x12 += j12; + x13 += j13; + x14 += j14; + x15 += j15; + + ++j12; + if (!j12) ++j13; + + WriteLE32(c + 0, x0); + WriteLE32(c + 4, x1); + WriteLE32(c + 8, x2); + WriteLE32(c + 12, x3); + WriteLE32(c + 16, x4); + WriteLE32(c + 20, x5); + WriteLE32(c + 24, x6); + WriteLE32(c + 28, x7); + WriteLE32(c + 32, x8); + WriteLE32(c + 36, x9); + WriteLE32(c + 40, x10); + WriteLE32(c + 44, x11); + WriteLE32(c + 48, x12); + WriteLE32(c + 52, x13); + WriteLE32(c + 56, x14); + WriteLE32(c + 60, x15); + + if (bytes <= 64) { + if (bytes < 64) { + for (i = 0;i < bytes;++i) ctarget[i] = c[i]; + } + input[12] = j12; + input[13] = j13; + return; + } + bytes -= 64; + c += 64; + } +} diff --git a/src/crypto/chacha20.h b/src/crypto/chacha20.h new file mode 100644 index 00000000000..e8f8f687571 --- /dev/null +++ b/src/crypto/chacha20.h @@ -0,0 +1,27 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Copyright (c) 2021-2023 The Zcash developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_CHACHA20_H +#define BITCOIN_CRYPTO_CHACHA20_H + +#include +#include + +/** A PRNG class for ChaCha20. */ +class ChaCha20 +{ +private: + uint32_t input[16]; + +public: + ChaCha20(); + ChaCha20(const unsigned char* key, size_t keylen); + void SetKey(const unsigned char* key, size_t keylen); + void SetIV(uint64_t iv); + void Seek(uint64_t pos); + void Output(unsigned char* output, size_t bytes); +}; + +#endif // BITCOIN_CRYPTO_CHACHA20_H diff --git a/src/crypto/common.h b/src/crypto/common.h index fd62087b7f0..5a3aaef2449 100644 --- a/src/crypto/common.h +++ b/src/crypto/common.h @@ -1,4 +1,5 @@ // Copyright (c) 2014 The Bitcoin Core developers +// Copyright (c) 2016-2023 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or https://www.opensource.org/licenses/mit-license.php . @@ -123,4 +124,25 @@ int inline init_and_check_sodium() return 0; } +/** Return the smallest number n such that (x >> n) == 0 (or 64 if the highest bit in x is set. */ +uint64_t static inline CountBits(uint64_t x) +{ +#ifdef HAVE_DECL___BUILTIN_CLZL + if (sizeof(unsigned long) >= sizeof(uint64_t)) { + return x ? 8 * sizeof(unsigned long) - __builtin_clzl(x) : 0; + } +#endif +#ifdef HAVE_DECL___BUILTIN_CLZLL + if (sizeof(unsigned long long) >= sizeof(uint64_t)) { + return x ? 8 * sizeof(unsigned long long) - __builtin_clzll(x) : 0; + } +#endif + int ret = 0; + while (x) { + x >>= 1; + ++ret; + } + return ret; +} + #endif // BITCOIN_CRYPTO_COMMON_H diff --git a/src/init.cpp b/src/init.cpp index d677476373f..5be0e52578f 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -386,8 +386,8 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-addressindex", strprintf(_("Maintain a full address index, used to query for the balance, txids and unspent outputs for addresses (default: %u)"), DEFAULT_ADDRESSINDEX)); strUsage += HelpMessageOpt("-timestampindex", strprintf(_("Maintain a timestamp index for block hashes, used to query blocks hashes by a range of timestamps (default: %u)"), DEFAULT_TIMESTAMPINDEX)); strUsage += HelpMessageOpt("-spentindex", strprintf(_("Maintain a full spent index, used to query the spending txid and input index for an outpoint (default: %u)"), DEFAULT_SPENTINDEX)); - strUsage += HelpMessageOpt("-alwayssubmitnotarizations", strprintf(_("Submit notarizations to notary chain whenevever merge mining/staking and eligible (default = false, only as needed)"), DEFAULT_SPENTINDEX)); - strUsage += HelpMessageOpt("-allowdelayednotarizations", strprintf(_("Do not notarize in order to prevent slower notarizations (default = false, notarize to prevent slowing down)"), DEFAULT_SPENTINDEX)); + strUsage += HelpMessageOpt("-alwayssubmitnotarizations", strprintf(_("Submit notarizations to notary chain whenevever merge mining/staking and eligible (default = %u, only as needed)"), DEFAULT_SPENTINDEX)); + strUsage += HelpMessageOpt("-allowdelayednotarizations", strprintf(_("Do not notarize in order to prevent slower notarizations (default = %u, notarize to prevent slowing down)"), DEFAULT_SPENTINDEX)); strUsage += HelpMessageGroup(_("Connection options:")); strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open")); strUsage += HelpMessageOpt("-banscore=", strprintf(_("Threshold for disconnecting misbehaving peers (default: %u)"), 100)); @@ -526,7 +526,6 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-blocktime=", strprintf(_("Set target block time (in seconds) for difficulty adjustment (default: %d)"), CCurrencyDefinition::DEFAULT_BLOCKTIME_TARGET)); strUsage += HelpMessageOpt("-powaveragingwindow=", strprintf(_("Set averaging window for PoW difficulty adjustment, in blocks (default: %d)"), CCurrencyDefinition::DEFAULT_AVERAGING_WINDOW)); strUsage += HelpMessageOpt("-notarizationperiod=", strprintf(_("Set minimum spacing consensus between cross-chain notarization, in blocks (default: %d, min 10 min)"), CCurrencyDefinition::BLOCK_NOTARIZATION_MODULO)); - strUsage += HelpMessageOpt("-alwayssubmitnotarizations=", strprintf(_("Always submit cross-chain notarizations when allowed, even when not necessary (default: false)"))); strUsage += HelpMessageOpt("-testnet", _("loads PBaaS network in testmode")); strUsage += HelpMessageGroup(_("Node relay options:")); diff --git a/src/main.cpp b/src/main.cpp index 51e0aafac4c..3e33750be18 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -336,7 +336,7 @@ namespace { int64_t GetBlockTimeout(int64_t nTime, int nValidatedQueuedBefore, const Consensus::Params &consensusParams) { //return nTime + 500000 * consensusParams.PoWTargetSpacing(nHeight) * (4 + nValidatedQueuedBefore); - return nTime + 500000 * consensusParams.nPowTargetSpacing * (4 + nValidatedQueuedBefore); + return nTime + 500000 * ConnectedChains.ThisChain().blockTime * (4 + nValidatedQueuedBefore); } void InitializeNode(NodeId nodeid, const CNode *pnode) { @@ -3576,7 +3576,7 @@ void PartitionCheck(bool (*initialDownloadCheck)(const CChainParams&), Consensus::Params consensusParams = Params().GetConsensus(); - int BLOCKS_EXPECTED = SPAN_SECONDS / consensusParams.nPowTargetSpacing; + int BLOCKS_EXPECTED = SPAN_SECONDS / ConnectedChains.ThisChain().blockTime; LOCK(cs); @@ -5640,7 +5640,7 @@ bool ActivateBestChain(CValidationState& state, const CChainParams& chainparams, LOCK(cs_vNodes); BOOST_FOREACH(CNode* pnode, vNodes) if (chainHeight > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : nBlockEstimate)) - pnode->PushInventory(CInv(MSG_BLOCK, hashNewTip)); + pnode->PushBlockInventory(hashNewTip); } // Notify external listeners about the new tip. GetMainSignals().UpdatedBlockTip(pindexNewTip); @@ -7997,10 +7997,16 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam } else // MSG_FILTERED_BLOCK) { - LOCK(pfrom->cs_filter); - if (pfrom->pfilter) + bool send = false; + CMerkleBlock merkleBlock; { - CMerkleBlock merkleBlock(block, *pfrom->pfilter); + LOCK(pfrom->cs_filter); + if (pfrom->pfilter) { + send = true; + merkleBlock = CMerkleBlock(block, *pfrom->pfilter); + } + } + if (send) { pfrom->PushMessage("merkleblock", merkleBlock); // CMerkleBlock just contains hashes, so also push any transactions in the block the client did not see // This avoids hurting performance by pointlessly requiring a round-trip @@ -8009,8 +8015,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // Thus, the protocol spec specified allows for us to provide duplicate txn here, // however we MUST always provide at least what the remote peer needs typedef std::pair PairType; - BOOST_FOREACH(PairType& pair, merkleBlock.vMatchedTxn) - if (!pfrom->setInventoryKnown.count(CInv(MSG_TX, pair.second))) + for (PairType& pair : merkleBlock.vMatchedTxn) pfrom->PushMessage("tx", block.vtx[pair.first]); } // else @@ -8033,35 +8038,40 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam else if (inv.IsKnownType()) { LogPrint("getdata", "%s: inv 3 %d\n", __func__, inv.type); - - // Check the mempool to see if a transaction is expiring soon. If so, do not send to peer. - // Note that a transaction enters the mempool first, before the serialized form is cached - // in mapRelay after a successful relay. - bool isExpiringSoon = false; bool pushed = false; - CTransaction tx; - bool isInMempool = mempool.lookup(inv.hash, tx); - if (isInMempool) { - isExpiringSoon = IsExpiringSoonTx(tx, currentHeight + 1); - } - if (!isExpiringSoon) { - // Send stream from relay memory + if (inv.type == MSG_TX) + { + // Check the mempool to see if a transaction is expiring soon. If so, do not send to peer. + // Note that a transaction enters the mempool first, before the serialized form is cached + // in mapRelay after a successful relay. + bool isExpiringSoon = false; + auto txinfo = mempool.info(inv.hash); + bool isInMempool = txinfo.tx ? true : false; + if (isInMempool) { - LOCK(cs_mapRelay); - map::iterator mi = mapRelay.find(inv); - if (mi != mapRelay.end()) { - pfrom->PushMessage(inv.GetCommand(), (*mi).second); - pushed = true; - } + isExpiringSoon = IsExpiringSoonTx(*txinfo.tx, currentHeight + 1); } - if (!pushed && inv.type == MSG_TX) { - if (isInMempool) { - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); - ss.reserve(1000); - ss << tx; - pfrom->PushMessage("tx", ss); - pushed = true; + + if (!isExpiringSoon) { + // Send stream from relay memory + MapRelay::iterator mi; + { + LOCK(cs_mapRelay); + mi = mapRelay.find(inv.hash); + if (mi != mapRelay.end()) { + pfrom->PushMessage(inv.GetCommand(), *(*mi).second); + pushed = true; + } + } + if (!pushed && inv.type == MSG_TX) { + if (isInMempool) { + CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); + ss.reserve(1000); + ss << *(*mi).second; + pfrom->PushMessage("tx", ss); + pushed = true; + } } } } @@ -8071,9 +8081,6 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam } } - // Track requests for our stuff. - GetMainSignals().Inventory(inv.hash); - if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK) break; } @@ -8230,6 +8237,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, { pfrom->PushMessage("getaddr"); pfrom->fGetAddr = true; + + // When requesting a getaddr, accept an additional MAX_ADDR_TO_SEND addresses in response + // (bypassing the MAX_ADDR_PROCESSING_TOKEN_BUCKET limit). + pfrom->m_addr_token_bucket += MAX_ADDR_TO_SEND; } addrman.Good(pfrom->addr); } else { @@ -8350,13 +8361,41 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, vector vAddrOk; int64_t nNow = GetAdjustedTime(); int64_t nSince = nNow - 10 * 60; - BOOST_FOREACH(CAddress& addr, vAddr) + + // Update/increment addr rate limiting bucket. + const int64_t current_time = GetTimeMicros(); + if (pfrom->m_addr_token_bucket < MAX_ADDR_PROCESSING_TOKEN_BUCKET) { + // Don't increment bucket if it's already full + const auto time_diff = std::max(current_time - pfrom->m_addr_token_timestamp, (int64_t) 0); + const double increment = (time_diff / 1000000) * MAX_ADDR_RATE_PER_SECOND; + pfrom->m_addr_token_bucket = std::min(pfrom->m_addr_token_bucket + increment, MAX_ADDR_PROCESSING_TOKEN_BUCKET); + } + pfrom->m_addr_token_timestamp = current_time; + + uint64_t num_proc = 0; + uint64_t num_rate_limit = 0; + std::shuffle(vAddr.begin(), vAddr.end(), ZcashRandomEngine()); + + for (CAddress& addr : vAddr) { boost::this_thread::interruption_point(); + // Apply rate limiting if the address is not whitelisted + if (pfrom->m_addr_token_bucket < 1.0) { + if (!pfrom->IsWhitelistedRange(addr)) { + ++num_rate_limit; + continue; + } + } else { + pfrom->m_addr_token_bucket -= 1.0; + } + if (addr.nTime <= 100000000 || addr.nTime > nNow + 10 * 60) addr.nTime = nNow - 5 * 24 * 60 * 60; - pfrom->AddAddressKnown(addr); + + pfrom->AddAddressIfNotAlreadyKnown(addr); + ++num_proc; + bool fReachable = IsReachable(addr); if (addr.nTime > nSince && !pfrom->fGetAddr && vAddr.size() <= 10 && addr.IsRoutable()) { @@ -8391,6 +8430,16 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (fReachable) vAddrOk.push_back(addr); } + + pfrom->m_addr_processed += num_proc; + pfrom->m_addr_rate_limited += num_rate_limit; + LogPrint("net", "ProcessMessage: Received addr: %u addresses (%u processed, %u rate-limited) from peer=%d%s\n", + vAddr.size(), + num_proc, + num_rate_limit, + pfrom->GetId(), + fLogIPs ? ", peeraddr=" + pfrom->addr.ToString() : ""); + addrman.Add(vAddrOk, pfrom->addr, 2 * 60 * 60); if (vAddr.size() < 1000) pfrom->fGetAddr = false; @@ -8409,6 +8458,13 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, return error("message inv size() = %u", vInv.size()); } + bool fBlocksOnly = GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY); + // Allow whitelisted peers to send data other than blocks in blocks only mode if whitelistrelay is true + if (pfrom->fWhitelisted && GetBoolArg("-whitelistrelay", DEFAULT_WHITELISTRELAY)) + { + fBlocksOnly = false; + } + LOCK(cs_main); std::vector vToFetch; @@ -8418,7 +8474,6 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, const CInv &inv = vInv[nInv]; boost::this_thread::interruption_point(); - pfrom->AddInventoryKnown(inv); bool fAlreadyHave = AlreadyHave(inv); LogPrint("net", "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom->id); @@ -8440,7 +8495,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, pfrom->PushMessage("getheaders", chainActive.GetLocator(pindexBestHeader), inv.hash); CNodeState *nodestate = State(pfrom->GetId()); - if (chainActive.Tip()->GetBlockTime() > GetAdjustedTime() - chainparams.GetConsensus().PoWTargetSpacing(pindexBestHeader->GetHeight()) * 20 && + if (chainActive.Tip()->GetBlockTime() > GetAdjustedTime() - (ConnectedChains.ThisChain().blockTime * 20) && nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { vToFetch.push_back(inv); // Mark block as in flight already, even though the actual "getdata" message only goes out @@ -8450,9 +8505,14 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, LogPrint("net", "getheaders (%d) %s to peer=%d\n", pindexBestHeader->GetHeight(), inv.hash.ToString(), pfrom->id); } } - - // Track requests for our stuff - GetMainSignals().Inventory(inv.hash); + else + { + pfrom->AddKnownWTxId(WTxId(inv.hash)); + if (fBlocksOnly) + LogPrint("net", "transaction (%s) inv sent in violation of protocol peer=%d\n", inv.hash.ToString(), pfrom->id); + else if (!fAlreadyHave && !IsInitialBlockDownload(Params())) + pfrom->AskFor(inv); + } if (pfrom->nSendSize > (SendBufferSize() * 2)) { Misbehaving(pfrom->GetId(), 50); @@ -8513,13 +8573,13 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } // If pruning, don't inv blocks unless we have on disk and are likely to still have // for some reasonable time window (1 hour) that block relay might require. - const int nPrunedBlocksLikelyToHave = MIN_BLOCKS_TO_KEEP - 3600 / chainparams.GetConsensus().PoWTargetSpacing(pindex->GetHeight()); + const int nPrunedBlocksLikelyToHave = MIN_BLOCKS_TO_KEEP - 3600 / ConnectedChains.ThisChain().blockTime; if (fPruneMode && (!(pindex->nStatus & BLOCK_HAVE_DATA) || pindex->GetHeight() <= chainActive.Tip()->GetHeight() - nPrunedBlocksLikelyToHave)) { LogPrint("net", " getblocks stopping, pruned or too old block at %d %s\n", pindex->GetHeight(), pindex->GetBlockHash().ToString()); break; } - pfrom->PushInventory(CInv(MSG_BLOCK, pindex->GetBlockHash())); + pfrom->PushBlockInventory(pindex->GetBlockHash()); if (--nLimit <= 0) { // When this block is requested, we'll send an inv that'll @@ -8589,16 +8649,28 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, else if (strCommand == "tx") { + // Stop processing the transaction early if + // We are in blocks only mode and peer is either not whitelisted or whitelistrelay is off + if (GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY) && (!pfrom->fWhitelisted || !GetBoolArg("-whitelistrelay", DEFAULT_WHITELISTRELAY))) + { + LogPrint("net", "transaction sent in violation of protocol peer=%d\n", pfrom->id); + return true; + } + vector vWorkQueue; vector vEraseQueue; CTransaction tx; vRecv >> tx; + const uint256& txid = tx.GetHash(); + const WTxId& wtxid = WTxId(tx.GetHash()); + CInv inv(MSG_TX, tx.GetHash()); - pfrom->AddInventoryKnown(inv); LOCK(cs_main); + pfrom->AddKnownWTxId(wtxid); + bool fMissingInputs = false; CValidationState state; @@ -8841,8 +8913,6 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, CInv inv(MSG_BLOCK, block.GetHash()); LogPrint("net", "received block %s peer=%d\n", inv.hash.ToString(), pfrom->id); - pfrom->AddInventoryKnown(inv); - CValidationState state; // Process all blocks from whitelisted peers, even if not requested, // unless we're still syncing with the network. @@ -8958,7 +9028,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (pingUsecTime > 0) { // Successful ping time measurement, replace previous pfrom->nPingUsecTime = pingUsecTime; - pfrom->nMinPingUsecTime = std::min(pfrom->nMinPingUsecTime, pingUsecTime); + pfrom->nMinPingUsecTime = std::min((int64_t)pfrom->nMinPingUsecTime, pingUsecTime); } else { // This should never happen sProblem = "Timing mishap"; @@ -9054,9 +9124,8 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, LOCK(pfrom->cs_filter); delete pfrom->pfilter; pfrom->pfilter = new CBloomFilter(filter); - pfrom->pfilter->UpdateEmptyFull(); + pfrom->fRelayTxes = true; } - pfrom->fRelayTxes = true; } @@ -9237,6 +9306,22 @@ bool ProcessMessages(CNode* pfrom) return fOk; } +class CompareInvMempoolOrder +{ + CTxMemPool *mp; +public: + CompareInvMempoolOrder(CTxMemPool *mempool) + { + mp = mempool; + } + + bool operator()(std::set::iterator a, std::set::iterator b) + { + /* As std::make_heap produces a max-heap, we want the entries with the + * highest feerate to sort later. */ + return mp->CompareDepthAndScore(*b, *a); + } +}; bool SendMessages(CNode* pto, bool fSendTrickle) { @@ -9281,38 +9366,24 @@ bool SendMessages(CNode* pto, bool fSendTrickle) return true; // Address refresh broadcast - static int64_t nLastRebroadcast; - if (!IsInitialBlockDownload(chainParams) && (GetTime() - nLastRebroadcast > 24 * 60 * 60)) + int64_t nNow = GetTimeMicros(); + if (!IsInitialBlockDownload(Params()) && pto->nNextLocalAddrSend < nNow) { - TRY_LOCK(cs_vNodes, lockNodes); - if (lockNodes) - { - BOOST_FOREACH(CNode* pnode, vNodes) - { - // Periodically clear addrKnown to allow refresh broadcasts - if (nLastRebroadcast) - pnode->addrKnown.reset(); - - // Rebroadcast our address - AdvertizeLocal(pnode); - } - if (!vNodes.empty()) - nLastRebroadcast = GetTime(); - } + AdvertizeLocal(pto); + pto->nNextLocalAddrSend = PoissonNextSend(nNow, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); } // // Message: addr // - if (fSendTrickle) - { + if (pto->nNextAddrSend < nNow) { + pto->nNextAddrSend = PoissonNextSend(nNow, AVG_ADDRESS_BROADCAST_INTERVAL); vector vAddr; vAddr.reserve(pto->vAddrToSend.size()); - BOOST_FOREACH(const CAddress& addr, pto->vAddrToSend) + for (const CAddress& addr : pto->vAddrToSend) { - if (!pto->addrKnown.contains(addr.GetKey())) + if (pto->AddAddressIfNotAlreadyKnown(addr)) { - pto->addrKnown.insert(addr.GetKey()); vAddr.push_back(addr); // receiver rejects addr messages larger than 1000 if (vAddr.size() >= 1000) @@ -9374,52 +9445,108 @@ bool SendMessages(CNode* pto, bool fSendTrickle) // Message: inventory // vector vInv; - vector vInvWait; { LOCK(pto->cs_inventory); - vInv.reserve(pto->vInventoryToSend.size()); - vInvWait.reserve(pto->vInventoryToSend.size()); - BOOST_FOREACH(const CInv& inv, pto->vInventoryToSend) - { - if (pto->setInventoryKnown.count(inv)) - continue; + // Avoid possibly adding to pto->filterInventoryKnown after it has been reset in CloseSocketDisconnect. + if (pto->fDisconnect) { + // We can safely return here because SendMessages would, in any case, do nothing after + // this block if pto->fDisconnect is set. + return true; + } + vInv.reserve(std::max(pto->vInventoryBlockToSend.size(), INVENTORY_BROADCAST_MAX)); - // trickle out tx inv to protect privacy - if (inv.type == MSG_TX && !fSendTrickle) - { - // 1/4 of tx invs blast to all immediately - static uint256 hashSalt; - if (hashSalt.IsNull()) - hashSalt = GetRandHash(); - uint256 hashRand = ArithToUint256(UintToArith256(inv.hash) ^ UintToArith256(hashSalt)); - hashRand = Hash(BEGIN(hashRand), END(hashRand)); - bool fTrickleWait = ((UintToArith256(hashRand) & 3) != 0); + // Add blocks + for (const uint256& hash : pto->vInventoryBlockToSend) { + vInv.push_back(CInv(MSG_BLOCK, hash)); + if (vInv.size() == MAX_INV_SZ) { + pto->PushMessage("inv", vInv); + vInv.clear(); + } + } + pto->vInventoryBlockToSend.clear(); - if (fTrickleWait) - { - vInvWait.push_back(inv); + // Check whether periodic sends should happen + fSendTrickle = fSendTrickle || pto->fWhitelisted; + if (pto->nNextInvSend < nNow) { + fSendTrickle = true; + // Use half the delay for outbound peers, as there is less privacy concern for them. + pto->nNextInvSend = PoissonNextSend(nNow, INVENTORY_BROADCAST_INTERVAL >> !pto->fInbound); + } + + // Time to send but the peer has requested we not relay transactions. + if (fSendTrickle) { + LOCK(pto->cs_filter); + if (!pto->fRelayTxes) pto->setInventoryTxToSend.clear(); + } + + uint32_t currentHeight = GetHeight(); + + // Determine transactions to relay + if (fSendTrickle) { + // Produce a vector with all candidates for sending + vector::iterator> vInvTx; + vInvTx.reserve(pto->setInventoryTxToSend.size()); + for (std::set::iterator it = pto->setInventoryTxToSend.begin(); it != pto->setInventoryTxToSend.end(); it++) { + vInvTx.push_back(it); + } + // Sort the inventory we send for privacy and priority reasons. + // A heap is used so that not all items need sorting if only a few are being sent. + CompareInvMempoolOrder compareInvMempoolOrder(&mempool); + std::make_heap(vInvTx.begin(), vInvTx.end(), compareInvMempoolOrder); + // No reason to drain out at many times the network's capacity, + // especially since we have many peers and some will draw much shorter delays. + unsigned int nRelayedTransactions = 0; + LOCK(pto->cs_filter); + while (!vInvTx.empty() && nRelayedTransactions < INVENTORY_BROADCAST_MAX) { + // Fetch the top element from the heap + std::pop_heap(vInvTx.begin(), vInvTx.end(), compareInvMempoolOrder); + std::set::iterator it = vInvTx.back(); + vInvTx.pop_back(); + uint256 hash = *it; + // Remove it from the to-be-sent set + pto->setInventoryTxToSend.erase(it); + // Check if not in the filter already + if (pto->HasKnownTxId(hash)) { continue; } - } + // Not in the mempool anymore? don't bother sending it. + CTransaction txForInv; + if (!mempool.lookup(hash, txForInv)) { + continue; + } + CInv inv(MSG_TX, hash); - // returns true if wasn't already contained in the set - if (pto->setInventoryKnown.insert(inv).second) - { + if (IsExpiringSoonTx(txForInv, currentHeight + 1)) continue; + if (pto->pfilter && !pto->pfilter->IsRelevantAndUpdate(txForInv)) continue; + + // Send vInv.push_back(inv); - if (vInv.size() >= 1000) + nRelayedTransactions++; { + // Expire old relay messages + while (!vRelayExpiration.empty() && vRelayExpiration.front().first < nNow) + { + mapRelay.erase(vRelayExpiration.front().second); + vRelayExpiration.pop_front(); + } + + auto ret = mapRelay.insert(std::make_pair(inv.hash, std::make_shared(txForInv))); + if (ret.second) { + vRelayExpiration.push_back(std::make_pair(nNow + 15 * 60 * 1000000, ret.first)); + } + } + if (vInv.size() == MAX_INV_SZ) { pto->PushMessage("inv", vInv); vInv.clear(); } + pto->AddKnownTxId(hash); } } - pto->vInventoryToSend = vInvWait; } if (!vInv.empty()) pto->PushMessage("inv", vInv); // Detect whether we're stalling - int64_t nNow = GetTimeMicros(); if (!pto->fDisconnect && state.nStallingSince && state.nStallingSince < nNow - 1000000 * BLOCK_STALLING_TIMEOUT) { // Stalling only triggers when the block download window cannot move. During normal steady state, // the download window should be much larger than the to-be-downloaded set of blocks, so disconnection @@ -9453,17 +9580,19 @@ bool SendMessages(CNode* pto, bool fSendTrickle) // // Message: getdata (blocks) // - static uint256 zero; + // + // Message: getdata (blocks) + // vector vGetData; - if (!pto->fDisconnect && !pto->fClient && (fFetch || !IsInitialBlockDownload(chainParams)) && state.nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (!pto->fDisconnect && !pto->fClient && (fFetch || !IsInitialBlockDownload(Params())) && state.nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { vector vToDownload; NodeId staller = -1; FindNextBlocksToDownload(pto->GetId(), MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller); - BOOST_FOREACH(CBlockIndex *pindex, vToDownload) { + for (CBlockIndex *pindex : vToDownload) { vGetData.push_back(CInv(MSG_BLOCK, pindex->GetBlockHash())); - MarkBlockAsInFlight(pto->GetId(), pindex->GetBlockHash(), consensusParams, pindex); + MarkBlockAsInFlight(pto->GetId(), pindex->GetBlockHash(), Params().GetConsensus(), pindex); LogPrint("net", "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), - pindex->GetHeight(), pto->id); + pindex->GetHeight(), pto->id); } if (state.nBlocksInFlight == 0 && staller != -1) { if (State(staller)->nStallingSince == 0) { @@ -9472,20 +9601,6 @@ bool SendMessages(CNode* pto, bool fSendTrickle) } } } - /*CBlockIndex *pindex; - if ( komodo_requestedhash != zero && komodo_requestedcount < 16 && (pindex= mapBlockIndex[komodo_requestedhash]) != 0 ) - { - LogPrint("net","komodo_requestedhash.%d request %s to nodeid.%d\n",komodo_requestedcount,komodo_requestedhash.ToString().c_str(),pto->GetId()); - fprintf(stderr,"komodo_requestedhash.%d request %s to nodeid.%d\n",komodo_requestedcount,komodo_requestedhash.ToString().c_str(),pto->GetId()); - vGetData.push_back(CInv(MSG_BLOCK, komodo_requestedhash)); - MarkBlockAsInFlight(pto->GetId(), komodo_requestedhash, consensusParams, pindex); - komodo_requestedcount++; - if ( komodo_requestedcount > 16 ) - { - memset(&komodo_requestedhash,0,sizeof(komodo_requestedhash)); - komodo_requestedcount = 0; - } - }*/ // // Message: getdata (non-blocks) @@ -9520,8 +9635,6 @@ std::string CBlockFileInfo::ToString() const { return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, DateTimeStrFormat("%Y-%m-%d", nTimeFirst), DateTimeStrFormat("%Y-%m-%d", nTimeLast)); } - - static class CMainCleanup { public: diff --git a/src/main.h b/src/main.h index ff5c922d2cb..8b679797bb5 100644 --- a/src/main.h +++ b/src/main.h @@ -111,6 +111,20 @@ static const unsigned int DATABASE_FLUSH_INTERVAL = 24 * 60 * 60; /** Maximum length of reject messages. */ static const unsigned int MAX_REJECT_MESSAGE_LENGTH = 111; static const int64_t DEFAULT_MAX_TIP_AGE = 24 * 60 * 60; +/** Default for DEFAULT_WHITELISTRELAY. */ +static const bool DEFAULT_WHITELISTRELAY = true; +/** Default for DEFAULT_WHITELISTFORCERELAY. */ +static const bool DEFAULT_WHITELISTFORCERELAY = true; +/** Average delay between local address broadcasts in seconds. */ +static const unsigned int AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL = 24 * 24 * 60; +/** Average delay between peer address broadcasts in seconds. */ +static const unsigned int AVG_ADDRESS_BROADCAST_INTERVAL = 30; +/** Average delay between trickled inventory transmissions in seconds. + * Blocks and whitelisted receivers bypass this, outbound peers get half this delay. */ +static const unsigned int INVENTORY_BROADCAST_INTERVAL = 5; +/** Maximum number of inventory items to send per transmission. + * Limits the impact of low-fee transaction floods. */ +static const unsigned int INVENTORY_BROADCAST_MAX = 7 * INVENTORY_BROADCAST_INTERVAL; //static const bool DEFAULT_ADDRESSINDEX = false; //static const bool DEFAULT_SPENTINDEX = false; diff --git a/src/memusage.h b/src/memusage.h index 87e6525578c..c06724d5a06 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -70,6 +70,15 @@ struct stl_tree_node X x; }; +struct stl_shared_counter +{ + /* Various platforms use different sized counters here. + * Conservatively assume that they won't be larger than size_t. */ + void* class_type; + size_t use_count; + size_t weak_count; +}; + template static inline size_t DynamicUsage(const std::vector& v) { @@ -82,18 +91,45 @@ static inline size_t DynamicUsage(const prevector& v) return MallocUsage(v.allocated_memory()); } -template -static inline size_t DynamicUsage(const std::set& s) +template +static inline size_t DynamicUsage(const std::set& s) { return MallocUsage(sizeof(stl_tree_node)) * s.size(); } template -static inline size_t DynamicUsage(const std::map& m) +static inline size_t IncrementalDynamicUsage(const std::set& s) +{ + return MallocUsage(sizeof(stl_tree_node)); +} + +template +static inline size_t DynamicUsage(const std::map& m) { return MallocUsage(sizeof(stl_tree_node >)) * m.size(); } +template +static inline size_t IncrementalDynamicUsage(const std::map& m) +{ + return MallocUsage(sizeof(stl_tree_node >)); +} + +template +static inline size_t DynamicUsage(const std::unique_ptr& p) +{ + return p ? MallocUsage(sizeof(X)) : 0; +} + +template +static inline size_t DynamicUsage(const std::shared_ptr& p) +{ + // A shared_ptr can either use a single continuous memory block for both + // the counter and the storage (when using std::make_shared), or separate. + // We can't observe the difference, however, so assume the worst. + return p ? MallocUsage(sizeof(X)) + MallocUsage(sizeof(stl_shared_counter)) : 0; +} + // Boost data structures template diff --git a/src/net.cpp b/src/net.cpp index c2641a745d3..65267c91a5a 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -102,8 +102,11 @@ std::string strSubVersion; vector vNodes; CCriticalSection cs_vNodes; -map mapRelay; -deque > vRelayExpiration; + +MapRelay mapRelay; +/** Expiration-time ordered list of (expire time, relay map entry) pairs, protected by cs_main). */ +std::deque> vRelayExpiration; + CCriticalSection cs_mapRelay; limitedmap mapAlreadyAskedFor(MAX_INV_SZ); @@ -575,6 +578,15 @@ void CNode::CloseSocketDisconnect() } } + { + LOCK(cs_addrKnown); + addrKnown.reset(); + } + { + LOCK(cs_inventory); + filterInventoryKnown.reset(); + } + // in case this fails, we'll empty the recv buffer when the CNode is deleted TRY_LOCK(cs_vRecvMsg, lockRecv); if (lockRecv) @@ -600,7 +612,7 @@ void CNode::PushVersion() if (PROTOCOL_VERSION >= MIN_PBAAS_VERSION) { CIdentityID nodeID = VERUS_NODEID; - PushMessage("version", PROTOCOL_VERSION, + PushMessage("version", PROTOCOL_VERSION, nLocalServices, nTime, addrYou, addrMe, nLocalHostNonce, nodeID, strSubVersion, nBestHeight, true); } else @@ -735,6 +747,9 @@ void CNode::copyStats(CNodeStats &stats) stats.dPingTime = (((double)nPingUsecTime) / 1e6); stats.dPingWait = (((double)nPingUsecWait) / 1e6); + stats.m_addr_processed = m_addr_processed.load(); + stats.m_addr_rate_limited = m_addr_rate_limited.load(); + // Leave string empty if addrLocal invalid (not filled in yet) stats.addrLocal = addrLocal.IsValid() ? addrLocal.ToString() : ""; @@ -2213,32 +2228,11 @@ void RelayTransaction(const CTransaction& tx) void RelayTransaction(const CTransaction& tx, const CDataStream& ss) { - CInv inv(MSG_TX, tx.GetHash()); - { - LOCK(cs_mapRelay); - // Expire old relay messages - while (!vRelayExpiration.empty() && vRelayExpiration.front().first < GetTime()) - { - mapRelay.erase(vRelayExpiration.front().second); - vRelayExpiration.pop_front(); - } - - // Save original serialized message so newer versions are preserved - mapRelay.insert(std::make_pair(inv, ss)); - vRelayExpiration.push_back(std::make_pair(GetTime() + 15 * 60, inv)); - } + uint256 txHash = tx.GetHash(); LOCK(cs_vNodes); - BOOST_FOREACH(CNode* pnode, vNodes) + for (CNode* pnode : vNodes) { - if(!pnode->fRelayTxes) - continue; - LOCK(pnode->cs_filter); - if (pnode->pfilter) - { - if (pnode->pfilter->IsRelevantAndUpdate(tx)) - pnode->PushInventory(inv); - } else - pnode->PushInventory(inv); + pnode->PushTxInventory(WTxId(txHash)); } } @@ -2406,9 +2400,7 @@ unsigned int ReceiveFloodSize() { return 1000*GetArg("-maxreceivebuffer", 5*1000 unsigned int SendBufferSize() { return 1000*GetArg("-maxsendbuffer", 1*1000); } CNode::CNode(SOCKET hSocketIn, const CAddress& addrIn, const std::string& addrNameIn, bool fInboundIn, SSL *sslIn) : - ssSend(SER_NETWORK, INIT_PROTO_VERSION), - addrKnown(5000, 0.001), - setInventoryKnown(SendBufferSize() / 1000) + addrKnown(5000, 0.001), ssSend(SER_NETWORK, INIT_PROTO_VERSION) { ssl = sslIn; nServices = 0; @@ -2438,6 +2430,9 @@ CNode::CNode(SOCKET hSocketIn, const CAddress& addrIn, const std::string& addrNa nStartingHeight = -1; fGetAddr = false; fRelayTxes = false; + nNextLocalAddrSend = 0; + nNextAddrSend = 0; + nNextInvSend = 0; fSentAddr = false; pfilter = new CBloomFilter(); nPingNonceSent = 0; @@ -2623,3 +2618,7 @@ void CNode::EndMessage() UNLOCK_FUNCTION(cs_vSend) LEAVE_CRITICAL_SECTION(cs_vSend); } + +int64_t PoissonNextSend(int64_t nNow, int average_interval_seconds) { + return nNow + (int64_t)(log1p(GetRand(1ULL << 48) * -0.0000000000000035527136788 /* -1/2^48 */) * average_interval_seconds * -1000000.0 + 0.5); +} diff --git a/src/net.h b/src/net.h index 465bd65f931..de819b5214f 100644 --- a/src/net.h +++ b/src/net.h @@ -18,6 +18,8 @@ #include "sync.h" #include "uint256.h" #include "utilstrencodings.h" +#include "utiltime.h" +#include "primitives/transaction.h" #include #include @@ -52,6 +54,15 @@ static const int TIMEOUT_INTERVAL = 20 * 60; static const unsigned int MAX_INV_SZ = 50000; /** The maximum number of new addresses to accumulate before announcing. */ static const unsigned int MAX_ADDR_TO_SEND = 1000; + +/** The maximum rate of address records we're willing to process on average. Can be bypassed using + * the NetPermissionFlags::Addr permission. */ +static constexpr double MAX_ADDR_RATE_PER_SECOND{0.1}; +/** The soft limit of the address processing token bucket (the regular MAX_ADDR_RATE_PER_SECOND + * based increments won't go above this, but the MAX_ADDR_TO_SEND increment following GETADDR + * is exempt from this limit. */ +static constexpr size_t MAX_ADDR_PROCESSING_TOKEN_BUCKET{MAX_ADDR_TO_SEND}; + /** Maximum length of incoming protocol messages (no message over 2 MiB is currently acceptable). */ static const unsigned int MAX_PROTOCOL_MESSAGE_LENGTH = 2 * 1024 * 1024; /** Maximum length of strSubVer in `version` message */ @@ -66,6 +77,8 @@ static const size_t SETASKFOR_MAX_SZ = 2 * MAX_INV_SZ; static const unsigned int DEFAULT_MAX_PEER_CONNECTIONS = 384; /** The period before a network upgrade activates, where connections to upgrading peers are preferred (in blocks). */ static const int NETWORK_UPGRADE_PEER_PREFERENCE_BLOCK_PERIOD = 24 * 24 * 3; +/** Default for blocks only*/ +static const bool DEFAULT_BLOCKSONLY = false; unsigned int ReceiveFloodSize(); unsigned int SendBufferSize(); @@ -163,8 +176,10 @@ extern int nMaxConnections; extern std::vector vNodes; extern CCriticalSection cs_vNodes; -extern std::map mapRelay; -extern std::deque > vRelayExpiration; +/** Relay map, protected by cs_main. */ +typedef std::map> MapRelay; +extern MapRelay mapRelay; +extern std::deque> vRelayExpiration; extern CCriticalSection cs_mapRelay; extern limitedmap mapAlreadyAskedFor; @@ -210,6 +225,8 @@ class CNodeStats double dPingTime; double dPingWait; std::string addrLocal; + uint64_t m_addr_processed{0}; + uint64_t m_addr_rate_limited{0}; }; @@ -312,8 +329,16 @@ class CNode CCriticalSection cs_filter; CBloomFilter* pfilter; - int nRefCount; + std::atomic nRefCount; NodeId id; + + CRollingBloomFilter addrKnown; + mutable CCriticalSection cs_addrKnown; + + // Inventory based relay + // This filter is protected by cs_inventory and contains both txids and wtxids. + CRollingBloomFilter filterInventoryKnown; + protected: // Denial-of-service detection/prevention @@ -347,28 +372,46 @@ class CNode // flood relay std::vector vAddrToSend; - CRollingBloomFilter addrKnown; bool fGetAddr; std::set setKnown; - // inventory based relay - mruset setInventoryKnown; - std::vector vInventoryToSend; + int64_t nNextAddrSend; + int64_t nNextLocalAddrSend; + + /** Number of addr messages that can be processed from this peer. Start at 1 to + * permit self-announcement. */ + double m_addr_token_bucket{1.0}; + /** When m_addr_token_bucket was last updated */ + int64_t m_addr_token_timestamp{GetTimeMicros()}; + /** Total number of addresses that were dropped due to rate limiting. */ + std::atomic m_addr_rate_limited{0}; + /** Total number of addresses that were processed (excludes rate limited ones). */ + std::atomic m_addr_processed{0}; + + // Set of transaction ids we still have to announce. + // They are sorted by the mempool before relay, so the order is not important. + std::set setInventoryTxToSend; + // List of block ids we still have to announce. + // There is no final sorting before sending, as they are always sent immediately + // and in the order requested. + std::vector vInventoryBlockToSend; + CCriticalSection cs_inventory; std::set setAskFor; + int64_t nNextInvSend; std::multimap mapAskFor; // Ping time measurement: // The pong reply we're expecting, or 0 if no pong expected. - uint64_t nPingNonceSent; + std::atomic nPingNonceSent; // Time (in usec) the last ping was sent, or 0 if no ping was ever sent. - int64_t nPingUsecStart; + std::atomic nPingUsecStart; // Last measured round-trip time. - int64_t nPingUsecTime; + std::atomic nPingUsecTime; // Best measured round-trip time. - int64_t nMinPingUsecTime; + std::atomic nMinPingUsecTime; // Whether a ping is requested. - bool fPingQueued; + std::atomic fPingQueued; CNode(SOCKET hSocketIn, const CAddress &addrIn, const std::string &addrNameIn = "", bool fInboundIn = false, SSL *sslIn = NULL); ~CNode(); @@ -426,19 +469,12 @@ class CNode nRefCount--; } - - - void AddAddressKnown(const CAddress& addr) - { - addrKnown.insert(addr.GetKey()); - } - void PushAddress(const CAddress& addr) { // Known checking here is only to save space from duplicates. // SendMessages will filter it again for knowns that were added // after addresses were pushed. - if (addr.IsValid() && !addrKnown.contains(addr.GetKey())) { + if (addr.IsValid() && !IsAddressKnown(addr)) { if (vAddrToSend.size() >= MAX_ADDR_TO_SEND) { vAddrToSend[insecure_rand() % vAddrToSend.size()] = addr; } else { @@ -447,21 +483,62 @@ class CNode } } + void AddKnownWTxId(const WTxId& wtxid) + { + LOCK(cs_inventory); + if (!fDisconnect) { + filterInventoryKnown.insert(wtxid.ToBytes()); + } + } - void AddInventoryKnown(const CInv& inv) + void AddKnownTxId(const uint256& txid) { - { - LOCK(cs_inventory); - setInventoryKnown.insert(inv); + LOCK(cs_inventory); + if (!fDisconnect) { + filterInventoryKnown.insert(txid); } } - void PushInventory(const CInv& inv) + bool HasKnownTxId(const uint256& txid) { - { - LOCK(cs_inventory); - if (!setInventoryKnown.count(inv)) - vInventoryToSend.push_back(inv); + LOCK(cs_inventory); + return filterInventoryKnown.contains(txid); + } + + bool AddAddressIfNotAlreadyKnown(const CAddress& addr) + { + LOCK(cs_addrKnown); + // Avoid adding to addrKnown after it has been reset in CloseSocketDisconnect. + if (fDisconnect) { + return false; + } + if (!addrKnown.contains(addr.GetKey())) { + addrKnown.insert(addr.GetKey()); + return true; + } else { + return false; + } + } + + bool IsAddressKnown(const CAddress& addr) const + { + LOCK(cs_addrKnown); + return addrKnown.contains(addr.GetKey()); + } + + void PushTxInventory(const WTxId& wtxid) + { + LOCK(cs_inventory); + if (!fDisconnect && !filterInventoryKnown.contains(wtxid.ToBytes())) { + setInventoryTxToSend.insert(wtxid.hash); + } + } + + void PushBlockInventory(const uint256& hash) + { + LOCK(cs_inventory); + if (!fDisconnect) { + vInventoryBlockToSend.push_back(hash); } } @@ -712,4 +789,7 @@ class CAddrDB bool Read(CAddrMan& addr); }; +/** Return a timestamp in the future (in microseconds) for exponentially distributed events. */ +int64_t PoissonNextSend(int64_t nNow, int average_interval_seconds); + #endif // BITCOIN_NET_H diff --git a/src/pbaas/notarization.cpp b/src/pbaas/notarization.cpp index 8f18bba0a27..e9378d3916b 100644 --- a/src/pbaas/notarization.cpp +++ b/src/pbaas/notarization.cpp @@ -2382,7 +2382,7 @@ std::tuple GetPriorReferen if (!autoProof.chainObjects.size()) { - LogPrint("notarization", "%s: notarization block hash not found in block index or active chain"); + LogPrint("notarization", "%s: notarization block hash proof type not found in block index or active chain at height \n", __func__); return retVal; } @@ -3639,17 +3639,8 @@ CPBaaSNotarization IsValidPrimaryChainEvidence(const CCurrencyDefinition &extern int heightChange = futureProofRoot.rootHeight - lastNotarization.proofRoots[lastNotarization.currencyID].rootHeight; numExpectedCheckpoints = CPBaaSNotarization::GetNumCheckpoints(heightChange); - if (numExpectedCheckpoints && - chainActive[lastNotarization.proofRoots[ASSETCHAINS_CHAINID].rootHeight]->nTime > PBAAS_TESTFORK_TIME) - { - blocksPerCheckpoint = CPBaaSNotarization::GetBlocksPerCheckpoint(heightChange); - proofState = EXPECT_CHECKPOINT; - } - else - { - validBasicEvidence = !challengeProofRoot.IsValid(); - proofState = validBasicEvidence ? EXPECT_NOTHING : EXPECT_COMMITMENT_PROOF; - } + validBasicEvidence = !challengeProofRoot.IsValid(); + proofState = validBasicEvidence ? EXPECT_NOTHING : EXPECT_COMMITMENT_PROOF; } else { @@ -4464,16 +4455,6 @@ bool CPBaaSNotarization::CreateAcceptedNotarization(const CCurrencyDefinition &e bool additionalEvidenceRequired = externalSystem.proofProtocol == externalSystem.PROOF_PBAASMMR && externalSystem.notarizationProtocol != externalSystem.NOTARIZATION_NOTARY_CHAINID; - // proof of the last confirmed notarization must be present and match the one on this chain or - // prove the block 1 coinbase of the PBaaS chain - std::set evidenceTypes; - evidenceTypes.insert(CHAINOBJ_TRANSACTION_PROOF); - CCrossChainProof autoProof(notaryEvidence.GetSelectEvidence(evidenceTypes)); - if (additionalEvidenceRequired && !autoProof.chainObjects.size()) - { - return state.Error(errorPrefix + "Missing evidence of prior notarization, required to accept submission"); - } - // create an accepted notarization based on the cross-chain notarization provided CPBaaSNotarization newNotarization = earnedNotarization; @@ -4484,6 +4465,20 @@ bool CPBaaSNotarization::CreateAcceptedNotarization(const CCurrencyDefinition &e } LOCK(cs_main); + uint32_t height = chainActive.Height(); + + bool fullEvidence = !PBAAS_TESTMODE || chainActive.LastTip()->nTime > (PBAAS_TESTFORK_TIME - (20 * 60)); + + // proof of the last confirmed notarization must be present and match the one on this chain or + // prove the block 1 coinbase of the PBaaS chain + std::set evidenceTypes; + evidenceTypes.insert(CHAINOBJ_TRANSACTION_PROOF); + CCrossChainProof autoProof(notaryEvidence.GetSelectEvidence(evidenceTypes)); + if (additionalEvidenceRequired && + !autoProof.chainObjects.size()) + { + return state.Error(errorPrefix + "Missing evidence of prior notarization, required to accept submission"); + } // now, we'll want to put our own proposer as the beneficiary of this notarization, if possible if (!VERUS_NOTARYID.IsNull()) @@ -4503,7 +4498,6 @@ bool CPBaaSNotarization::CreateAcceptedNotarization(const CCurrencyDefinition &e newNotarization.proposer.SetAuxDest(DestinationToTransferDestination(VERUS_DEFAULTID), 0); } - uint32_t height = chainActive.Height(); CProofRoot ourRoot = newNotarization.proofRoots[ASSETCHAINS_CHAINID]; CChainNotarizationData cnd; @@ -4536,9 +4530,11 @@ bool CPBaaSNotarization::CreateAcceptedNotarization(const CCurrencyDefinition &e // proof of the current notarization by the first proof root mentioned above CTransaction outTx; bool isPartial = false; - uint256 txMMRRoot = ((CChainObject *)autoProof.chainObjects[0])->object.CheckPartialTransaction(outTx, &isPartial); - uint256 txHash = ((CChainObject *)autoProof.chainObjects[0])->object.TransactionHash(); + uint256 txMMRRoot; + uint256 txHash; + txMMRRoot = ((CChainObject *)autoProof.chainObjects[0])->object.CheckPartialTransaction(outTx, &isPartial); + txHash = ((CChainObject *)autoProof.chainObjects[0])->object.TransactionHash(); if (txMMRRoot != newNotarization.proofRoots[newNotarization.currencyID].stateRoot) { return state.Error(errorPrefix + "state root mismatch in prior notarization proof"); @@ -4625,7 +4621,7 @@ bool CPBaaSNotarization::CreateAcceptedNotarization(const CCurrencyDefinition &e } // challenge roots must include all UTXOs between our last agreed index and the end - int numChallenges = cnd.vtx.size() - (priorNotarizationIdx + 1); + int numChallenges = fullEvidence ? cnd.vtx.size() - (priorNotarizationIdx + 1) : 0; bool isChallengeStronger = false; CProofRoot strongestRoot(CProofRoot::TYPE_PBAAS, CProofRoot::VERSION_INVALID); if (numChallenges) @@ -4900,16 +4896,17 @@ bool CPBaaSNotarization::CreateAcceptedNotarization(const CCurrencyDefinition &e // add prior unspent accepted notarization as our input txBuilder.AddTransparentInput(CUTXORef(lastTxId, lastTxOutNum), lastTx.vout[lastTxOutNum].scriptPubKey, lastTx.vout[lastTxOutNum].nValue); - if ((height - priorNotarizationHeight) < - CPBaaSNotarization::GetAdjustedNotarizationModulo(ConnectedChains.ThisChain().blockNotarizationModulo, - height - mapBlockIndex[txes[cnd.lastConfirmed].second]->GetHeight())) + uint32_t adjustedNotarizationModulo = (chainActive[height]->nTime > (PBAAS_TESTFORK_TIME - (20 * 60))) ? + CPBaaSNotarization::GetAdjustedNotarizationModulo(ConnectedChains.ThisChain().blockNotarizationModulo, + height - mapBlockIndex[txes[cnd.lastConfirmed].second]->GetHeight()) : + ConnectedChains.ThisChain().blockNotarizationModulo; + + if ((height - priorNotarizationHeight) < adjustedNotarizationModulo) { LogPrint("notarization", "%s: cannot create accepted notarization earlier than %u blocks since the prior notarization it confirms\n", - CPBaaSNotarization::GetAdjustedNotarizationModulo(ConnectedChains.ThisChain().blockNotarizationModulo, - height - mapBlockIndex[txes[cnd.lastConfirmed].second]->GetHeight())); + adjustedNotarizationModulo); return state.Error(errorPrefix + "cannot create accepted notarization earlier than " + - std::to_string(CPBaaSNotarization::GetAdjustedNotarizationModulo(ConnectedChains.ThisChain().blockNotarizationModulo, - height - mapBlockIndex[txes[cnd.lastConfirmed].second]->GetHeight())) + + std::to_string(adjustedNotarizationModulo) + " blocks since the prior notarization it confirms"); } @@ -5064,10 +5061,10 @@ bool CPBaaSNotarization::CreateAcceptedNotarization(const CCurrencyDefinition &e } if (inputSet.count(oneInput.txIn.prevout)) { - if (LogAcceptCategory("notarization")) + if (LogAcceptCategory("notarization") && LogAcceptCategory("verbose")) { - printf("%s: duplicate input: %s\n", __func__, oneInput.txIn.prevout.ToString().c_str()); - LogPrintf("%s: duplicate input: %s\n", __func__, oneInput.txIn.prevout.ToString().c_str()); + printf("%s: skipping duplicate input: %s\n", __func__, oneInput.txIn.prevout.ToString().c_str()); + LogPrintf("%s: skipping duplicate input: %s\n", __func__, oneInput.txIn.prevout.ToString().c_str()); } continue; } @@ -5084,6 +5081,7 @@ bool CPBaaSNotarization::CreateAcceptedNotarization(const CCurrencyDefinition &e { if (!inputSet.count(oneInput.txIn.prevout) && !IsScriptTooLargeToSpend(oneInput.scriptPubKey)) { + inputSet.insert(oneInput.txIn.prevout); txBuilder.AddTransparentInput(oneInput.txIn.prevout, oneInput.scriptPubKey, oneInput.nValue); } } @@ -5689,9 +5687,11 @@ bool CPBaaSNotarization::CreateEarnedNotarization(const CRPCChainData &externalS } CBlockIndex *pConfirmedNotarizationIndex = mapBlockIndex[txes[0].second]; - uint32_t adjustedNotarizationModulo = CPBaaSNotarization::GetAdjustedNotarizationModulo(ConnectedChains.ThisChain().blockNotarizationModulo, + uint32_t adjustedNotarizationModulo = (chainActive[height]->nTime > (PBAAS_TESTFORK_TIME - (20 * 60))) ? + CPBaaSNotarization::GetAdjustedNotarizationModulo(ConnectedChains.ThisChain().blockNotarizationModulo, pCurNotarizationIndex->GetHeight() - - pConfirmedNotarizationIndex->GetHeight()); + pConfirmedNotarizationIndex->GetHeight()) : + ConnectedChains.ThisChain().blockNotarizationModulo; int blockPeriodNumber = pCurNotarizationIndex->GetHeight() / adjustedNotarizationModulo; int priorBlockPeriod = pPriorNotarizationIndex->GetHeight() / adjustedNotarizationModulo; if (priorBlockPeriod >= blockPeriodNumber) @@ -6350,10 +6350,11 @@ bool CPBaaSNotarization::CreateEarnedNotarization(const CRPCChainData &externalS // first determine if the prior notarization we agree with would make this one moot, given the current // block notarization modulo - uint32_t adjustedNotarizationModulo = CPBaaSNotarization::GetAdjustedNotarizationModulo(ConnectedChains.ThisChain().blockNotarizationModulo, - (height + 1) - - mapBlockIndex[txes[cnd.lastConfirmed].second]->GetHeight()); - + uint32_t adjustedNotarizationModulo = (chainActive[height]->nTime > (PBAAS_TESTFORK_TIME - (20 * 60))) ? + CPBaaSNotarization::GetAdjustedNotarizationModulo(ConnectedChains.ThisChain().blockNotarizationModulo, + (height + 1) - mapBlockIndex[txes[cnd.lastConfirmed].second]->GetHeight()) : + ConnectedChains.ThisChain().blockNotarizationModulo; + int blockPeriodNumber = (height + 1) / adjustedNotarizationModulo; int priorBlockPeriod = mapBlockIt->second->GetHeight() / adjustedNotarizationModulo; @@ -7518,26 +7519,34 @@ bool CPBaaSNotarization::ConfirmOrRejectNotarizations(CWallet *pWallet, if (LogAcceptCategory("notarization")) { - for (int j = 0; j < spendsToClose.size(); j++) + if (LogAcceptCategory("verbose")) { - auto &oneEntrySpends = spendsToClose[j]; - LogPrintf("%s: index #%d - %s\n", __func__, j, confirmedOutputNums.count(j) ? "confirmed" : invalidatedOutputNums.count(j) ? "rejected" : "pending"); - for (auto &oneSpend : oneEntrySpends) + for (int j = 0; j < spendsToClose.size(); j++) { - UniValue scriptUni(UniValue::VOBJ); - ScriptPubKeyToUniv(oneSpend.scriptPubKey, scriptUni, false, false); - LogPrintf("txid: %s:%d, nValue: %ld\n", oneSpend.txIn.prevout.hash.GetHex().c_str(), oneSpend.txIn.prevout.n, oneSpend.nValue); + auto &oneEntrySpends = spendsToClose[j]; + LogPrintf("%s: index #%d - %s\n", __func__, j, confirmedOutputNums.count(j) ? "confirmed" : invalidatedOutputNums.count(j) ? "rejected" : "pending"); + for (auto &oneSpend : oneEntrySpends) + { + UniValue scriptUni(UniValue::VOBJ); + ScriptPubKeyToUniv(oneSpend.scriptPubKey, scriptUni, false, false); + LogPrintf("txid: %s:%d, nValue: %ld\n", oneSpend.txIn.prevout.hash.GetHex().c_str(), oneSpend.txIn.prevout.n, oneSpend.nValue); + } } - } - for (int j = 0; j < evidenceVec.size(); j++) - { - auto &oneEvidenceVec = evidenceVec[j]; - LogPrintf("evidence vector: index #%d\n", j); - for (auto &oneEvidence : oneEvidenceVec) + for (int j = 0; j < evidenceVec.size(); j++) { - LogPrintf("%s\n", oneEvidence.ToUniValue().write(1,2).c_str()); + auto &oneEvidenceVec = evidenceVec[j]; + LogPrintf("evidence vector: index #%d\n", j); + for (auto &oneEvidence : oneEvidenceVec) + { + LogPrintf("%s\n", oneEvidence.ToUniValue().write(1,2).c_str()); + } } } + else + { + LogPrintf("spendsToClose size: %ld\n", spendsToClose.size()); + LogPrintf("evidence vector size: %ld\n", evidenceVec.size()); + } } CNotaryEvidence ne(ASSETCHAINS_CHAINID, cnd.vtx[idx].first, CNotaryEvidence::STATE_CONFIRMING); @@ -7664,7 +7673,7 @@ bool CPBaaSNotarization::ConfirmOrRejectNotarizations(CWallet *pWallet, if (!evidenceVec.size()) { LogPrintf("%s: failed to package evidence for notarization of system %s\n", __func__, EncodeDestination(CIdentityID(cnd.vtx[0].second.currencyID)).c_str()); - return false; + return state.Error(errorPrefix + "failed to package evidence for notarization"); } for (auto &oneProof : evidenceVec) { @@ -7703,6 +7712,7 @@ bool CPBaaSNotarization::ConfirmOrRejectNotarizations(CWallet *pWallet, { std::set inputSet; std::vector nonEvidenceSpends; + std::vector notarizationSpends; for (auto &oneInput : spendsToClose[idx]) { @@ -7734,15 +7744,22 @@ bool CPBaaSNotarization::ConfirmOrRejectNotarizations(CWallet *pWallet, inputSet.insert(oneInput.txIn.prevout); txBuilder.AddTransparentInput(oneInput.txIn.prevout, oneInput.scriptPubKey, oneInput.nValue); } + else if (tP.IsValid() && + (tP.evalCode == EVAL_EARNEDNOTARIZATION || + tP.evalCode == EVAL_ACCEPTEDNOTARIZATION)) + { + notarizationSpends.push_back(oneInput); + } else { nonEvidenceSpends.push_back(oneInput); } } - for (auto &oneInput : nonEvidenceSpends) + for (auto &oneInput : notarizationSpends) { if (!inputSet.count(oneInput.txIn.prevout) && !IsScriptTooLargeToSpend(oneInput.scriptPubKey)) { + inputSet.insert(oneInput.txIn.prevout); txBuilder.AddTransparentInput(oneInput.txIn.prevout, oneInput.scriptPubKey, oneInput.nValue); } } @@ -7817,19 +7834,20 @@ bool CPBaaSNotarization::ConfirmOrRejectNotarizations(CWallet *pWallet, extraEvidenceSpends.push_back(oneInput); } } - else if (LogAcceptCategory("notarization")) + else if (LogAcceptCategory("notarization") && LogAcceptCategory("verbose")) { - printf("%s: duplicate input: %s\n", __func__, oneInput.txIn.prevout.ToString().c_str()); - LogPrintf("%s: duplicate input: %s\n", __func__, oneInput.txIn.prevout.ToString().c_str()); + printf("%s: skipping duplicate input: %s\n", __func__, oneInput.txIn.prevout.ToString().c_str()); + LogPrintf("%s: skipping duplicate input: %s\n", __func__, oneInput.txIn.prevout.ToString().c_str()); } } - for (auto &oneInput : extraEvidenceSpends) + /* for (auto &oneInput : extraEvidenceSpends) { if (!IsScriptTooLargeToSpend(oneInput.scriptPubKey)) { + // added to inputSet above already, so no need here oneConfirmedBuilder.AddTransparentInput(oneInput.txIn.prevout, oneInput.scriptPubKey, oneInput.nValue); } - } + } */ if (makeInputTx) { @@ -7870,10 +7888,10 @@ bool CPBaaSNotarization::ConfirmOrRejectNotarizations(CWallet *pWallet, oneInvalidatedBuilder.AddTransparentInput(oneInput.txIn.prevout, oneInput.scriptPubKey, oneInput.nValue); } } - else if (LogAcceptCategory("notarization")) + else if (LogAcceptCategory("notarization") && LogAcceptCategory("verbose")) { - printf("%s: duplicate input: %s\n", __func__, oneInput.txIn.prevout.ToString().c_str()); - LogPrintf("%s: duplicate input: %s\n", __func__, oneInput.txIn.prevout.ToString().c_str()); + printf("%s: skipping duplicate input: %s\n", __func__, oneInput.txIn.prevout.ToString().c_str()); + LogPrintf("%s: skipping duplicate input: %s\n", __func__, oneInput.txIn.prevout.ToString().c_str()); } } @@ -8133,8 +8151,11 @@ std::vector CPBaaSNotarization::SubmitFinalizedNotarizations(const CRPC !(crosschainCND = CChainNotarizationData(result, true, &blockHashes, &counterEvidence, &evidence)).IsValid() || (!externalSystem.chainDefinition.IsGateway() && !crosschainCND.IsConfirmed())) { - LogPrintf("Unable to get notarization data from %s\n", EncodeDestination(CIdentityID(systemID)).c_str()); - printf("Unable to get notarization data from %s\n", EncodeDestination(CIdentityID(systemID)).c_str()); + if (LogAcceptCategory("notarization")) + { + LogPrintf("Unable to get notarization data from %s\n", EncodeDestination(CIdentityID(systemID)).c_str()); + printf("Unable to get notarization data from %s\n", EncodeDestination(CIdentityID(systemID)).c_str()); + } return retVal; } @@ -8293,6 +8314,11 @@ std::vector CPBaaSNotarization::SubmitFinalizedNotarizations(const CRPC for (auto &oneProofRoot : oneNotarization.second.proofRoots) { + uint32_t adjustedNotarizationModulo = (chainActive[nHeight]->nTime > (PBAAS_TESTFORK_TIME - (20 * 60))) ? + CPBaaSNotarization::GetAdjustedNotarizationModulo(externalSystem.chainDefinition.blockNotarizationModulo, + cnd.vtx[cnd.lastConfirmed].second.proofRoots[oneProofRoot.first].rootHeight - oneProofRoot.second.rootHeight) : + externalSystem.chainDefinition.blockNotarizationModulo; + if (oneProofRoot.first == ASSETCHAINS_CHAINID) { if (oneProofRoot.second != CProofRoot::GetProofRoot(oneProofRoot.second.rootHeight)) @@ -8329,8 +8355,7 @@ std::vector CPBaaSNotarization::SubmitFinalizedNotarizations(const CRPC oneProofRoot.first == systemID && (!cnd.vtx[cnd.lastConfirmed].second.proofRoots.count(oneProofRoot.first) || (cnd.vtx[cnd.lastConfirmed].second.proofRoots[oneProofRoot.first].rootHeight - oneProofRoot.second.rootHeight) < - CPBaaSNotarization::GetAdjustedNotarizationModulo(externalSystem.chainDefinition.blockNotarizationModulo, - cnd.vtx[cnd.lastConfirmed].second.proofRoots[oneProofRoot.first].rootHeight - oneProofRoot.second.rootHeight))) + adjustedNotarizationModulo)) { LogPrint("notarization", "Skip submission - no new enough confirmed notarizations for %s\n", EncodeDestination(CIdentityID(systemID)).c_str()); return retVal; @@ -8449,8 +8474,11 @@ std::vector CPBaaSNotarization::SubmitFinalizedNotarizations(const CRPC ConnectedChains.ThisChain().launchSystemID == externalSystem.chainDefinition.GetID()) || ::AsVector(newConfirmedNotarization) == ::AsVector(cnd.vtx[cnd.lastConfirmed].second))) { - LogPrintf("Unable to get notarization data from %s\n", EncodeDestination(CIdentityID(systemID)).c_str()); - printf("Unable to get notarization data from %s\n", EncodeDestination(CIdentityID(systemID)).c_str()); + if (LogAcceptCategory("notarization")) + { + LogPrintf("Unable to get notarization data from %s\n", EncodeDestination(CIdentityID(systemID)).c_str()); + printf("Unable to get notarization data from %s\n", EncodeDestination(CIdentityID(systemID)).c_str()); + } return retVal; } earnedNotarizationIndexEntry.first.blockHeight = 1; @@ -8479,6 +8507,7 @@ std::vector CPBaaSNotarization::SubmitFinalizedNotarizations(const CRPC // if we have a cross-confirmed notarization to prove here, create a proof if (!earnedNotarizationIndexEntry.first.txhash.IsNull()) { + allEvidence.output = cnd.vtx[cnd.lastConfirmed].first; UniValue proofRequests(UniValue::VARR); UniValue proofRequest(UniValue::VOBJ); proofRequest.pushKV("type", EncodeDestination(CIdentityID(CNotaryEvidence::PrimaryProofKey()))); @@ -8498,7 +8527,6 @@ std::vector CPBaaSNotarization::SubmitFinalizedNotarizations(const CRPC { proofRequest.pushKV("challengeroots", challengeRoots); } - allEvidence.output = cnd.vtx[cnd.lastConfirmed].first; proofRequest.pushKV("evidence", allEvidence.ToUniValue()); proofRequest.pushKV("confirmnotarizationref", cnd.vtx[cnd.lastConfirmed].first.ToUniValue()); proofRequest.pushKV("entropyhash", blockHashes[confirmingIdx].GetHex()); diff --git a/src/pbaas/pbaas.cpp b/src/pbaas/pbaas.cpp index 95db1b7aca8..792f70641e4 100644 --- a/src/pbaas/pbaas.cpp +++ b/src/pbaas/pbaas.cpp @@ -5019,18 +5019,27 @@ bool CConnectedChains::GetUnspentByIndex(const uint160 &indexID, std::vector spentInMempool; auto memPoolOuts = mempool.FilterUnspent(unconfirmedUTXOs, spentInMempool); + LRUCache> txesBeingSpent(50, 0.3); + for (auto &oneConfirmed : confirmedUTXOs) { BlockMap::iterator blockIt; - uint256 blockHash; - CTransaction tx; + std::pair txAndBlkHash; + bool fromCache = false; + bool fromChain = false; if (spentInMempool.count(COutPoint(oneConfirmed.first.txhash, oneConfirmed.first.index)) || - !myGetTransaction(oneConfirmed.first.txhash, tx, blockHash) || - (blockIt = mapBlockIndex.find(blockHash)) == mapBlockIndex.end() || + (!(fromCache = txesBeingSpent.Get(oneConfirmed.first.txhash, txAndBlkHash)) && + !(fromChain = myGetTransaction(oneConfirmed.first.txhash, txAndBlkHash.first, txAndBlkHash.second))) || + (blockIt = mapBlockIndex.find(txAndBlkHash.second)) == mapBlockIndex.end() || !chainActive.Contains(blockIt->second)) { + if (fromChain) + { + txesBeingSpent.Put(oneConfirmed.first.txhash, txAndBlkHash); + } continue; } + txesBeingSpent.Put(oneConfirmed.first.txhash, txAndBlkHash); oneConfirmed.second.blockHeight = blockIt->second->GetHeight(); COptCCParams p; diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 3e4bb5ad4cd..39d38f3a030 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -1718,4 +1718,38 @@ class CUTXORef : public COutPoint UniValue ToUniValue() const; }; +struct WTxId +{ + const uint256 hash; + const uint256 authDigest; + + WTxId() : + authDigest(LEGACY_TX_AUTH_DIGEST) {} + + WTxId(const uint256& hashIn, const uint256& authDigestIn=LEGACY_TX_AUTH_DIGEST) : + hash(hashIn), authDigest(authDigestIn) {} + + const std::vector ToBytes() const { + std::vector vData(hash.begin(), hash.end()); + vData.insert(vData.end(), authDigest.begin(), authDigest.end()); + return vData; + } + + friend bool operator<(const WTxId& a, const WTxId& b) + { + return (a.hash < b.hash || + (a.hash == b.hash && a.authDigest < b.authDigest)); + } + + friend bool operator==(const WTxId& a, const WTxId& b) + { + return a.hash == b.hash && a.authDigest == b.authDigest; + } + + friend bool operator!=(const WTxId& a, const WTxId& b) + { + return a.hash != b.hash || a.authDigest != b.authDigest; + } +}; + #endif // BITCOIN_PRIMITIVES_TRANSACTION_H diff --git a/src/random.cpp b/src/random.cpp index 1dfca983238..1c6923e51fe 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -1,5 +1,6 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2014 The Bitcoin Core developers +// Copyright (c) 2016-2023 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or https://www.opensource.org/licenses/mit-license.php . @@ -16,6 +17,7 @@ #include #ifndef _WIN32 +#include #include #endif @@ -54,6 +56,11 @@ uint64_t GetRand(uint64_t nMax) return (nRand % nMax); } +int64_t GetRandInt64(int64_t nMax) +{ + return GetRand(nMax); +} + int GetRandInt(int nMax) { return GetRand(nMax); @@ -66,6 +73,52 @@ uint256 GetRandHash() return hash; } +void FastRandomContext::RandomSeed() +{ + uint256 seed = GetRandHash(); + rng.SetKey(seed.begin(), 32); + requires_seed = false; +} + +uint256 FastRandomContext::rand256() +{ + if (bytebuf_size < 32) { + FillByteBuffer(); + } + uint256 ret; + memcpy(ret.begin(), bytebuf + 64 - bytebuf_size, 32); + bytebuf_size -= 32; + return ret; +} + +std::vector FastRandomContext::randbytes(size_t len) +{ + std::vector ret(len); + if (len > 0) { + rng.Output(&ret[0], len); + } + return ret; +} + +FastRandomContext::FastRandomContext(const uint256& seed) : requires_seed(false), bytebuf_size(0), bitbuf_size(0) +{ + rng.SetKey(seed.begin(), 32); +} + +int GenIdentity(int n) +{ + return n-1; +} + +FastRandomContext::FastRandomContext(bool fDeterministic) : requires_seed(!fDeterministic), bytebuf_size(0), bitbuf_size(0) +{ + if (!fDeterministic) { + return; + } + uint256 seed; + rng.SetKey(seed.begin(), 32); +} + uint32_t insecure_rand_Rz = 11; uint32_t insecure_rand_Rw = 11; void seed_insecure_rand(bool fDeterministic) @@ -85,8 +138,3 @@ void seed_insecure_rand(bool fDeterministic) insecure_rand_Rw = tmp; } } - -int GenIdentity(int n) -{ - return n-1; -} diff --git a/src/random.h b/src/random.h index 38b66bf0fb5..748b5b1aaca 100644 --- a/src/random.h +++ b/src/random.h @@ -1,24 +1,57 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2014 The Bitcoin Core developers +// Copyright (c) 2016-2023 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or https://www.opensource.org/licenses/mit-license.php . #ifndef BITCOIN_RANDOM_H #define BITCOIN_RANDOM_H +#include "crypto/chacha20.h" +#include "crypto/common.h" #include "uint256.h" #include +#include #include /** - * Functions to gather random data via the libsodium CSPRNG + * Functions to gather random data via the rand_core OsRng */ void GetRandBytes(unsigned char* buf, size_t num); uint64_t GetRand(uint64_t nMax); +int64_t GetRandInt64(int64_t nMax); int GetRandInt(int nMax); uint256 GetRandHash(); +/** + * Implementation of a C++ Uniform Random Number Generator, backed by GetRandBytes. + */ +class ZcashRandomEngine +{ +public: + typedef uint64_t result_type; + + explicit ZcashRandomEngine() {} + + static constexpr result_type min() { + return std::numeric_limits::min(); + } + static constexpr result_type max() { + return std::numeric_limits::max(); + } + + result_type operator()() { + result_type nRand = 0; + GetRandBytes((unsigned char*)&nRand, sizeof(nRand)); + return nRand; + } + + double entropy() const noexcept { + return 0; + } +}; + /** * Identity function for MappedShuffle, so that elements retain their original order. */ @@ -49,6 +82,93 @@ void MappedShuffle(RandomAccessIterator first, } } +/** + * Fast randomness source. This is seeded once with secure random data, but + * is completely deterministic and insecure after that. + * This class is not thread-safe. + */ +class FastRandomContext { +private: + bool requires_seed; + ChaCha20 rng; + + unsigned char bytebuf[64]; + int bytebuf_size; + + uint64_t bitbuf; + int bitbuf_size; + + void RandomSeed(); + + void FillByteBuffer() + { + if (requires_seed) { + RandomSeed(); + } + rng.Output(bytebuf, sizeof(bytebuf)); + bytebuf_size = sizeof(bytebuf); + } + + void FillBitBuffer() + { + bitbuf = rand64(); + bitbuf_size = 64; + } + +public: + explicit FastRandomContext(bool fDeterministic = false); + + /** Initialize with explicit seed (only for testing) */ + explicit FastRandomContext(const uint256& seed); + + /** Generate a random 64-bit integer. */ + uint64_t rand64() + { + if (bytebuf_size < 8) FillByteBuffer(); + uint64_t ret = ReadLE64(bytebuf + 64 - bytebuf_size); + bytebuf_size -= 8; + return ret; + } + + /** Generate a random (bits)-bit integer. */ + uint64_t randbits(int bits) { + if (bits == 0) { + return 0; + } else if (bits > 32) { + return rand64() >> (64 - bits); + } else { + if (bitbuf_size < bits) FillBitBuffer(); + uint64_t ret = bitbuf & (~(uint64_t)0 >> (64 - bits)); + bitbuf >>= bits; + bitbuf_size -= bits; + return ret; + } + } + + /** Generate a random integer in the range [0..range). */ + uint64_t randrange(uint64_t range) + { + --range; + int bits = CountBits(range); + while (true) { + uint64_t ret = randbits(bits); + if (ret <= range) return ret; + } + } + + /** Generate random bytes. */ + std::vector randbytes(size_t len); + + /** Generate a random 32-bit integer. */ + uint32_t rand32() { return randbits(32); } + + /** generate a random uint256. */ + uint256 rand256(); + + /** Generate a random boolean. */ + bool randbool() { return randbits(1); } +}; + /** * Seed insecure_rand using the random pool. * @param Deterministic Use a deterministic seed diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 4bae110672b..b3a8c7daffd 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -162,6 +162,8 @@ UniValue getpeerinfo(const UniValue& params, bool fHelp) } obj.push_back(Pair("inflight", heights)); } + obj.pushKV("addr_processed", stats.m_addr_processed); + obj.pushKV("addr_rate_limited", stats.m_addr_rate_limited); obj.push_back(Pair("whitelisted", stats.fWhitelisted)); ret.push_back(obj); diff --git a/src/rpc/pbaasrpc.cpp b/src/rpc/pbaasrpc.cpp index 5979b005a34..995a4addeb7 100644 --- a/src/rpc/pbaasrpc.cpp +++ b/src/rpc/pbaasrpc.cpp @@ -3928,8 +3928,8 @@ UniValue getnotarizationdata(const UniValue& params, bool fHelp) "\nArguments\n" "1. \"currencyid\" (string, required) the hex-encoded ID or string name search for notarizations on\n" - "2. \"(getevidence)\" (bool, optiona) if true, returns notarization evidence as well as other data\n" - "1. \"(separatecounterevidence)\" (bool, optiona) if true, counter-evidence is processed and returned with proofroots\n" + "2. \"(getevidence)\" (bool, optional) if true, returns notarization evidence as well as other data\n" + "1. \"(separatecounterevidence)\" (bool, optional) if true, counter-evidence is processed and returned with proofroots\n" "\nResult:\n" "{\n" @@ -4876,9 +4876,11 @@ UniValue getnotarizationproofs(const UniValue& params, bool fHelp) } } + bool postTestFork = !PBAAS_TESTMODE || chainActive[nHeight]->nTime > (PBAAS_TESTFORK_TIME - (20 * 60)); + // put the challenge proofs in first before the primary proof CCrossChainProof challengeProof(CCrossChainProof::VERSION_INVALID); - if (challengeRoots.size()) + if (postTestFork && challengeRoots.size()) { CPrimaryProofDescriptor proofDescr(CPrimaryProofDescriptor::VERSION_CURRENT); challengeProof.version = CCrossChainProof::VERSION_CURRENT; @@ -4973,138 +4975,139 @@ UniValue getnotarizationproofs(const UniValue& params, bool fHelp) } } - // EXPECT_FUTURE_PROOF_ROOT - future root to prove this notarization - evidence.evidence << futureRoot; - - // if we have a notarization to prove, prove it and the finalization, otherwise, prove - // a header - if (confirmNotarization.IsValid()) - { - evidence.evidence << notarizationTxProof; - evidence.evidence << CPartialTransactionProof(std::get<2>(finalizationToProve), - std::vector(), - std::vector({(int)std::get<1>(finalizationToProve).n}), - chainActive[std::get<0>(finalizationToProve)], - futureRoot.rootHeight); - } - else + if (postTestFork) { - // we are requested to prove the existence of this notarization on this chain - // from the future root - curMMV.resize(futureRoot.rootHeight + 1); + // EXPECT_FUTURE_PROOF_ROOT - future root to prove this notarization + evidence.evidence << futureRoot; - CMMRProof headerProof; - if (!chainActive.GetBlockProof(curMMV, headerProof, confirmRoot.rootHeight)) + // if we have a notarization to prove, prove it and the finalization, otherwise, prove + // a header + if (confirmNotarization.IsValid()) { - oneRetObj.pushKV("error", "Cannot prove evidence for challenge"); - break; + evidence.evidence << notarizationTxProof; + evidence.evidence << CPartialTransactionProof(std::get<2>(finalizationToProve), + std::vector(), + std::vector({(int)std::get<1>(finalizationToProve).n}), + chainActive[std::get<0>(finalizationToProve)], + futureRoot.rootHeight); } + else + { + // we are requested to prove the existence of this notarization on this chain + // from the future root + curMMV.resize(futureRoot.rootHeight + 1); - CBlockHeaderAndProof blockHeaderProof(headerProof, chainActive[confirmRoot.rootHeight]->GetBlockHeader()); - evidence.evidence << blockHeaderProof; - - // now prove any challenge proof related info from the notarization / root to confirm - curMMV.resize(confirmRoot.rootHeight + 1); - } - - uint32_t startHeight = priorNotarization.IsValid() && priorNotarization.proofRoots.count(ASSETCHAINS_CHAINID) ? - priorNotarization.proofRoots[ASSETCHAINS_CHAINID].rootHeight : - priorRoot.IsValid() ? priorRoot.rootHeight : 1; - - uint32_t heightChange = futureRoot.rootHeight - startHeight; + CMMRProof headerProof; + if (!chainActive.GetBlockProof(curMMV, headerProof, confirmRoot.rootHeight)) + { + oneRetObj.pushKV("error", "Cannot prove evidence for challenge"); + break; + } - int blocksPerCheckpoint = CPBaaSNotarization::GetBlocksPerCheckpoint(heightChange); + CBlockHeaderAndProof blockHeaderProof(headerProof, chainActive[confirmRoot.rootHeight]->GetBlockHeader()); + evidence.evidence << blockHeaderProof; - // unless we have another proof range sized chunk at the end, don't add another checkpoint - int numCheckPoints = CPBaaSNotarization::GetNumCheckpoints(heightChange); + // now prove any challenge proof related info from the notarization / root to confirm + curMMV.resize(confirmRoot.rootHeight + 1); + } - // let any challengers know where we might diverge roots - for (int i = 1; i <= numCheckPoints; i++) - { - evidence.evidence << CProofRoot::GetProofRoot(startHeight + (i * blocksPerCheckpoint)); - } + uint32_t startHeight = priorNotarization.IsValid() && priorNotarization.proofRoots.count(ASSETCHAINS_CHAINID) ? + priorNotarization.proofRoots[ASSETCHAINS_CHAINID].rootHeight : + priorRoot.IsValid() ? priorRoot.rootHeight : 1; - bool provideFullEvidence = challengeProof.IsValid(); + uint32_t heightChange = futureRoot.rootHeight - startHeight; - if (provideFullEvidence) - { - uint32_t rangeStart = fromHeight; - int32_t rangeLen = futureRoot.rootHeight - rangeStart; + int blocksPerCheckpoint = CPBaaSNotarization::GetBlocksPerCheckpoint(heightChange); - curMMV.resize(futureRoot.rootHeight + 1); + // unless we have another proof range sized chunk at the end, don't add another checkpoint + int numCheckPoints = CPBaaSNotarization::GetNumCheckpoints(heightChange); - std::vector<__uint128_t> blockCommitmentsSmall = GetBlockCommitments(rangeStart, futureRoot.rootHeight, entropyHash); - if (!blockCommitmentsSmall.size()) + // let any challengers know where we might diverge roots + for (int i = 1; i <= numCheckPoints; i++) { - oneRetObj.pushKV("error", "Cannot get block commitments for challenge"); - break; + evidence.evidence << CProofRoot::GetProofRoot(startHeight + (i * blocksPerCheckpoint)); } - evidence.evidence << CHashCommitments(blockCommitmentsSmall); - std::vector<__uint128_t> checkSmallCommitments; - auto checkTypes = ((CChainObject *)evidence.evidence.chainObjects.back())->object.GetSmallCommitments(checkSmallCommitments); - if (checkSmallCommitments != blockCommitmentsSmall) + if (challengeProof.IsValid()) { - LogPrintf("%s: ERROR - commitments evidence mismatch\n", __func__); - } + uint32_t rangeStart = fromHeight; + int32_t rangeLen = futureRoot.rootHeight - rangeStart; - if (LogAcceptCategory("notarization")) - { - for (int currentOffset = 0; currentOffset < checkSmallCommitments.size(); currentOffset++) + curMMV.resize(futureRoot.rootHeight + 1); + + std::vector<__uint128_t> blockCommitmentsSmall = GetBlockCommitments(rangeStart, futureRoot.rootHeight, entropyHash); + if (!blockCommitmentsSmall.size()) { - LogPrintf("%s: reading small commitments\n", __func__); - auto commitmentVec = UnpackBlockCommitment(checkSmallCommitments[currentOffset]); - arith_uint256 powTarget, posTarget; - powTarget.SetCompact(commitmentVec[1]); - posTarget.SetCompact(commitmentVec[2]); - LogPrintf("nHeight: %u, nTime: %u, PoW target: %s, PoS target: %s, isPoS: %u\n", - commitmentVec[3] >> 1, - commitmentVec[0], - powTarget.GetHex().c_str(), - posTarget.GetHex().c_str(), - commitmentVec[3] & 1); + oneRetObj.pushKV("error", "Cannot get block commitments for challenge"); + break; } - } - - int loopNum = std::min(std::max(rangeLen / CPBaaSNotarization::NUM_HEADER_PROOF_RANGE_DIVISOR, - (int)CPBaaSNotarization::EXPECT_MIN_HEADER_PROOFS), - (int)CPBaaSNotarization::MAX_HEADER_PROOFS_PER_PROOF); + evidence.evidence << CHashCommitments(blockCommitmentsSmall); - uint256 headerSelectionHash = entropyHash; - if (LogAcceptCategory("notarization")) - { - LogPrintf("%s: creating evidence with entropyHash: %s\n", __func__, entropyHash.GetHex().c_str()); - } + std::vector<__uint128_t> checkSmallCommitments; + auto checkTypes = ((CChainObject *)evidence.evidence.chainObjects.back())->object.GetSmallCommitments(checkSmallCommitments); + if (checkSmallCommitments != blockCommitmentsSmall) + { + LogPrintf("%s: ERROR - commitments evidence mismatch\n", __func__); + } - int headerLoop; - for (headerLoop = 0; headerLoop < loopNum; headerLoop++) - { - int indexToProve = (UintToArith256(headerSelectionHash).GetLow64() % rangeLen); - headerSelectionHash = ::GetHash(headerSelectionHash); - uint32_t blockToProve = rangeStart + indexToProve; - CMMRProof headerProof; - if (!chainActive.GetBlockProof(curMMV, headerProof, blockToProve)) + if (LogAcceptCategory("notarization")) { - oneRetObj.pushKV("error", "Cannot prove evidence for challenge"); - break; + for (int currentOffset = 0; currentOffset < checkSmallCommitments.size(); currentOffset++) + { + LogPrintf("%s: reading small commitments\n", __func__); + auto commitmentVec = UnpackBlockCommitment(checkSmallCommitments[currentOffset]); + arith_uint256 powTarget, posTarget; + powTarget.SetCompact(commitmentVec[1]); + posTarget.SetCompact(commitmentVec[2]); + LogPrintf("nHeight: %u, nTime: %u, PoW target: %s, PoS target: %s, isPoS: %u\n", + commitmentVec[3] >> 1, + commitmentVec[0], + powTarget.GetHex().c_str(), + posTarget.GetHex().c_str(), + commitmentVec[3] & 1); + } } - CBlockHeaderAndProof blockHeaderProof(headerProof, chainActive[blockToProve]->GetBlockHeader()); - evidence.evidence << blockHeaderProof; + int loopNum = std::min(std::max(rangeLen / CPBaaSNotarization::NUM_HEADER_PROOF_RANGE_DIVISOR, + (int)CPBaaSNotarization::EXPECT_MIN_HEADER_PROOFS), + (int)CPBaaSNotarization::MAX_HEADER_PROOFS_PER_PROOF); + + uint256 headerSelectionHash = entropyHash; + if (LogAcceptCategory("notarization")) + { + LogPrintf("%s: creating evidence with entropyHash: %s\n", __func__, entropyHash.GetHex().c_str()); + } - if (blockHeaderProof.blockHeader.IsVerusPOSBlock()) + int headerLoop; + for (headerLoop = 0; headerLoop < loopNum; headerLoop++) { - CValidationState state; - if (!ProvePosBlock(rangeStart, chainActive[blockToProve], evidence, state)) + int indexToProve = (UintToArith256(headerSelectionHash).GetLow64() % rangeLen); + headerSelectionHash = ::GetHash(headerSelectionHash); + uint32_t blockToProve = rangeStart + indexToProve; + CMMRProof headerProof; + if (!chainActive.GetBlockProof(curMMV, headerProof, blockToProve)) { - oneRetObj.pushKV("error", "Cannot prove primary evidence for PoS block"); + oneRetObj.pushKV("error", "Cannot prove evidence for challenge"); break; } + CBlockHeaderAndProof blockHeaderProof(headerProof, chainActive[blockToProve]->GetBlockHeader()); + + evidence.evidence << blockHeaderProof; + + if (blockHeaderProof.blockHeader.IsVerusPOSBlock()) + { + CValidationState state; + if (!ProvePosBlock(rangeStart, chainActive[blockToProve], evidence, state)) + { + oneRetObj.pushKV("error", "Cannot prove primary evidence for PoS block"); + break; + } + } + } + if (headerLoop < loopNum) + { + break; } - } - if (headerLoop < loopNum) - { - break; } } diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index ef9c4cd283f..4554246f4ed 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -534,4 +534,30 @@ BOOST_AUTO_TEST_CASE(rolling_bloom) } } +BOOST_AUTO_TEST_CASE(rolling_bloom_reset) +{ + struct TestRollingBloomFilter : CRollingBloomFilter + { + TestRollingBloomFilter() : CRollingBloomFilter(100, 0.01) {} + bool is_data_empty() const { return CRollingBloomFilter::is_data_empty(); } + }; + + TestRollingBloomFilter rb; + BOOST_CHECK(rb.is_data_empty()); + + std::vector d = RandomData(); + rb.insert(d); + BOOST_CHECK(!rb.is_data_empty()); + BOOST_CHECK(rb.contains(d)); + + // reset() should ensure minimal memory usage. + rb.reset(); + BOOST_CHECK(rb.is_data_empty()); + BOOST_CHECK(!rb.contains(d)); + + rb.insert(d); + BOOST_CHECK(!rb.is_data_empty()); + BOOST_CHECK(rb.contains(d)); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/txmempool.cpp b/src/txmempool.cpp index ee74ff30e33..bed3ebecd5d 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -26,7 +26,7 @@ using namespace std; CTxMemPoolEntry::CTxMemPoolEntry(): nFee(0), nTxSize(0), nModSize(0), nUsageSize(0), nTime(0), dPriority(0.0), - hadNoDependencies(false), spendsCoinbase(false), hasReserve(false) + hadNoDependencies(false), spendsCoinbase(false), hasReserve(false), feeDelta(0) { nHeight = MEMPOOL_HEIGHT; } @@ -34,14 +34,14 @@ CTxMemPoolEntry::CTxMemPoolEntry(): CTxMemPoolEntry::CTxMemPoolEntry(const CTransaction& _tx, const CAmount& _nFee, int64_t _nTime, double _dPriority, unsigned int _nHeight, bool poolHasNoInputsOf, - bool _spendsCoinbase, uint32_t _nBranchId, bool hasreserve): - tx(_tx), nFee(_nFee), nTime(_nTime), dPriority(_dPriority), nHeight(_nHeight), + bool _spendsCoinbase, uint32_t _nBranchId, bool hasreserve, int64_t FeeDelta): + tx(std::make_shared(_tx)), nFee(_nFee), nTime(_nTime), dPriority(_dPriority), nHeight(_nHeight), hadNoDependencies(poolHasNoInputsOf), hasReserve(hasreserve), - spendsCoinbase(_spendsCoinbase), nBranchId(_nBranchId) + spendsCoinbase(_spendsCoinbase), feeDelta(FeeDelta), nBranchId(_nBranchId) { nTxSize = ::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION); - nModSize = tx.CalculateModifiedSize(nTxSize); - nUsageSize = RecursiveDynamicUsage(tx); + nModSize = _tx.CalculateModifiedSize(nTxSize); + nUsageSize = RecursiveDynamicUsage(*tx) + memusage::DynamicUsage(tx); feeRate = CFeeRate(nFee, nTxSize); } @@ -53,13 +53,13 @@ CTxMemPoolEntry::CTxMemPoolEntry(const CTxMemPoolEntry& other) double CTxMemPoolEntry::GetPriority(unsigned int currentHeight) const { - CAmount nValueIn = tx.GetValueOut()+nFee; + CAmount nValueIn = tx->GetValueOut()+nFee; CCurrencyState currencyState; unsigned int lastHeight = currentHeight < 1 ? 0 : currentHeight - 1; AssertLockHeld(cs_main); if (hasReserve && (currencyState = ConnectedChains.GetCurrencyState(currentHeight - 1, false)).IsValid()) { - nValueIn += currencyState.ReserveToNative(tx.GetReserveValueOut()); + nValueIn += currencyState.ReserveToNative(tx->GetReserveValueOut()); } double deltaPriority = ((double)(currentHeight-nHeight)*nValueIn)/nModSize; double dResult = dPriority + deltaPriority; @@ -82,6 +82,21 @@ CTxMemPool::~CTxMemPool() delete minerPolicyEstimator; } +bool CTxMemPool::CompareDepthAndScore(const uint256& hasha, const uint256& hashb) +{ + LOCK(cs); + indexed_transaction_set::const_iterator i = mapTx.find(hasha); + if (i == mapTx.end()) return false; + indexed_transaction_set::const_iterator j = mapTx.find(hashb); + if (j == mapTx.end()) return true; + // We don't actually compare by depth here because we haven't backported + // https://github.com/bitcoin/bitcoin/pull/6654 + // + // But the method name is left as-is because renaming it is not worth the + // merge conflicts. + return CompareTxMemPoolEntryByScore()(*i, *j); +} + void CTxMemPool::pruneSpent(const uint256 &hashTx, CCoins &coins) { LOCK(cs); @@ -1087,6 +1102,24 @@ void CTxMemPool::queryHashes(vector& vtxid) vtxid.push_back(mi->GetTx().GetHash()); } +std::shared_ptr CTxMemPool::get(const uint256& hash) const +{ + LOCK(cs); + indexed_transaction_set::const_iterator i = mapTx.find(hash); + if (i == mapTx.end()) + return nullptr; + return i->GetSharedTx(); +} + +TxMempoolInfo CTxMemPool::info(const uint256& hash) const +{ + LOCK(cs); + indexed_transaction_set::const_iterator i = mapTx.find(hash); + if (i == mapTx.end()) + return TxMempoolInfo(); + return TxMempoolInfo{i->GetSharedTx(), i->GetTime(), CFeeRate(i->GetFee(), i->GetTxSize())}; +} + bool CTxMemPool::lookup(uint256 hash, CTransaction& result) const { LOCK(cs); diff --git a/src/txmempool.h b/src/txmempool.h index 595344f67be..dd1b30f3682 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -46,7 +46,7 @@ static const unsigned int MEMPOOL_HEIGHT = 0x7FFFFFFF; class CTxMemPoolEntry { private: - CTransaction tx; + std::shared_ptr tx; CAmount nFee; //! Cached to avoid expensive parent-transaction lookups size_t nTxSize; //! ... and avoid recomputing tx size size_t nModSize; //! ... and modified size for priority @@ -58,18 +58,22 @@ class CTxMemPoolEntry bool hadNoDependencies; //! Not dependent on any other txs when it entered the mempool bool spendsCoinbase; //! keep track of transactions that spend a coinbase bool hasReserve; //! keep track of transactions that hold reserve currency + int64_t feeDelta; //!< Used for determining the priority of the transaction for mining in a block uint32_t nBranchId; //! Branch ID this transaction is known to commit to, cached for efficiency public: CTxMemPoolEntry(const CTransaction& _tx, const CAmount& _nFee, int64_t _nTime, double _dPriority, unsigned int _nHeight, - bool poolHasNoInputsOf, bool spendsCoinbase, uint32_t nBranchId, bool hasreserve=false); + bool poolHasNoInputsOf, bool spendsCoinbase, uint32_t nBranchId, bool hasreserve=false, int64_t FeeDelta=0); CTxMemPoolEntry(); CTxMemPoolEntry(const CTxMemPoolEntry& other); - const CTransaction& GetTx() const { return this->tx; } + const CTransaction& GetTx() const { return *this->tx; } + std::shared_ptr GetSharedTx() const { return this->tx; } double GetPriority(unsigned int currentHeight) const; CAmount GetFee() const { return nFee; } + void UpdateFeeDelta(int64_t FeeDelta) { feeDelta = FeeDelta; } + int64_t GetModifiedFee() const { return nFee + feeDelta; } CFeeRate GetFeeRate() const { return feeRate; } size_t GetTxSize() const { return nTxSize; } int64_t GetTime() const { return nTime; } @@ -81,6 +85,16 @@ class CTxMemPoolEntry uint32_t GetValidatedBranchId() const { return nBranchId; } }; +struct update_fee_delta +{ + update_fee_delta(int64_t _feeDelta) : feeDelta(_feeDelta) { } + + void operator() (CTxMemPoolEntry &e) { e.UpdateFeeDelta(feeDelta); } + +private: + int64_t feeDelta; +}; + // extracts a TxMemPoolEntry's transaction hash struct mempoolentry_txid { @@ -102,6 +116,24 @@ class CompareTxMemPoolEntryByFee } }; +/** \class CompareTxMemPoolEntryByScore + * + * Sort by score of entry ((fee+delta)/size) in descending order + */ +class CompareTxMemPoolEntryByScore +{ +public: + bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) const + { + double f1 = (double)a.GetModifiedFee() * b.GetTxSize(); + double f2 = (double)b.GetModifiedFee() * a.GetTxSize(); + if (f1 == f2) { + return b.GetTx().GetHash() < a.GetTx().GetHash(); + } + return f1 > f2; + } +}; + class CBlockPolicyEstimator; /** An inpoint - a combination of a transaction and an index n into its vin */ @@ -118,6 +150,21 @@ class CInPoint size_t DynamicMemoryUsage() const { return 0; } }; +/** + * Information about a mempool transaction. + */ +struct TxMempoolInfo +{ + /** The transaction itself */ + std::shared_ptr tx; + + /** Time the transaction entered the mempool. */ + int64_t nTime; + + /** Feerate of the transaction. */ + CFeeRate feeRate; +}; + /** * CTxMemPool stores valid-according-to-the-current-best-chain * transactions that may be included in the next block. @@ -227,6 +274,7 @@ class CTxMemPool bool IsKnownReserveTransaction(const uint256 &hash, CReserveTransactionDescriptor &txDesc); // know to be reserve transaction, get descriptor, update mempool void ApplyDeltas(const uint256 hash, double &dPriorityDelta, CAmount &nFeeDelta); void ClearPrioritisation(const uint256 hash); + bool CompareDepthAndScore(const uint256& hasha, const uint256& hashb); bool nullifierExists(const uint256& nullifier, ShieldedType type) const; @@ -245,6 +293,9 @@ class CTxMemPool return totalTxSize; } + std::shared_ptr get(const uint256& hash) const; + TxMempoolInfo info(const uint256& hash) const; + bool exists(uint256 hash) const { LOCK(cs); diff --git a/src/uint256.h b/src/uint256.h index b07047b04f5..09d66f38ead 100644 --- a/src/uint256.h +++ b/src/uint256.h @@ -159,4 +159,9 @@ inline uint256 uint256S(const std::string& str) return rv; } +// Defined here so we have access to it in both primitives/transaction.h and protocol.h. +/* The placeholder value used for the auth digest of pre-v5 transactions. */ +static const uint256 LEGACY_TX_AUTH_DIGEST = + uint256S("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); + #endif // BITCOIN_UINT256_H diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index 34bccff7c30..b07a74f0acb 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -20,19 +20,15 @@ void RegisterValidationInterface(CValidationInterface* pwalletIn) { g_signals.RescanWallet.connect(boost::bind(&CValidationInterface::RescanWallet, pwalletIn)); g_signals.SetBestChain.connect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); g_signals.ChainTip.connect(boost::bind(&CValidationInterface::ChainTip, pwalletIn, _1, _2, _3, _4, _5)); - g_signals.Inventory.connect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); g_signals.Broadcast.connect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, _1)); g_signals.BlockChecked.connect(boost::bind(&CValidationInterface::BlockChecked, pwalletIn, _1, _2)); g_signals.ScriptForMining.connect(boost::bind(&CValidationInterface::GetScriptForMining, pwalletIn, _1)); - g_signals.BlockFound.connect(boost::bind(&CValidationInterface::ResetRequestCount, pwalletIn, _1)); } void UnregisterValidationInterface(CValidationInterface* pwalletIn) { - g_signals.BlockFound.disconnect(boost::bind(&CValidationInterface::ResetRequestCount, pwalletIn, _1)); g_signals.ScriptForMining.disconnect(boost::bind(&CValidationInterface::GetScriptForMining, pwalletIn, _1)); g_signals.BlockChecked.disconnect(boost::bind(&CValidationInterface::BlockChecked, pwalletIn, _1, _2)); g_signals.Broadcast.disconnect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, _1)); - g_signals.Inventory.disconnect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); g_signals.ChainTip.disconnect(boost::bind(&CValidationInterface::ChainTip, pwalletIn, _1, _2, _3, _4, _5)); g_signals.SetBestChain.disconnect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); g_signals.RescanWallet.disconnect(boost::bind(&CValidationInterface::RescanWallet, pwalletIn)); @@ -47,7 +43,6 @@ void UnregisterAllValidationInterfaces() { g_signals.ScriptForMining.disconnect_all_slots(); g_signals.BlockChecked.disconnect_all_slots(); g_signals.Broadcast.disconnect_all_slots(); - g_signals.Inventory.disconnect_all_slots(); g_signals.ChainTip.disconnect_all_slots(); g_signals.SetBestChain.disconnect_all_slots(); g_signals.RescanWallet.disconnect_all_slots(); diff --git a/src/validationinterface.h b/src/validationinterface.h index 5be992bb9e9..530aca4221d 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -44,11 +44,9 @@ class CValidationInterface { virtual void ChainTip(const CBlockIndex *pindex, const CBlock *pblock, SproutMerkleTree sproutTree, SaplingMerkleTree saplingTree, bool added) {} virtual void SetBestChain(const CBlockLocator &locator) {} virtual void UpdatedTransaction(const uint256 &hash) {} - virtual void Inventory(const uint256 &hash) {} virtual void ResendWalletTransactions(int64_t nBestBlockTime) {} virtual void BlockChecked(const CBlock&, const CValidationState&) {} virtual void GetScriptForMining(boost::shared_ptr&) {}; - virtual void ResetRequestCount(const uint256 &hash) {}; friend void ::RegisterValidationInterface(CValidationInterface*); friend void ::UnregisterValidationInterface(CValidationInterface*); friend void ::UnregisterAllValidationInterfaces(); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 96c67bb01a3..e609a601425 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -4036,45 +4036,6 @@ int64_t CWalletTx::GetTxTime() const return n ? n : nTimeReceived; } -int CWalletTx::GetRequestCount() const -{ - // Returns -1 if it wasn't being tracked - int nRequests = -1; - { - LOCK(pwallet->cs_wallet); - if (IsCoinBase()) - { - // Generated block - if (!hashBlock.IsNull()) - { - map::const_iterator mi = pwallet->mapRequestCount.find(hashBlock); - if (mi != pwallet->mapRequestCount.end()) - nRequests = (*mi).second; - } - } - else - { - // Did anyone request this transaction? - map::const_iterator mi = pwallet->mapRequestCount.find(GetHash()); - if (mi != pwallet->mapRequestCount.end()) - { - nRequests = (*mi).second; - - // How about the block it's in? - if (nRequests == 0 && !hashBlock.IsNull()) - { - map::const_iterator mi = pwallet->mapRequestCount.find(hashBlock); - if (mi != pwallet->mapRequestCount.end()) - nRequests = (*mi).second; - else - nRequests = 1; // If it's in someone else's block it must have got out - } - } - } - } - return nRequests; -} - // GetAmounts will determine the transparent debits and credits for a given wallet tx. void CWalletTx::GetAmounts(list& listReceived, list& listSent, CAmount& nFee, string& strSentAccount, const isminefilter& filter) const diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 1a2d9398074..be92a0d5abf 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -654,7 +654,6 @@ class CWalletTx : public CMerkleTx bool WriteToDisk(CWalletDB *pwalletdb); int64_t GetTxTime() const; - int GetRequestCount() const; bool RelayWalletTransaction(); @@ -1371,22 +1370,7 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface void UpdatedTransaction(const uint256 &hashTx); - void Inventory(const uint256 &hash) - { - { - LOCK(cs_wallet); - std::map::iterator mi = mapRequestCount.find(hash); - if (mi != mapRequestCount.end()) - (*mi).second++; - } - } - void GetScriptForMining(boost::shared_ptr &script); - void ResetRequestCount(const uint256 &hash) - { - LOCK(cs_wallet); - mapRequestCount[hash] = 0; - }; unsigned int GetKeyPoolSize() {