From d2dd4fd14055b864c36cb37dbbaa66b167fac871 Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Tue, 2 Jan 2024 11:06:18 -0500 Subject: [PATCH 1/8] partial Merge bitcoin/bitcoin#26684: bench: add readblock benchmark 1c4b9cbe906507295d8b7d52855de1441ad411dd bench: add readblock benchmark (Andrew Toth) Pull request description: Requested in https://github.com/bitcoin/bitcoin/pull/13151#issuecomment-385962450. See https://github.com/bitcoin/bitcoin/pull/26415 and https://github.com/bitcoin/bitcoin/pull/21319. Benchmarking shows a >50x increase in speed on both nvme and spinning disk. Benchmark results: | ns/op | op/s | err% | ins/op | cyc/op | IPC | bra/op | miss% | total | benchmark |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:---------- | 5,377,375.00 | 185.96 | 0.2% | 60,125,513.00 | 11,633,676.00 | 5.168 | 3,588,800.00 | 0.4% | 0.09 | `ReadBlockFromDiskTest` | ns/op | op/s | err% | ins/op | cyc/op | IPC | bra/op | miss% | total | benchmark |--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:---------- | 89,945.58 | 11,117.83 | 0.7% | 12,743.90 | 64,530.33 | 0.197 | 2,595.20 | 0.2% | 0.01 | `ReadRawBlockFromDiskTest` ACKs for top commit: maflcko: lgtm ACK 1c4b9cbe906507295d8b7d52855de1441ad411dd achow101: ACK 1c4b9cbe906507295d8b7d52855de1441ad411dd TheCharlatan: ACK 1c4b9cbe906507295d8b7d52855de1441ad411dd Tree-SHA512: 71dbcd6c7e2be97eb3001e35d0a95ef8e0c9b10dc9193025c7f8e11a09017fa2fbf89489b686353cd88fb409fb729fe2c4a25c567d2988f64c9c164ab09fba9f (cherry picked from commit 00bf4a171115844b53e5c955d93be8505ad787bb) Dash adaptation: the benchmark reads Dash's bundled block813851 instead of block413567, deserializes it without TX_WITH_WITNESS, and is registered in src/bench/CMakeLists.txt instead of Makefile.bench.include. Partial: ReadRawBlockFromDiskTest is left out because Dash's BlockManager has no ReadRawBlockFromDisk. Co-Authored-By: Claude Opus 5.5 --- src/bench/CMakeLists.txt | 1 + src/bench/readblock.cpp | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 src/bench/readblock.cpp diff --git a/src/bench/CMakeLists.txt b/src/bench/CMakeLists.txt index 8ce707bd35e7..d47596999a80 100644 --- a/src/bench/CMakeLists.txt +++ b/src/bench/CMakeLists.txt @@ -46,6 +46,7 @@ add_executable(bench_bitcoin pool.cpp pow_hash.cpp prevector.cpp + readblock.cpp rollingbloom.cpp rpc_blockchain.cpp rpc_mempool.cpp diff --git a/src/bench/readblock.cpp b/src/bench/readblock.cpp new file mode 100644 index 000000000000..af4df56f5d4d --- /dev/null +++ b/src/bench/readblock.cpp @@ -0,0 +1,38 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include + +#include +#include +#include +#include +#include +#include + +static FlatFilePos WriteBlockToDisk(ChainstateManager& chainman) +{ + DataStream stream{benchmark::data::block813851}; + CBlock block; + stream >> block; + + return chainman.m_blockman.SaveBlockToDisk(block, 0, nullptr); +} + +static void ReadBlockFromDiskTest(benchmark::Bench& bench) +{ + const auto testing_setup{MakeNoLogFileContext(ChainType::MAIN)}; + ChainstateManager& chainman{*testing_setup->m_node.chainman}; + + CBlock block; + const auto pos{WriteBlockToDisk(chainman)}; + + bench.run([&] { + const auto success{chainman.m_blockman.ReadBlockFromDisk(block, pos)}; + assert(success); + }); +} + +BENCHMARK(ReadBlockFromDiskTest, benchmark::PriorityLevel::HIGH); From e94c315b6de48be73070075c92639c284d2b2c96 Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Tue, 23 Jan 2024 16:16:00 -0500 Subject: [PATCH 2/8] Merge bitcoin/bitcoin#28921: multiprocess: Add basic type conversion hooks 6acec6b9ff02b91de132bb1575d75908a8a2d27b multiprocess: Add type conversion code for UniValue types (Ryan Ofsky) 0cc74fce72e0c79849109ee5d7afe707991b3512 multiprocess: Add type conversion code for serializable types (Ryan Ofsky) 4aaee239211a5287fbc361c0eb158b105ae8c8db test: add ipc test to test multiprocess type conversion code (Ryan Ofsky) Pull request description: Add type conversion hooks to allow `UniValue` objects, and objects that have `CDataStream` `Serialize` and `Unserialize` methods to be used as arguments and return values in Cap'nProto interface methods. Also add unit test to verify the hooks are working and data can be round-tripped correctly. The non-test code in this PR was previously part of #10102 and has been split off for easier review, but the test code is new. --- This PR is part of the [process separation project](https://github.com/bitcoin/bitcoin/issues/28722). ACKs for top commit: achow101: ACK 6acec6b9ff02b91de132bb1575d75908a8a2d27b dergoegge: reACK 6acec6b9ff02b91de132bb1575d75908a8a2d27b Tree-SHA512: 5d2cbc5215d488b876d34420adf91205dabf09b736183dcc85aa86255e3804c2bac5bab6792dacd585ef99a1d92cf29c8afb3eb65e4d953abc7ffe41994340c6 (cherry picked from commit 2f218c664b32d50d8964882eeb3e4f82f3448d00) Dash adaptations: - The Makefile.am / Makefile.test.include hunks become the IPC test block in src/test/CMakeLists.txt, restored as upstream's CMake tree has it (bitcoin#30454). The build_msvc hunk is dropped; Dash has no MSVC build. - Dash's SpanReader still takes a serialization type and version (bitcoin#28451 is not backported), so CustomReadField passes SER_NETWORK, CLIENT_VERSION. - Dash has no Txid type (bitcoin#28107), so the test builds the COutPoint from a plain uint256. - Dash's libmultiprocess pin already carries the update from bitcoin#31105, whose ipc_test.cpp hunk (m_rpc_system is now a std::optional) Dash's partial #31105 left out because the file did not exist yet; it is applied here so the test builds against that pin. Co-Authored-By: Claude Opus 5.5 --- src/ipc/capnp/common-types.h | 108 +++++++++++++++++++++++++++++++++++ src/test/.gitignore | 2 + src/test/CMakeLists.txt | 25 ++++++-- src/test/ipc_test.capnp | 18 ++++++ src/test/ipc_test.cpp | 67 ++++++++++++++++++++++ src/test/ipc_test.h | 21 +++++++ src/test/ipc_tests.cpp | 13 +++++ 7 files changed, 250 insertions(+), 4 deletions(-) create mode 100644 src/ipc/capnp/common-types.h create mode 100644 src/test/.gitignore create mode 100644 src/test/ipc_test.capnp create mode 100644 src/test/ipc_test.cpp create mode 100644 src/test/ipc_test.h create mode 100644 src/test/ipc_tests.cpp diff --git a/src/ipc/capnp/common-types.h b/src/ipc/capnp/common-types.h new file mode 100644 index 000000000000..4de906997ce9 --- /dev/null +++ b/src/ipc/capnp/common-types.h @@ -0,0 +1,108 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_IPC_CAPNP_COMMON_TYPES_H +#define BITCOIN_IPC_CAPNP_COMMON_TYPES_H + +#include +#include +#include + +#include +#include +#include +#include + +namespace ipc { +namespace capnp { +//! Use SFINAE to define Serializeable trait which is true if type T has a +//! Serialize(stream) method, false otherwise. +template +struct Serializable { +private: + template + static std::true_type test(decltype(std::declval().Serialize(std::declval()))*); + template + static std::false_type test(...); + +public: + static constexpr bool value = decltype(test(nullptr))::value; +}; + +//! Use SFINAE to define Unserializeable trait which is true if type T has +//! an Unserialize(stream) method, false otherwise. +template +struct Unserializable { +private: + template + static std::true_type test(decltype(std::declval().Unserialize(std::declval()))*); + template + static std::false_type test(...); + +public: + static constexpr bool value = decltype(test(nullptr))::value; +}; +} // namespace capnp +} // namespace ipc + +//! Functions to serialize / deserialize common bitcoin types. +namespace mp { +//! Overload multiprocess library's CustomBuildField hook to allow any +//! serializable object to be stored in a capnproto Data field or passed to a +//! canproto interface. Use Priority<1> so this hook has medium priority, and +//! higher priority hooks could take precedence over this one. +template +void CustomBuildField( + TypeList, Priority<1>, InvokeContext& invoke_context, Value&& value, Output&& output, + // Enable if serializeable and if LocalType is not cv or reference + // qualified. If LocalType is cv or reference qualified, it is important to + // fall back to lower-priority Priority<0> implementation of this function + // that strips cv references, to prevent this CustomBuildField overload from + // taking precedence over more narrow overloads for specific LocalTypes. + std::enable_if_t::value && + std::is_same_v>>>* enable = nullptr) +{ + DataStream stream; + value.Serialize(stream); + auto result = output.init(stream.size()); + memcpy(result.begin(), stream.data(), stream.size()); +} + +//! Overload multiprocess library's CustomReadField hook to allow any object +//! with an Unserialize method to be read from a capnproto Data field or +//! returned from canproto interface. Use Priority<1> so this hook has medium +//! priority, and higher priority hooks could take precedence over this one. +template +decltype(auto) +CustomReadField(TypeList, Priority<1>, InvokeContext& invoke_context, Input&& input, ReadDest&& read_dest, + std::enable_if_t::value>* enable = nullptr) +{ + return read_dest.update([&](auto& value) { + if (!input.has()) return; + auto data = input.get(); + SpanReader stream{SER_NETWORK, CLIENT_VERSION, {data.begin(), data.end()}}; + value.Unserialize(stream); + }); +} + +template +void CustomBuildField(TypeList, Priority<1>, InvokeContext& invoke_context, Value&& value, Output&& output) +{ + std::string str = value.write(); + auto result = output.init(str.size()); + memcpy(result.begin(), str.data(), str.size()); +} + +template +decltype(auto) CustomReadField(TypeList, Priority<1>, InvokeContext& invoke_context, Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + auto data = input.get(); + value.read(std::string_view{data.begin(), data.size()}); + }); +} +} // namespace mp + +#endif // BITCOIN_IPC_CAPNP_COMMON_TYPES_H diff --git a/src/test/.gitignore b/src/test/.gitignore new file mode 100644 index 000000000000..036df1430c56 --- /dev/null +++ b/src/test/.gitignore @@ -0,0 +1,2 @@ +# capnp generated files +*.capnp.* diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 5d495f2984a3..ccbc7c648c15 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -215,10 +215,27 @@ if(ENABLE_WALLET) add_subdirectory(${PROJECT_SOURCE_DIR}/src/wallet/test wallet) endif() -# NOTE: Upstream also builds an IPC test library here from ipc_test.cpp, -# ipc_test.capnp and ipc_tests.cpp. Dash has not backported those sources -# (bitcoin#28921), and referencing them makes -DWITH_MULTIPROCESS=ON fail -# during generation. Restore this block along with the test sources. +if(WITH_MULTIPROCESS) + add_library(bitcoin_ipc_test STATIC EXCLUDE_FROM_ALL + ipc_test.cpp + ) + + target_capnp_sources(bitcoin_ipc_test ${PROJECT_SOURCE_DIR} + ipc_test.capnp + ) + + target_link_libraries(bitcoin_ipc_test + PRIVATE + core_interface + univalue + ) + + target_sources(test_bitcoin + PRIVATE + ipc_tests.cpp + ) + target_link_libraries(test_bitcoin bitcoin_ipc_test) +endif() function(add_boost_test source_file) if(NOT EXISTS ${source_file}) diff --git a/src/test/ipc_test.capnp b/src/test/ipc_test.capnp new file mode 100644 index 000000000000..55a3dc268393 --- /dev/null +++ b/src/test/ipc_test.capnp @@ -0,0 +1,18 @@ +# Copyright (c) 2023 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +@0xd71b0fc8727fdf83; + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("gen"); + +using Proxy = import "/mp/proxy.capnp"; +$Proxy.include("test/ipc_test.h"); +$Proxy.includeTypes("ipc/capnp/common-types.h"); + +interface FooInterface $Proxy.wrap("FooImplementation") { + add @0 (a :Int32, b :Int32) -> (result :Int32); + passOutPoint @1 (arg :Data) -> (result :Data); + passUniValue @2 (arg :Text) -> (result :Text); +} diff --git a/src/test/ipc_test.cpp b/src/test/ipc_test.cpp new file mode 100644 index 000000000000..33d2cfb0681f --- /dev/null +++ b/src/test/ipc_test.cpp @@ -0,0 +1,67 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +//! Unit test that tests execution of IPC calls without actually creating a +//! separate process. This test is primarily intended to verify behavior of type +//! conversion code that converts C++ objects to Cap'n Proto messages and vice +//! versa. +//! +//! The test creates a thread which creates a FooImplementation object (defined +//! in ipc_test.h) and a two-way pipe accepting IPC requests which call methods +//! on the object through FooInterface (defined in ipc_test.capnp). +void IpcTest() +{ + // Setup: create FooImplemention object and listen for FooInterface requests + std::promise>> foo_promise; + std::function disconnect_client; + std::thread thread([&]() { + mp::EventLoop loop("IpcTest", [](bool raise, const std::string& log) { LogPrintf("LOG%i: %s\n", raise, log); }); + auto pipe = loop.m_io_context.provider->newTwoWayPipe(); + + auto connection_client = std::make_unique(loop, kj::mv(pipe.ends[0])); + auto foo_client = std::make_unique>( + connection_client->m_rpc_system->bootstrap(mp::ServerVatId().vat_id).castAs(), + connection_client.get(), /* destroy_connection= */ false); + foo_promise.set_value(std::move(foo_client)); + disconnect_client = [&] { loop.sync([&] { connection_client.reset(); }); }; + + auto connection_server = std::make_unique(loop, kj::mv(pipe.ends[1]), [&](mp::Connection& connection) { + auto foo_server = kj::heap>(std::make_shared(), connection); + return capnp::Capability::Client(kj::mv(foo_server)); + }); + connection_server->onDisconnect([&] { connection_server.reset(); }); + loop.loop(); + }); + std::unique_ptr> foo{foo_promise.get_future().get()}; + + // Test: make sure arguments were sent and return value is received + BOOST_CHECK_EQUAL(foo->add(1, 2), 3); + + COutPoint txout1{uint256{100}, 200}; + COutPoint txout2{foo->passOutPoint(txout1)}; + BOOST_CHECK(txout1 == txout2); + + UniValue uni1{UniValue::VOBJ}; + uni1.pushKV("i", 1); + uni1.pushKV("s", "two"); + UniValue uni2{foo->passUniValue(uni1)}; + BOOST_CHECK_EQUAL(uni1.write(), uni2.write()); + + // Test cleanup: disconnect pipe and join thread + disconnect_client(); + thread.join(); +} diff --git a/src/test/ipc_test.h b/src/test/ipc_test.h new file mode 100644 index 000000000000..bcfcc2125c68 --- /dev/null +++ b/src/test/ipc_test.h @@ -0,0 +1,21 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_IPC_TEST_H +#define BITCOIN_TEST_IPC_TEST_H + +#include +#include + +class FooImplementation +{ +public: + int add(int a, int b) { return a + b; } + COutPoint passOutPoint(COutPoint o) { return o; } + UniValue passUniValue(UniValue v) { return v; } +}; + +void IpcTest(); + +#endif // BITCOIN_TEST_IPC_TEST_H diff --git a/src/test/ipc_tests.cpp b/src/test/ipc_tests.cpp new file mode 100644 index 000000000000..6e144b0f418b --- /dev/null +++ b/src/test/ipc_tests.cpp @@ -0,0 +1,13 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include + +BOOST_AUTO_TEST_SUITE(ipc_tests) +BOOST_AUTO_TEST_CASE(ipc_tests) +{ + IpcTest(); +} +BOOST_AUTO_TEST_SUITE_END() From 790e0fcab87691ea1d0b0da83b19b6aa14749b29 Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Mon, 11 Mar 2024 09:34:19 -0400 Subject: [PATCH 3/8] Merge bitcoin/bitcoin#29458: refactor: Preallocate result in TryParseHex to avoid resizing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit a19235c14b3dc02de30b5d769de29d1752c23dbd Preallocate result in `TryParseHex` to avoid resizing (Lőrinc) b7489ecb52c1f99facb7c81c5e46963394d0620d Add benchmark for TryParseHex (Lőrinc) Pull request description: This pull request introduces optimizations to the `TryParseHex` function, focusing primarily on the ideal case (valid hexadecimal input without spaces). A new benchmark, `HexParse` was introduced in a separate commit. The main optimization preallocates the result vector based on the input string's length. This aims to completely avoid costly dynamic reallocations when no spaces are present. ------------ Before: ``` | ns/base16 | base16/s | err% | total | benchmark |--------------------:|--------------------:|--------:|----------:|:---------- | 1.60 | 623,238,893.11 | 0.3% | 0.01 | `HexParse` | 1.65 | 606,747,566.34 | 0.6% | 0.01 | `HexParse` | 1.60 | 626,149,544.07 | 0.3% | 0.01 | `HexParse` ``` After: ``` | ns/base16 | base16/s | err% | total | benchmark |--------------------:|--------------------:|--------:|----------:|:---------- | 0.68 | 1,465,555,976.27 | 0.8% | 0.01 | `HexParse` | 0.68 | 1,472,962,920.18 | 0.3% | 0.01 | `HexParse` | 0.68 | 1,476,159,423.00 | 0.3% | 0.01 | `HexParse` ``` ACKs for top commit: achow101: ACK a19235c14b3dc02de30b5d769de29d1752c23dbd hebasto: ACK a19235c14b3dc02de30b5d769de29d1752c23dbd. andrewtoth: Re-ACK a19235c14b3dc02de30b5d769de29d1752c23dbd Empact: Re-ACK https://github.com/bitcoin/bitcoin/pull/29458/commits/a19235c14b3dc02de30b5d769de29d1752c23dbd Tree-SHA512: e09a59791104be3fd1026862ce98de9efafa1f949626fa01e3b7d58e6a2ef02a11f0de55ddba5c43230a53effd24e6d368c1e12848b17e8ce91d7908a59333f0 (cherry picked from commit 6dda050865e298e2e3d5f23a8435981d1e017dbe) Dash adaptation: the benchmark is registered in src/bench/CMakeLists.txt instead of Makefile.bench.include. Co-Authored-By: Claude Opus 5.5 --- src/bench/CMakeLists.txt | 1 + src/bench/parse_hex.cpp | 36 ++++++++++++++++++++++++++++++++++++ src/util/strencodings.cpp | 2 ++ 3 files changed, 39 insertions(+) create mode 100644 src/bench/parse_hex.cpp diff --git a/src/bench/CMakeLists.txt b/src/bench/CMakeLists.txt index d47596999a80..b624a2540856 100644 --- a/src/bench/CMakeLists.txt +++ b/src/bench/CMakeLists.txt @@ -41,6 +41,7 @@ add_executable(bench_bitcoin mempool_stress.cpp merkle_root.cpp net_processing.cpp + parse_hex.cpp peer_eviction.cpp poly1305.cpp pool.cpp diff --git a/src/bench/parse_hex.cpp b/src/bench/parse_hex.cpp new file mode 100644 index 000000000000..db3ead043c81 --- /dev/null +++ b/src/bench/parse_hex.cpp @@ -0,0 +1,36 @@ +// Copyright (c) 2024- The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include + +std::string generateHexString(size_t length) { + const auto hex_digits = "0123456789ABCDEF"; + FastRandomContext rng(/*fDeterministic=*/true); + + std::string data; + while (data.size() < length) { + auto digit = hex_digits[rng.randbits(4)]; + data.push_back(digit); + } + return data; +} + +static void HexParse(benchmark::Bench& bench) +{ + auto data = generateHexString(130); // Generates 678B0EDA0A1FD30904D5A65E3568DB82DB2D918B0AD8DEA18A63FECCB877D07CAD1495C7157584D877420EF38B8DA473A6348B4F51811AC13C786B962BEE5668F9 by default + + bench.batch(data.size()).unit("base16").run([&] { + auto result = TryParseHex(data); + assert(result != std::nullopt); // make sure we're measuring the successful case + ankerl::nanobench::doNotOptimizeAway(result); + }); +} + +BENCHMARK(HexParse, benchmark::PriorityLevel::HIGH); diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index 6fc6c30bb400..7b5ded2975b5 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -81,6 +81,8 @@ template std::optional> TryParseHex(std::string_view str) { std::vector vch; + vch.reserve(str.size() / 2); // two hex characters form a single byte + auto it = str.begin(); while (it != str.end()) { if (IsSpace(*it)) { From 101403d88af5626c6c2d01edf678717b51d66b99 Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Tue, 11 Jun 2024 16:41:44 -0400 Subject: [PATCH 4/8] Merge bitcoin/bitcoin#28339: validation: improve performance of CheckBlockIndex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 5bc2077e8f592442b089affdf0b5795fbc053bb8 validation: allow to specify frequency for -checkblockindex (Martin Zumsande) d5a631b9597e5029a5048d9b8ad84ea4536bbac0 validation: improve performance of CheckBlockIndex (Martin Zumsande) 32c80413fdb063199f3bee719c4651bd63f05fce bench: add benchmark for checkblockindex (Martin Zumsande) Pull request description: `CheckBlockIndex() ` are consistency checks that are currently enabled by default on regtest. The function is rather slow, which is annoying if you * attempt to run it on other networks, especially if not fully synced * want to generate a long chain on regtest and see block generation slow down because you forgot to disable `-checkblockindex` or don't know it existed. One reason why it's slow is that in order to be able to traverse the block tree depth-first from genesis, it inserts pointers to all block indices into a `std::multimap` - for which inserts and lookups become slow once there are hundred thousands of entries. However, typically the block index is mostly chain-like with just a few forks so a multimap isn't really needed for the most part. This PR suggests to store the block indices of the chain ending in the best header in a vector instead, and store only the rest of the indices in a multimap. This does not change the actual consistency checks that are being performed for each index, just the way the block index tree is stored and traversed. This adds a bit of complication to make sure each block is visited (note that there are asserts that check it), making sure that the two containers are traversed correctly, but it speeds up the function considerably: On master, a single invocation of `CheckBlockIndex` takes ~1.4s on mainnet for me (4.9s on testnet which has >2.4 million blocks). With this branch, the runtime goes down to ~0.27s (0.85s on testnet).This is a speedup by a factor ~5. ACKs for top commit: achow101: ACK 5bc2077e8f592442b089affdf0b5795fbc053bb8 furszy: ACK 5bc2077e8f592442b089affdf0b5795fbc053bb8 ryanofsky: Code review ACK 5bc2077e8f592442b089affdf0b5795fbc053bb8. Just added suggested assert and simplification since last review Tree-SHA512: 6b9c3e3e5069d6152b45a09040f962380d114851ff0f9ff1771cf8cad7bb4fa0ba25cd787ceaa3dfa5241fb249748e2ee6987af0ccb24b786a5301b2836f8487 (cherry picked from commit 891e4bf3740774485c40d1c1a54075a1b69d6dde) Dash adaptations: - Dash's CheckBlockIndex has no snapshot (assumeutxo) bookkeeping, so the snap_update_firsts() call next to the reworded comment is not added. - The -checkblockindex help text takes upstream's new wording, which also repairs Dash's garbled "for the block tree, and occasionally". - The benchmark is registered in src/bench/CMakeLists.txt instead of Makefile.bench.include. Co-Authored-By: Claude Opus 5.5 --- src/bench/CMakeLists.txt | 1 + src/bench/checkblockindex.cpp | 20 ++++++++++ src/init.cpp | 2 +- src/kernel/chainstatemanager_opts.h | 2 +- src/node/chainstatemanager_args.cpp | 5 ++- src/test/util/setup_common.cpp | 2 +- src/validation.cpp | 58 ++++++++++++++++++++++------- src/validation.h | 2 +- 8 files changed, 73 insertions(+), 19 deletions(-) create mode 100644 src/bench/checkblockindex.cpp diff --git a/src/bench/CMakeLists.txt b/src/bench/CMakeLists.txt index b624a2540856..b3dc500f0c9f 100644 --- a/src/bench/CMakeLists.txt +++ b/src/bench/CMakeLists.txt @@ -24,6 +24,7 @@ add_executable(bench_bitcoin ccoins_caching.cpp chacha20.cpp checkblock.cpp + checkblockindex.cpp checkqueue.cpp disconnected_transactions.cpp crypto_hash.cpp diff --git a/src/bench/checkblockindex.cpp b/src/bench/checkblockindex.cpp new file mode 100644 index 000000000000..e8a848dbd409 --- /dev/null +++ b/src/bench/checkblockindex.cpp @@ -0,0 +1,20 @@ +// Copyright (c) 2023-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include + +static void CheckBlockIndex(benchmark::Bench& bench) +{ + auto testing_setup{MakeNoLogFileContext()}; + // Mine some more blocks + testing_setup->mineBlocks(1000); + bench.run([&] { + testing_setup->m_node.chainman->CheckBlockIndex(); + }); +} + + +BENCHMARK(CheckBlockIndex, benchmark::PriorityLevel::HIGH); diff --git a/src/init.cpp b/src/init.cpp index 2936d9e06fd9..a69e4b4f9495 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -821,7 +821,7 @@ void SetupServerArgs(ArgsManager& argsman) hidden_args.emplace_back("-zmqpubsequencehwm="); #endif - argsman.AddArg("-checkblockindex", strprintf("Do a consistency check for the block tree, and occasionally. (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-checkblockindex", strprintf("Do a consistency check for the block tree, chainstate, and other validation data structures every operations. Use 0 to disable. (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checkblocks=", strprintf("How many blocks to check at startup (default: %u, 0 = all)", DEFAULT_CHECKBLOCKS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checklevel=", strprintf("How thorough the block verification of -checkblocks is: %s (0-4, default: %u)", Join(CHECKLEVEL_DOC, ", "), DEFAULT_CHECKLEVEL), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checkaddrman=", strprintf("Run addrman consistency checks every operations. Use 0 to disable. (default: %u)", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); diff --git a/src/kernel/chainstatemanager_opts.h b/src/kernel/chainstatemanager_opts.h index 7fc875f26a30..75b10560e2c2 100644 --- a/src/kernel/chainstatemanager_opts.h +++ b/src/kernel/chainstatemanager_opts.h @@ -36,7 +36,7 @@ namespace kernel { struct ChainstateManagerOpts { const CChainParams& chainparams; fs::path datadir; - std::optional check_block_index{}; + std::optional check_block_index{}; bool checkpoints_enabled{DEFAULT_CHECKPOINTS_ENABLED}; //! If set, it will override the minimum work we will assume exists on some valid chain. std::optional minimum_chain_work{}; diff --git a/src/node/chainstatemanager_args.cpp b/src/node/chainstatemanager_args.cpp index 40d1c9540157..de242ffb503b 100644 --- a/src/node/chainstatemanager_args.cpp +++ b/src/node/chainstatemanager_args.cpp @@ -22,7 +22,10 @@ namespace node { std::optional ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts) { - if (auto value{args.GetBoolArg("-checkblockindex")}) opts.check_block_index = *value; + if (auto value{args.GetIntArg("-checkblockindex")}) { + // Interpret bare -checkblockindex argument as 1 instead of 0. + opts.check_block_index = args.GetArg("-checkblockindex")->empty() ? 1 : *value; + } if (auto value{args.GetBoolArg("-checkpoints")}) opts.checkpoints_enabled = *value; diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 35407c665f43..288c74fc80b0 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -284,7 +284,7 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, const std::vecto const ChainstateManager::Options chainman_opts{ .chainparams = chainparams, .datadir = m_args.GetDataDirNet(), - .check_block_index = true, + .check_block_index = 1, .superblock_status = [&sbman = *m_node.sbman](const CDeterministicMNList& mn_list, int height) { return sbman.GetStatus(mn_list, height); }, .mn_sync = m_node.mn_sync.get(), .notifications = *m_node.notifications, diff --git a/src/validation.cpp b/src/validation.cpp index 5f893abbd353..998a5734715d 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -5834,6 +5834,14 @@ void ChainstateManager::LoadExternalBlockFile( } } +bool ChainstateManager::ShouldCheckBlockIndex() const +{ + // Assert to verify Flatten() has been called. + if (!*Assert(m_options.check_block_index)) return false; + if (GetRand(*m_options.check_block_index) >= 1) return false; + return true; +} + void ChainstateManager::CheckBlockIndex() { if (!ShouldCheckBlockIndex()) { @@ -5850,19 +5858,28 @@ void ChainstateManager::CheckBlockIndex() return; } - // Build forward-pointing map of the entire block tree. + // Build forward-pointing data structure for the entire block tree. + // For performance reasons, indexes of the best header chain are stored in a vector (within CChain). + // All remaining blocks are stored in a multimap. + // The best header chain can differ from the active chain: E.g. its entries may belong to blocks that + // are not yet validated. + CChain best_hdr_chain; + assert(m_best_header); + best_hdr_chain.SetTip(*m_best_header); + std::multimap forward; for (auto& [_, block_index] : m_blockman.m_block_index) { - forward.emplace(block_index.pprev, &block_index); + // Only save indexes in forward that are not part of the best header chain. + if (!best_hdr_chain.Contains(&block_index)) { + // Only genesis, which must be part of the best header chain, can have a nullptr parent. + assert(block_index.pprev); + forward.emplace(block_index.pprev, &block_index); + } } + assert(forward.size() + best_hdr_chain.Height() + 1 == m_blockman.m_block_index.size()); - assert(forward.size() == m_blockman.m_block_index.size()); - - std::pair::iterator,std::multimap::iterator> rangeGenesis = forward.equal_range(nullptr); - CBlockIndex *pindex = rangeGenesis.first->second; - rangeGenesis.first++; - assert(rangeGenesis.first == rangeGenesis.second); // There is only one index entry with parent nullptr. - + CBlockIndex* pindex = best_hdr_chain[0]; + assert(pindex); // Iterate over the entire block tree, using depth-first search. // Along the way, remember whether there are blocks on the path from genesis // block being explored which are the first to have certain properties. @@ -6035,13 +6052,20 @@ void ChainstateManager::CheckBlockIndex() // assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // Perhaps too slow // End: actual consistency checks. - // Try descending into the first subnode. + + // Try descending into the first subnode. Always process forks first and the best header chain after. std::pair::iterator,std::multimap::iterator> range = forward.equal_range(pindex); if (range.first != range.second) { - // A subnode was found. + // A subnode not part of the best header chain was found. pindex = range.first->second; nHeight++; continue; + } else if (best_hdr_chain.Contains(pindex)) { + // Descend further into best header chain. + nHeight++; + pindex = best_hdr_chain[nHeight]; + if (!pindex) break; // we are finished, since the best header chain is always processed last + continue; } // This is a leaf node. // Move upwards until we reach a node of which we have not yet visited the last child. @@ -6068,9 +6092,15 @@ void ChainstateManager::CheckBlockIndex() // Proceed to the next one. rangePar.first++; if (rangePar.first != rangePar.second) { - // Move to the sibling. + // Move to a sibling not part of the best header chain. pindex = rangePar.first->second; break; + } else if (pindexPar == best_hdr_chain[nHeight - 1]) { + // Move to pindex's sibling on the best-chain, if it has one. + pindex = best_hdr_chain[nHeight]; + // There will not be a next block if (and only if) parent block is the best header. + assert((pindex == nullptr) == (pindexPar == best_hdr_chain.Tip())); + break; } else { // Move up further. pindex = pindexPar; @@ -6080,8 +6110,8 @@ void ChainstateManager::CheckBlockIndex() } } - // Check that we actually traversed the entire map. - assert(nNodes == forward.size()); + // Check that we actually traversed the entire block index. + assert(nNodes == forward.size() + best_hdr_chain.Height() + 1); } std::string Chainstate::ToString() diff --git a/src/validation.h b/src/validation.h index 1b914f00a593..ae08f5ef99bd 100644 --- a/src/validation.h +++ b/src/validation.h @@ -965,7 +965,7 @@ class ChainstateManager const CChainParams& GetParams() const { return m_options.chainparams; } const Consensus::Params& GetConsensus() const { return m_options.chainparams.GetConsensus(); } - bool ShouldCheckBlockIndex() const { return *Assert(m_options.check_block_index); } + bool ShouldCheckBlockIndex() const; const arith_uint256& MinimumChainWork() const { return *Assert(m_options.minimum_chain_work); } const uint256& AssumedValidBlock() const { return *Assert(m_options.assumed_valid_block); } kernel::Notifications& GetNotifications() const { return m_options.notifications; }; From 72eec94f75a15cc2455827d57c648158de21d87a Mon Sep 17 00:00:00 2001 From: fanquake Date: Thu, 2 Nov 2023 09:44:24 +0000 Subject: [PATCH 5/8] Merge bitcoin/bitcoin#28770: refactor: Remove unused circular include dependency from validation.cpp fa7d31910ab181c7e0e5f1fa1e23a49e208aec2a refactor: Remove unused circular include dependency from validation.cpp (MarcoFalke) Pull request description: Also, sort includes ACKs for top commit: ismaelsadeeq: ACK fa7d31910ab181c7e0e5f1fa1e23a49e208aec2a hebasto: ACK fa7d31910ab181c7e0e5f1fa1e23a49e208aec2a TheCharlatan: ACK fa7d31910ab181c7e0e5f1fa1e23a49e208aec2a Tree-SHA512: ea5e0001644d70ecfbccf87e27b393786a0eda79af4923ff68a0096d4d5b910cf6eeed8667ecbf55f3a164f500d3f5aeaf9d81bb190296c30ce0cc93c165717d (cherry picked from commit b2240f65228abc34837b69ffff149f184c1e3ded) Dash adaptation: Dash's validation.cpp never included kernel/chain.h, so only kernel/coinstats.h moves into the sorted include list. Prerequisite for bitcoin#30344, which moves mempool_persist to node/. Co-Authored-By: Claude Opus 5.5 --- src/validation.cpp | 4 +--- test/lint/lint-circular-dependencies.py | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/validation.cpp b/src/validation.cpp index 998a5734715d..8f0ea0c27a43 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -10,9 +10,6 @@ #include -#include -#include - #include #include #include @@ -30,6 +27,7 @@ #include #include #include +#include #include #include #include diff --git a/test/lint/lint-circular-dependencies.py b/test/lint/lint-circular-dependencies.py index a8912268946b..d1b543603106 100755 --- a/test/lint/lint-circular-dependencies.py +++ b/test/lint/lint-circular-dependencies.py @@ -20,7 +20,6 @@ "qt/transactiontablemodel -> qt/walletmodel -> qt/transactiontablemodel", "wallet/wallet -> wallet/walletdb -> wallet/wallet", "kernel/coinstats -> validation -> kernel/coinstats", - "kernel/mempool_persist -> validation -> kernel/mempool_persist", # Temporary, removed in followup https://github.com/bitcoin/bitcoin/pull/24230 "index/base -> node/context -> net_processing -> index/blockfilterindex -> index/base", From b8f7390a3117a5075383788549e6884dc1eeef15 Mon Sep 17 00:00:00 2001 From: glozow Date: Tue, 2 Jul 2024 10:04:43 +0100 Subject: [PATCH 6/8] Merge bitcoin/bitcoin#30344: kernel: remove mempool_persist f1478c05458562a9bef5c2ba43959d758e7b4745 mempool: move LoadMempool/DumpMempool to node (Cory Fields) 6d242ff1e9ca02fd8f5cb3ffe82dfb48a52366cc kernel: remove mempool_persist.cpp (Cory Fields) Pull request description: DumpMempool/LoadMempool are not necessary for the kernel. Noticed while working on instantiated logging. I suppose these could have been left in on purpose, but I'm assuming it was probably just an oversight. ACKs for top commit: TheCharlatan: Re-ACK f1478c05458562a9bef5c2ba43959d758e7b4745 glozow: ACK f1478c0545 stickies-v: ACK f1478c05458562a9bef5c2ba43959d758e7b4745 Tree-SHA512: 5825da0cf2e67470524eb6ebe397eb90755a368469a25f184df99ab935b3eb6d89eb802b41a6c3661e869bba3bbfa8ba9d95281bc75ebbf790ec5d9d1f79c66f (cherry picked from commit d2c8d161b46bd62256a17abd086d8ae138c043c3) Dash adaptation: the Makefile.am hunk becomes the matching move in src/CMakeLists.txt (bitcoin_node) and the removal of mempool_persist.cpp from the chainstate kernel library in src/kernel/CMakeLists.txt. Co-Authored-By: Claude Opus 5.5 --- src/CMakeLists.txt | 2 +- src/init.cpp | 6 +++--- src/kernel/CMakeLists.txt | 1 - src/{kernel => node}/mempool_persist.cpp | 6 +++--- src/{kernel => node}/mempool_persist.h | 10 +++++----- src/rpc/mempool.cpp | 8 ++++---- src/test/fuzz/validation_load_mempool.cpp | 6 +++--- 7 files changed, 19 insertions(+), 20 deletions(-) rename src/{kernel => node}/mempool_persist.cpp (98%) rename src/{kernel => node}/mempool_persist.h (84%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ce4aca735453..f9505740f66b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -342,7 +342,6 @@ add_library(bitcoin_node STATIC EXCLUDE_FROM_ALL kernel/context.cpp kernel/cs_main.cpp kernel/disconnected_transactions.cpp - kernel/mempool_persist.cpp kernel/mempool_removal_reason.cpp instantsend/db.cpp instantsend/instantsend.cpp @@ -394,6 +393,7 @@ add_library(bitcoin_node STATIC EXCLUDE_FROM_ALL node/interfaces.cpp node/kernel_notifications.cpp node/mempool_args.cpp + node/mempool_persist.cpp node/mempool_persist_args.cpp node/miner.cpp node/minisketchwrapper.cpp diff --git a/src/init.cpp b/src/init.cpp index a69e4b4f9495..ca94d87b176b 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -12,7 +12,6 @@ #include #include -#include #include #include @@ -63,6 +62,7 @@ #include #include #include +#include #include #include #include @@ -173,8 +173,6 @@ using kernel::CoinStatsHashType; using kernel::DEFAULT_STOPAFTERBLOCKIMPORT; -using kernel::DumpMempool; -using kernel::LoadMempool; using kernel::ValidationCacheSizes; using node::ApplyArgsManOptions; @@ -183,6 +181,8 @@ using node::CacheSizes; using node::CalculateCacheSizes; using node::DEFAULT_PERSIST_MEMPOOL; using node::DEFAULT_PRINTPRIORITY; +using node::DumpMempool; +using node::LoadMempool; using node::g_indexes_ready_to_sync; using node::KernelNotifications; using node::LoadChainstate; diff --git a/src/kernel/CMakeLists.txt b/src/kernel/CMakeLists.txt index 04ad20d008dc..fd082829f2f7 100644 --- a/src/kernel/CMakeLists.txt +++ b/src/kernel/CMakeLists.txt @@ -59,7 +59,6 @@ add_library(bitcoinkernel STATIC EXCLUDE_FROM_ALL context.cpp cs_main.cpp disconnected_transactions.cpp - mempool_persist.cpp mempool_removal_reason.cpp ../key.cpp ../key_io.cpp diff --git a/src/kernel/mempool_persist.cpp b/src/node/mempool_persist.cpp similarity index 98% rename from src/kernel/mempool_persist.cpp rename to src/node/mempool_persist.cpp index 2e9d26a87472..c7c0e5fcc164 100644 --- a/src/kernel/mempool_persist.cpp +++ b/src/node/mempool_persist.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include +#include #include #include @@ -32,7 +32,7 @@ using fsbridge::FopenFn; -namespace kernel { +namespace node { static const uint64_t MEMPOOL_DUMP_VERSION = 1; @@ -203,4 +203,4 @@ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mock return true; } -} // namespace kernel +} // namespace node diff --git a/src/kernel/mempool_persist.h b/src/node/mempool_persist.h similarity index 84% rename from src/kernel/mempool_persist.h rename to src/node/mempool_persist.h index e124a8eadf8f..7c5754a90c96 100644 --- a/src/kernel/mempool_persist.h +++ b/src/node/mempool_persist.h @@ -2,15 +2,15 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_KERNEL_MEMPOOL_PERSIST_H -#define BITCOIN_KERNEL_MEMPOOL_PERSIST_H +#ifndef BITCOIN_NODE_MEMPOOL_PERSIST_H +#define BITCOIN_NODE_MEMPOOL_PERSIST_H #include class Chainstate; class CTxMemPool; -namespace kernel { +namespace node { /** Dump the mempool to a file. */ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, @@ -28,7 +28,7 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active_chainstate, ImportMempoolOptions&& opts); -} // namespace kernel +} // namespace node -#endif // BITCOIN_KERNEL_MEMPOOL_PERSIST_H +#endif // BITCOIN_NODE_MEMPOOL_PERSIST_H diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index dfe293434a0e..f2979ef68b37 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include @@ -31,7 +31,7 @@ #include -using kernel::DumpMempool; +using node::DumpMempool; using node::DEFAULT_MAX_RAW_TX_FEE_RATE; using node::MempoolPath; @@ -774,13 +774,13 @@ static RPCHelpMan importmempool() const UniValue& use_current_time{request.params[1]["use_current_time"]}; const UniValue& apply_fee_delta{request.params[1]["apply_fee_delta_priority"]}; const UniValue& apply_unbroadcast{request.params[1]["apply_unbroadcast_set"]}; - kernel::ImportMempoolOptions opts{ + node::ImportMempoolOptions opts{ .use_current_time = use_current_time.isNull() ? true : use_current_time.get_bool(), .apply_fee_delta_priority = apply_fee_delta.isNull() ? false : apply_fee_delta.get_bool(), .apply_unbroadcast_set = apply_unbroadcast.isNull() ? false : apply_unbroadcast.get_bool(), }; - if (!kernel::LoadMempool(mempool, load_path, chainstate, std::move(opts))) { + if (!node::LoadMempool(mempool, load_path, chainstate, std::move(opts))) { throw JSONRPCError(RPC_MISC_ERROR, "Unable to import mempool file, see debug.log for details."); } diff --git a/src/test/fuzz/validation_load_mempool.cpp b/src/test/fuzz/validation_load_mempool.cpp index 5d020b4d5938..79981b35bce0 100644 --- a/src/test/fuzz/validation_load_mempool.cpp +++ b/src/test/fuzz/validation_load_mempool.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include +#include #include #include @@ -19,8 +19,8 @@ #include #include -using kernel::DumpMempool; -using kernel::LoadMempool; +using node::DumpMempool; +using node::LoadMempool; using node::MempoolPath; From da19d74a5b1193bafb227ef4ee7bf1d6d346e7c6 Mon Sep 17 00:00:00 2001 From: merge-script Date: Tue, 16 Jul 2024 12:13:02 +0100 Subject: [PATCH 7/8] Merge bitcoin/bitcoin#28263: Add fuzz test for FSChaCha20Poly1305, AEADChacha20Poly1305 8607773750e60931e51a33e48cd077a1dedf9db3 Add fuzz test for FSChaCha20Poly1305 (stratospher) c807f3322897ca8c0da114556e5936e389da5059 Add fuzz test for AEADChacha20Poly1305 (stratospher) Pull request description: This PR adds fuzz tests for `AEADChaCha20Poly1305` and `FSChaCha20Poly1305` introduced in #28008. Run using: ``` $ FUZZ=crypto_aeadchacha20poly1305 src/test/fuzz/fuzz $ FUZZ=crypto_fschacha20poly1305 src/test/fuzz/fuzz ``` ACKs for top commit: dergoegge: tACK 8607773750e60931e51a33e48cd077a1dedf9db3 marcofleon: Tested ACK 8607773750e60931e51a33e48cd077a1dedf9db3. Ran both targets for ~200 CPU hours. Coverage of intended targets looks good to me. The simulation of damaged keys and checks that follow seem useful as well. Tree-SHA512: b6b85661d896e653caeed330f941fde665fc2bbd97ecd340808a3f365c469fe9134aa77316569a771dc36d1158cac1a5f76700bcfc45fff12aef07562e48feb9 (cherry picked from commit 1db0be83535fc5cab6d2b42db27a38c155e15b4e) Dash adaptations: - Dash has no InsecureRandomContext (bitcoin#29625 and its follow-ups are not backported), so both targets seed a FastRandomContext from a fuzzer-chosen uint256. - The target is registered in src/test/fuzz/CMakeLists.txt instead of Makefile.test.include. Co-Authored-By: Claude Opus 5.5 --- src/test/fuzz/CMakeLists.txt | 1 + src/test/fuzz/crypto_chacha20poly1305.cpp | 200 ++++++++++++++++++++++ 2 files changed, 201 insertions(+) create mode 100644 src/test/fuzz/crypto_chacha20poly1305.cpp diff --git a/src/test/fuzz/CMakeLists.txt b/src/test/fuzz/CMakeLists.txt index 7baca6d0431e..d44545320aca 100644 --- a/src/test/fuzz/CMakeLists.txt +++ b/src/test/fuzz/CMakeLists.txt @@ -29,6 +29,7 @@ add_executable(fuzz crypto_aes256.cpp crypto_aes256cbc.cpp crypto_chacha20.cpp + crypto_chacha20poly1305.cpp crypto_common.cpp crypto_diff_fuzz_chacha20.cpp crypto_hkdf_hmac_sha256_l32.cpp diff --git a/src/test/fuzz/crypto_chacha20poly1305.cpp b/src/test/fuzz/crypto_chacha20poly1305.cpp new file mode 100644 index 000000000000..9eb3f5271cc4 --- /dev/null +++ b/src/test/fuzz/crypto_chacha20poly1305.cpp @@ -0,0 +1,200 @@ +// Copyright (c) 2020-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +constexpr static inline void crypt_till_rekey(FSChaCha20Poly1305& aead, int rekey_interval, bool encrypt) +{ + for (int i = 0; i < rekey_interval; ++i) { + std::byte dummy_tag[FSChaCha20Poly1305::EXPANSION] = {{}}; + if (encrypt) { + aead.Encrypt(Span{dummy_tag}.first(0), Span{dummy_tag}.first(0), dummy_tag); + } else { + aead.Decrypt(dummy_tag, Span{dummy_tag}.first(0), Span{dummy_tag}.first(0)); + } + } +} + +FUZZ_TARGET(crypto_aeadchacha20poly1305) +{ + FuzzedDataProvider provider{buffer.data(), buffer.size()}; + + auto key = provider.ConsumeBytes(32); + key.resize(32); + AEADChaCha20Poly1305 aead(key); + + // Initialize RNG deterministically, to generate contents and AAD. We assume that there are no + // (potentially buggy) edge cases triggered by specific values of contents/AAD, so we can avoid + // reading the actual data for those from the fuzzer input (which would need large amounts of + // data). + FastRandomContext rng{ConsumeUInt256(provider)}; + + LIMITED_WHILE(provider.ConsumeBool(), 10000) + { + // Mode: + // - Bit 0: whether to use single-plain Encrypt/Decrypt; otherwise use a split at prefix. + // - Bit 2: whether this ciphertext will be corrupted (making it the last sent one) + // - Bit 3-4: controls the maximum aad length (max 511 bytes) + // - Bit 5-7: controls the maximum content length (max 16383 bytes, for performance reasons) + unsigned mode = provider.ConsumeIntegral(); + bool use_splits = mode & 1; + bool damage = mode & 4; + unsigned aad_length_bits = 3 * ((mode >> 3) & 3); + unsigned aad_length = provider.ConsumeIntegralInRange(0, (1 << aad_length_bits) - 1); + unsigned length_bits = 2 * ((mode >> 5) & 7); + unsigned length = provider.ConsumeIntegralInRange(0, (1 << length_bits) - 1); + // Generate aad and content. + auto aad = rng.randbytes(aad_length); + auto plain = rng.randbytes(length); + std::vector cipher(length + AEADChaCha20Poly1305::EXPANSION); + // Generate nonce + AEADChaCha20Poly1305::Nonce96 nonce = {(uint32_t)rng(), rng()}; + + if (use_splits && length > 0) { + size_t split_index = provider.ConsumeIntegralInRange(1, length); + aead.Encrypt(Span{plain}.first(split_index), Span{plain}.subspan(split_index), aad, nonce, cipher); + } else { + aead.Encrypt(plain, aad, nonce, cipher); + } + + // Test Keystream output + std::vector keystream(length); + aead.Keystream(nonce, keystream); + for (size_t i = 0; i < length; ++i) { + assert((plain[i] ^ keystream[i]) == cipher[i]); + } + + std::vector decrypted_contents(length); + bool ok{false}; + + // damage the key + unsigned key_position = provider.ConsumeIntegralInRange(0, 31); + std::byte damage_val{(uint8_t)(1U << (key_position & 7))}; + std::vector bad_key = key; + bad_key[key_position] ^= damage_val; + + AEADChaCha20Poly1305 bad_aead(bad_key); + ok = bad_aead.Decrypt(cipher, aad, nonce, decrypted_contents); + assert(!ok); + + // Optionally damage 1 bit in either the cipher (corresponding to a change in transit) + // or the aad (to make sure that decryption will fail if the AAD mismatches). + if (damage) { + unsigned damage_bit = provider.ConsumeIntegralInRange(0, (cipher.size() + aad.size()) * 8U - 1U); + unsigned damage_pos = damage_bit >> 3; + std::byte damage_val{(uint8_t)(1U << (damage_bit & 7))}; + if (damage_pos >= cipher.size()) { + aad[damage_pos - cipher.size()] ^= damage_val; + } else { + cipher[damage_pos] ^= damage_val; + } + } + + if (use_splits && length > 0) { + size_t split_index = provider.ConsumeIntegralInRange(1, length); + ok = aead.Decrypt(cipher, aad, nonce, Span{decrypted_contents}.first(split_index), Span{decrypted_contents}.subspan(split_index)); + } else { + ok = aead.Decrypt(cipher, aad, nonce, decrypted_contents); + } + + // Decryption *must* fail if the packet was damaged, and succeed if it wasn't. + assert(!ok == damage); + if (!ok) break; + assert(decrypted_contents == plain); + } +} + +FUZZ_TARGET(crypto_fschacha20poly1305) +{ + FuzzedDataProvider provider{buffer.data(), buffer.size()}; + + uint32_t rekey_interval = provider.ConsumeIntegralInRange(32, 512); + auto key = provider.ConsumeBytes(32); + key.resize(32); + FSChaCha20Poly1305 enc_aead(key, rekey_interval); + FSChaCha20Poly1305 dec_aead(key, rekey_interval); + + // Initialize RNG deterministically, to generate contents and AAD. We assume that there are no + // (potentially buggy) edge cases triggered by specific values of contents/AAD, so we can avoid + // reading the actual data for those from the fuzzer input (which would need large amounts of + // data). + FastRandomContext rng{ConsumeUInt256(provider)}; + + LIMITED_WHILE(provider.ConsumeBool(), 10000) + { + // Mode: + // - Bit 0: whether to use single-plain Encrypt/Decrypt; otherwise use a split at prefix. + // - Bit 2: whether this ciphertext will be corrupted (making it the last sent one) + // - Bit 3-4: controls the maximum aad length (max 511 bytes) + // - Bit 5-7: controls the maximum content length (max 16383 bytes, for performance reasons) + unsigned mode = provider.ConsumeIntegral(); + bool use_splits = mode & 1; + bool damage = mode & 4; + unsigned aad_length_bits = 3 * ((mode >> 3) & 3); + unsigned aad_length = provider.ConsumeIntegralInRange(0, (1 << aad_length_bits) - 1); + unsigned length_bits = 2 * ((mode >> 5) & 7); + unsigned length = provider.ConsumeIntegralInRange(0, (1 << length_bits) - 1); + // Generate aad and content. + auto aad = rng.randbytes(aad_length); + auto plain = rng.randbytes(length); + std::vector cipher(length + FSChaCha20Poly1305::EXPANSION); + + crypt_till_rekey(enc_aead, rekey_interval, true); + if (use_splits && length > 0) { + size_t split_index = provider.ConsumeIntegralInRange(1, length); + enc_aead.Encrypt(Span{plain}.first(split_index), Span{plain}.subspan(split_index), aad, cipher); + } else { + enc_aead.Encrypt(plain, aad, cipher); + } + + std::vector decrypted_contents(length); + bool ok{false}; + + // damage the key + unsigned key_position = provider.ConsumeIntegralInRange(0, 31); + std::byte damage_val{(uint8_t)(1U << (key_position & 7))}; + std::vector bad_key = key; + bad_key[key_position] ^= damage_val; + + FSChaCha20Poly1305 bad_fs_aead(bad_key, rekey_interval); + crypt_till_rekey(bad_fs_aead, rekey_interval, false); + ok = bad_fs_aead.Decrypt(cipher, aad, decrypted_contents); + assert(!ok); + + // Optionally damage 1 bit in either the cipher (corresponding to a change in transit) + // or the aad (to make sure that decryption will fail if the AAD mismatches). + if (damage) { + unsigned damage_bit = provider.ConsumeIntegralInRange(0, (cipher.size() + aad.size()) * 8U - 1U); + unsigned damage_pos = damage_bit >> 3; + std::byte damage_val{(uint8_t)(1U << (damage_bit & 7))}; + if (damage_pos >= cipher.size()) { + aad[damage_pos - cipher.size()] ^= damage_val; + } else { + cipher[damage_pos] ^= damage_val; + } + } + + crypt_till_rekey(dec_aead, rekey_interval, false); + if (use_splits && length > 0) { + size_t split_index = provider.ConsumeIntegralInRange(1, length); + ok = dec_aead.Decrypt(cipher, aad, Span{decrypted_contents}.first(split_index), Span{decrypted_contents}.subspan(split_index)); + } else { + ok = dec_aead.Decrypt(cipher, aad, decrypted_contents); + } + + // Decryption *must* fail if the packet was damaged, and succeed if it wasn't. + assert(!ok == damage); + if (!ok) break; + assert(decrypted_contents == plain); + } +} From 0e291d3bfc9eab9ea71a19e293bc21d11bfddae3 Mon Sep 17 00:00:00 2001 From: Ava Chow Date: Mon, 12 Aug 2024 16:06:38 -0400 Subject: [PATCH 8/8] Merge bitcoin/bitcoin#28209: fuzz: a target for the block index database MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 86b38529d5014612c3e7bb59fdc4dad3bff2aa64 qa: a fuzz target for the block index database (Antoine Poinsot) Pull request description: This introduces a small fuzz target for `CBlockTreeDB` which asserts a few invariants by using an in-memory LevelDb. ACKs for top commit: achow101: ACK 86b38529d5014612c3e7bb59fdc4dad3bff2aa64 TheCharlatan: Re-ACK 86b38529d5014612c3e7bb59fdc4dad3bff2aa64 maflcko: review ACK 86b38529d5014612c3e7bb59fdc4dad3bff2aa64 🥒 brunoerg: utACK 86b38529d5014612c3e7bb59fdc4dad3bff2aa64 Tree-SHA512: ab75b4ae1c7e0a4b15f8a6ceffdf509fbc79833e6ea073ecef68558d53b83663d1b30362aaa2d77c22b8890a572f5b1d4b1c5abbca483c8c8f9b1fb5b276a59a (cherry picked from commit ee57737bd66017f4e5a25604eacc8c35143b29ce) Dash adaptations: - Dash still calls the block tree database CBlockTreeDB; it becomes kernel::BlockTreeDB with bitcoin#28191, which Dash has not backported. - Dash's CBlockTreeDB::LoadBlockIndexGuts() takes no interrupt argument (it checks ShutdownRequested(); the SignalInterrupt argument comes with bitcoin#27861), so the target calls it without m_interrupt. - The target is registered in src/test/fuzz/CMakeLists.txt instead of Makefile.test.include. Co-Authored-By: Claude Opus 5.5 --- src/test/fuzz/CMakeLists.txt | 1 + src/test/fuzz/block_index.cpp | 133 ++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 src/test/fuzz/block_index.cpp diff --git a/src/test/fuzz/CMakeLists.txt b/src/test/fuzz/CMakeLists.txt index d44545320aca..2ffb195a4cb7 100644 --- a/src/test/fuzz/CMakeLists.txt +++ b/src/test/fuzz/CMakeLists.txt @@ -17,6 +17,7 @@ add_executable(fuzz bitset.cpp block.cpp block_header.cpp + block_index.cpp blockfilter.cpp bloom_filter.cpp buffered_file.cpp diff --git a/src/test/fuzz/block_index.cpp b/src/test/fuzz/block_index.cpp new file mode 100644 index 000000000000..6056e6f87da7 --- /dev/null +++ b/src/test/fuzz/block_index.cpp @@ -0,0 +1,133 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +const BasicTestingSetup* g_setup; + +// Hardcoded block hash and nBits to make sure the blocks we store pass the pow check. +uint256 g_block_hash; + +bool operator==(const CBlockFileInfo& a, const CBlockFileInfo& b) +{ + return a.nBlocks == b.nBlocks && + a.nSize == b.nSize && + a.nUndoSize == b.nUndoSize && + a.nHeightFirst == b.nHeightFirst && + a.nHeightLast == b.nHeightLast && + a.nTimeFirst == b.nTimeFirst && + a.nTimeLast == b.nTimeLast; +} + +CBlockHeader ConsumeBlockHeader(FuzzedDataProvider& provider) +{ + CBlockHeader header; + header.nVersion = provider.ConsumeIntegral(); + header.hashPrevBlock = g_block_hash; + header.hashMerkleRoot = g_block_hash; + header.nTime = provider.ConsumeIntegral(); + header.nBits = Params().GenesisBlock().nBits; + header.nNonce = provider.ConsumeIntegral(); + return header; +} + +} // namespace + +void init_block_index() +{ + static const auto testing_setup = MakeNoLogFileContext<>(ChainType::MAIN); + g_setup = testing_setup.get(); + g_block_hash = Params().GenesisBlock().GetHash(); +} + +FUZZ_TARGET(block_index, .init = init_block_index) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + auto block_index = CBlockTreeDB(DBParams{ + .path = "", // Memory only. + .cache_bytes = 1 << 20, // 1MB. + .memory_only = true, + }); + + // Generate a number of block files to be stored in the index. + int files_count = fuzzed_data_provider.ConsumeIntegralInRange(1, 100); + std::vector> files; + files.reserve(files_count); + std::vector> files_info; + files_info.reserve(files_count); + for (int i = 0; i < files_count; i++) { + if (auto file_info = ConsumeDeserializable(fuzzed_data_provider)) { + files.push_back(std::make_unique(std::move(*file_info))); + files_info.emplace_back(i, files.back().get()); + } else { + return; + } + } + + // Generate a number of block headers to be stored in the index. + int blocks_count = fuzzed_data_provider.ConsumeIntegralInRange(files_count * 10, files_count * 100); + std::vector> blocks; + blocks.reserve(blocks_count); + std::vector blocks_info; + blocks_info.reserve(blocks_count); + for (int i = 0; i < blocks_count; i++) { + CBlockHeader header{ConsumeBlockHeader(fuzzed_data_provider)}; + blocks.push_back(std::make_unique(std::move(header))); + blocks.back()->phashBlock = &g_block_hash; + blocks_info.push_back(blocks.back().get()); + } + + // Store these files and blocks in the block index. It should not fail. + assert(block_index.WriteBatchSync(files_info, files_count - 1, blocks_info)); + + // We should be able to read every block file info we stored. Its value should correspond to + // what we stored above. + CBlockFileInfo info; + for (const auto& [n, file_info]: files_info) { + assert(block_index.ReadBlockFileInfo(n, info)); + assert(info == *file_info); + } + + // We should be able to read the last block file number. Its value should be consistent. + int last_block_file; + assert(block_index.ReadLastBlockFile(last_block_file)); + assert(last_block_file == files_count - 1); + + // We should be able to flip and read the reindexing flag. + bool reindexing; + block_index.WriteReindexing(true); + block_index.ReadReindexing(reindexing); + assert(reindexing); + block_index.WriteReindexing(false); + block_index.ReadReindexing(reindexing); + assert(!reindexing); + + // We should be able to set and read the value of any random flag. + const std::string flag_name = fuzzed_data_provider.ConsumeRandomLengthString(100); + bool flag_value; + block_index.WriteFlag(flag_name, true); + block_index.ReadFlag(flag_name, flag_value); + assert(flag_value); + block_index.WriteFlag(flag_name, false); + block_index.ReadFlag(flag_name, flag_value); + assert(!flag_value); + + // We should be able to load everything we've previously stored. Note to assert on the + // return value we need to make sure all blocks pass the pow check. + const auto params{Params().GetConsensus()}; + const auto inserter = [&](const uint256&) { + return blocks.back().get(); + }; + WITH_LOCK(::cs_main, assert(block_index.LoadBlockIndexGuts(params, inserter))); +}