Index bypassed blocks from views without materializing.

include/bitcoin/node/chasers/chaser_validate.hpp

@@ -67,16 +67,18 @@ class BCN_API chaser_validate
     virtual void post_scan(const header_link& link, bool bypass) NOEXCEPT;
     virtual void validate_block(const header_link& link, bool bypass) NOEXCEPT;
     virtual void scan_block(const header_link& link, bool bypass) NOEXCEPT;
-    virtual code validate(bool& batched, bool& capturing, bool bypass,
+    virtual code validate(bool& batched, bool& capturing,
         const system::chain::block& block, const header_link& link,
         const system::chain::context& ctx) NOEXCEPT;
-    virtual code populate(bool bypass, const system::chain::block& block,
+    virtual code populate(const system::chain::block& block,
         const system::chain::context& ctx) NOEXCEPT;
-    virtual code validate_silent(const header_link& link, bool bank) NOEXCEPT;
+    virtual code validate_view(const header_link& link, bool filter,
+        bool silent, bool bank) NOEXCEPT;
+    virtual code complete_view(const header_link& link, bool filter,
+        bool silent, bool bank) NOEXCEPT;
     virtual code validate_pooled(bool& pooled, const header_link& link,
         const system::chain::context& ctx) NOEXCEPT;
-    virtual code complete_pooled(const header_link& link,
-        const system::chain::context& ctx) NOEXCEPT;
+    virtual code complete_pooled(const header_link& link) NOEXCEPT;
     virtual void complete_block(const code& ec, const header_link& link,
         size_t height, bool bypass, bool batched=false,
         bool capturing=false) NOEXCEPT;

include/bitcoin/node/error.hpp

@@ -137,7 +137,12 @@ enum error_t : uint8_t
     batch5,
     batch6,
     batch7,
-    batch8
+    batch8,
+
+    /// configuration
+    silent_unwitnessed,
+    silent_pruned,
+    filter_pruned
 };
 
 // No current need for error_code equivalence mapping.

src/chasers/chaser_validate_parallel.cpp

@@ -76,29 +76,28 @@ void chaser_validate::validate_block(const header_link& link,
     }
     else if (pooled)
     {
-        ec = complete_pooled(link, ctx);
+        ec = complete_pooled(link);
     }
-    else if (bypass && !filter_ && query.is_silent(link, ctx.height))
+    else if (bypass)
     {
-        ec = validate_silent(link, is_silent_capturing(link));
+        const auto silent = query.is_silent(link, ctx.height);
+        ec = complete_view(link, filter_, silent, silent &&
+            is_silent_capturing(link));
     }
     else
     {
-        // TODO: implement allocator parameter resulting in full allocation to
-        // shared_ptr<block>, to optimize deallocate (12% of milestone/filter).
         const auto block = query.get_block(link, node_witness_);
 
         if (!block)
         {
             ec = error::validate2;
         }
-        else if ((ec = populate(bypass, *block, ctx)))
+        else if ((ec = populate(*block, ctx)))
         {
             if (!query.set_block_unconfirmable(link))
                 ec = error::validate4;
         }
-        else if ((ec = validate(batched, capturing, bypass, *block, link,
-            ctx)))
+        else if ((ec = validate(batched, capturing, *block, link, ctx)))
         {
             if (!query.set_block_unconfirmable(link))
                 ec = error::validate5;
@@ -132,30 +131,18 @@ void chaser_validate::validate_block(const header_link& link,
 // helpers
 // ----------------------------------------------------------------------------
 
-code chaser_validate::populate(bool bypass, const chain::block& block,
+code chaser_validate::populate(const chain::block& block,
     const chain::context& ctx) NOEXCEPT
 {
-    const auto& query = archive();
+    // Internal maturity and time locks are verified here because they are
+    // the only necessary confirmation checks for internal spends.
+    if (const auto ec = block.populate(ctx))
+        return ec;
 
-    if (bypass)
-    {
-        // Populating for optional indexes only (no validation metadata).
-        block.populate(ctx);
-        if (!query.populate_without_metadata(block))
-            return system::error::missing_previous_output;
-    }
-    else
-    {
-        // Internal maturity and time locks are verified here because they are
-        // the only necessary confirmation checks for internal spends.
-        if (const auto ec = block.populate(ctx))
-            return ec;
+    // Metadata identifies internal spends allowing confirmation bypass.
+    if (!archive().populate_with_metadata(block))
+        return system::error::missing_previous_output;
 
-        // Metadata identifies internal spends allowing confirmation bypass.
-        if (!query.populate_with_metadata(block))
-            return system::error::missing_previous_output;
-    }
-
     return error::success;
 }
 
@@ -168,36 +155,57 @@ void chaser_validate::scan_block(const header_link& link,
 
     size_t height{};
     const auto ec = archive().get_height(height, link) ?
-        validate_silent(link, false) : error::validate3;
+        complete_view(link, false, true, false) : error::validate3;
 
     --validate_backlog_;
     complete_block(ec, link, height, bypass);
 }
 
-// Bypassed without client filters, a block is read only for its silent
-// payment records, so it is not materialized.
-code chaser_validate::validate_silent(const header_link& link,
-    bool bank) NOEXCEPT
+// A bypassed block is read only for its optional indexes, so it is not
+// materialized. A filter requires all prevouts, otherwise only those of the
+// silent payment eligible txs are populated.
+code chaser_validate::validate_view(const header_link& link, bool filter,
+    bool silent, bool bank) NOEXCEPT
 {
     auto& query = archive();
-    auto wire = query.get_wire_block(link, node_witness_);
-    chain::view::block block{ std::move(wire), node_witness_ };
+    const auto witness = node_witness_ && silent;
+    auto wire = query.get_wire_block(link, witness);
+    chain::view::block block{ std::move(wire), witness };
 
     if (!block.is_valid())
         return error::validate2;
 
-    // Only the prevouts of silent payment eligible txs are populated.
     data_chunk prevouts{};
     std::vector<bool> selected{};
-    if (!query.get_silent_prevouts(prevouts, selected, link, block) ||
+    if (!(filter ?
+        query.get_block_prevouts(prevouts, selected, link, block) :
+        query.get_silent_prevouts(prevouts, selected, link, block)) ||
         !block.populate(std::move(prevouts), selected))
-        return query.set_block_unconfirmable(link) ?
-            code{ system::error::missing_previous_output } : error::validate4;
+        return system::error::missing_previous_output;
+
+    if (filter && !query.set_filter_body(link, block))
+        return error::validate8;
 
-    if (bank ? !commit_silent_batch(link, block) :
-        !query.set_silent(link, block))
+    // Banked records are set when the bank drains.
+    if (silent && (bank ? !commit_silent_batch(link, block) :
+        !query.set_silent(link, block)))
         return error::validate9;
 
+    return error::success;
+}
+
+code chaser_validate::complete_view(const header_link& link, bool filter,
+    bool silent, bool bank) NOEXCEPT
+{
+    auto& query = archive();
+    const auto ec = validate_view(link, filter, silent, bank);
+    if (ec == system::error::missing_previous_output)
+        return query.set_block_unconfirmable(link) ? ec : error::validate4;
+
+    if (ec)
+        return ec;
+
+    // Valid must be set after set_filter_body.
     return query.set_block_valid(link) ? error::success : error::validate10;
 }
 
@@ -219,23 +227,16 @@ code chaser_validate::validate_pooled(bool& pooled, const header_link& link,
 }
 
 // A pooled block is valid and SP indexed, so produce only for client filters.
-code chaser_validate::complete_pooled(const header_link& link,
-    const chain::context& ctx) NOEXCEPT
+code chaser_validate::complete_pooled(const header_link& link) NOEXCEPT
 {
     auto& query = archive();
     if (filter_)
     {
-        bool batched{}, capturing{};
-        constexpr auto bypass = true;
-        const auto block = query.get_block(link, node_witness_);
-        if (!block)
-            return error::validate2;
-
-        if (populate(bypass, *block, ctx))
+        const auto ec = validate_view(link, true, false, false);
+        if (ec == system::error::missing_previous_output)
             return error::validate11;
 
-        if (const auto ec = validate(batched, capturing, bypass, *block, link,
-            ctx))
+        if (ec)
             return ec;
     }
 
@@ -246,48 +247,45 @@ code chaser_validate::complete_pooled(const header_link& link,
     return query.set_block_valid(link) ? error::success : error::validate10;
 }
 
-code chaser_validate::validate(bool& batched, bool& capturing, bool bypass,
+code chaser_validate::validate(bool& batched, bool& capturing,
     const chain::block& block, const header_link& link,
     const chain::context& ctx) NOEXCEPT
 {
     auto& query = archive();
 
-    if (!bypass)
-    {
-        code ec{};
-        if (((ec = block.check(false))) || ((ec = block.check(ctx, false))))
-            return ec;
+    code ec{};
+    if (((ec = block.check(false))) || ((ec = block.check(ctx, false))))
+        return ec;
 
-        if ((ec = block.accept(ctx, subsidy_interval_, initial_subsidy_)))
-            return ec;
+    if ((ec = block.accept(ctx, subsidy_interval_, initial_subsidy_)))
+        return ec;
 
-        // Initialize signature capture (appends to this thread's accumulators).
-        const auto capture = get_capture(link);
-        capturing = capture.enabled;
+    // Initialize signature capture (appends to this thread's accumulators).
+    const auto capture = get_capture(link);
+    capturing = capture.enabled;
 
-        ec = block.connect(ctx, capture);
+    ec = block.connect(ctx, capture);
 
-        // At least one signature batch was attempted (batch completion).
-        batched = capture.batched;
+    // At least one signature batch was attempted (batch completion).
+    batched = capture.batched;
 
-        // Commit (or discard) the captured signatures, marking the block prevalid
-        // contingent on batch verification. The commit epoch is mutually
-        // exclusive with batch verification (turnstile).
-        if (capturing)
-        {
-            if (!ec && batched)
-                ec = commit_capture(batched, link);
-            else
-                clear_capture();
-        }
+    // Commit (or discard) the captured signatures, marking the block prevalid
+    // contingent on batch verification. The commit epoch is mutually
+    // exclusive with batch verification (turnstile).
+    if (capturing)
+    {
+        if (!ec && batched)
+            ec = commit_capture(batched, link);
+        else
+            clear_capture();
+    }
 
-        if (ec)
-            return ec;
+    if (ec)
+        return ec;
 
-        // Prevouts optimize confirmation.
-        if (!query.set_prevouts(link, block))
-            return error::validate7;
-    }
+    // Prevouts optimize confirmation.
+    if (!query.set_prevouts(link, block))
+        return error::validate7;
 
     if (!query.set_filter_body(link, block))
         return error::validate8;

src/configuration.cpp

@@ -39,13 +39,18 @@ code configuration::initialize() NOEXCEPT
 {
     const auto silent = node.silent_start_height != max_uint32;
     if (silent && !node.require_witness)
-        return network::error::invalid_configuration;
+        return error::silent_unwitnessed;
 
     const auto checkpoint = bitcoin.top_checkpoint().height();
     const auto milestone = bitcoin.milestone.height();
     const auto bypassed = std::max(checkpoint, milestone);
     if (silent && node.limited_blocks && node.silent_start_height <= bypassed)
-        return network::error::invalid_configuration;
+        return error::silent_pruned;
+
+    const auto filter = to_bool(database.filter_bk.buckets) &&
+        to_bool(database.filter_tx.buckets);
+    if (filter && node.limited_blocks)
+        return error::filter_pruned;
 
     // Batch tables are unused when their batching does not run.
     if (!node.batch_verify_())

src/error.cpp

@@ -127,7 +127,12 @@ DEFINE_ERROR_T_MESSAGE_MAP(error)
     { batch5, "batch5" },
     { batch6, "batch6" },
     { batch7, "batch7" },
-    { batch8, "batch8" }
+    { batch8, "batch8" },
+
+    // configuration
+    { silent_unwitnessed, "silent payments require witness" },
+    { silent_pruned, "silent payments start in pruned blocks" },
+    { filter_pruned, "client filters require unpruned blocks" }
 };
 
 DEFINE_ERROR_T_CATEGORY(error, "node", "node code")

test/configuration.cpp

@@ -39,12 +39,12 @@ BOOST_AUTO_TEST_CASE(configuration__initialize__mainnet__success_peer_identifier
     BOOST_REQUIRE_EQUAL(instance.network.peer.identifier, 0xd9b4bef9_u32);
 }
 
-BOOST_AUTO_TEST_CASE(configuration__initialize__silent_without_witness__invalid_configuration)
+BOOST_AUTO_TEST_CASE(configuration__initialize__silent_without_witness__silent_unwitnessed)
 {
     node::configuration instance(chain::selection::mainnet);
     instance.node.silent_start_height = 0;
     instance.node.require_witness = false;
-    BOOST_REQUIRE_EQUAL(instance.initialize(), network::error::invalid_configuration);
+    BOOST_REQUIRE_EQUAL(instance.initialize(), node::error::silent_unwitnessed);
 }
 
 BOOST_AUTO_TEST_CASE(configuration__initialize__silent_with_witness__success)
@@ -55,13 +55,13 @@ BOOST_AUTO_TEST_CASE(configuration__initialize__silent_with_witness__success)
     BOOST_REQUIRE(!instance.initialize());
 }
 
-BOOST_AUTO_TEST_CASE(configuration__initialize__limited_silent_at_bypassed__invalid_configuration)
+BOOST_AUTO_TEST_CASE(configuration__initialize__limited_silent_at_bypassed__silent_pruned)
 {
     node::configuration instance(chain::selection::mainnet);
     const auto bypassed = std::max(instance.bitcoin.top_checkpoint().height(), instance.bitcoin.milestone.height());
     instance.node.limited_blocks = true;
     instance.node.silent_start_height = possible_narrow_cast<uint32_t>(bypassed);
-    BOOST_REQUIRE_EQUAL(instance.initialize(), network::error::invalid_configuration);
+    BOOST_REQUIRE_EQUAL(instance.initialize(), node::error::silent_pruned);
 }
 
 BOOST_AUTO_TEST_CASE(configuration__initialize__limited_silent_above_bypassed__success)
@@ -70,6 +70,8 @@ BOOST_AUTO_TEST_CASE(configuration__initialize__limited_silent_above_bypassed__s
     const auto bypassed = std::max(instance.bitcoin.top_checkpoint().height(), instance.bitcoin.milestone.height());
     instance.node.limited_blocks = true;
     instance.node.silent_start_height = possible_narrow_cast<uint32_t>(add1(bypassed));
+    instance.database.filter_bk.buckets = 0;
+    instance.database.filter_tx.buckets = 0;
     BOOST_REQUIRE(!instance.initialize());
 }
 
@@ -82,6 +84,33 @@ BOOST_AUTO_TEST_CASE(configuration__initialize__unlimited_silent_at_bypassed__su
     BOOST_REQUIRE(!instance.initialize());
 }
 
+BOOST_AUTO_TEST_CASE(configuration__initialize__limited_filter__filter_pruned)
+{
+    node::configuration instance(chain::selection::mainnet);
+    instance.node.limited_blocks = true;
+    instance.database.filter_bk.buckets = 1;
+    instance.database.filter_tx.buckets = 1;
+    BOOST_REQUIRE_EQUAL(instance.initialize(), node::error::filter_pruned);
+}
+
+BOOST_AUTO_TEST_CASE(configuration__initialize__limited_partial_filter__success)
+{
+    node::configuration instance(chain::selection::mainnet);
+    instance.node.limited_blocks = true;
+    instance.database.filter_bk.buckets = 1;
+    instance.database.filter_tx.buckets = 0;
+    BOOST_REQUIRE(!instance.initialize());
+}
+
+BOOST_AUTO_TEST_CASE(configuration__initialize__unlimited_filter__success)
+{
+    node::configuration instance(chain::selection::mainnet);
+    instance.node.limited_blocks = false;
+    instance.database.filter_bk.buckets = 1;
+    instance.database.filter_tx.buckets = 1;
+    BOOST_REQUIRE(!instance.initialize());
+}
+
 BOOST_AUTO_TEST_CASE(configuration__initialize__cpu_batching_without_gpu__tables_sized)
 {
     node::configuration instance(chain::selection::mainnet);