Trim headers when loading block index from disk.

Author: gwillen

src/txdb.cpp

@@ -307,12 +307,47 @@ bool CBlockTreeDB::WritePAKList(const std::vector<std::vector<unsigned char> >&
         return Write(std::make_pair(DB_PAK, uint256S("1")), offline_list) && Write(std::make_pair(DB_PAK, uint256S("2")), online_list) && Write(std::make_pair(DB_PAK, uint256S("3")), reject);
 }
 
-bool CBlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex)
+bool CBlockTreeDB::WalkBlockIndexGutsForMaxHeight(int* nHeight) {
+    std::unique_ptr<CDBIterator> pcursor(NewIterator());
+    *nHeight = 0;
+    int i = 0;
+    pcursor->Seek(std::make_pair(DB_BLOCK_INDEX, uint256()));
+    while (pcursor->Valid()) {
+        if (ShutdownRequested()) return false;
+        std::pair<uint8_t, uint256> key;
+        if (pcursor->GetKey(key) && key.first == DB_BLOCK_INDEX) {
+            i++;
+            if (i > 10'000) {
+                // Under the (accurate) assumption hat the headers on disk are effectively in random height order,
+                //   we have a good-enough (conservative) estimate of the max height very quickly, and don't need to
+                //   waste more time. Shortcutting like this will cause us to keep a few extra headers, which is fine.
+                break;
+            }
+            CDiskBlockIndex diskindex;
+            if (pcursor->GetValue(diskindex)) {
+                if (diskindex.nHeight > *nHeight) {
+                    *nHeight = diskindex.nHeight;
+                }
+                pcursor->Next();
+            } else {
+                return error("%s: failed to read value", __func__);
+            }
+        } else {
+            break;
+        }
+    }
+    return true;
+}
+
+bool CBlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex, int trim_below_height)
 {
     std::unique_ptr<CDBIterator> pcursor(NewIterator());
 
     pcursor->Seek(std::make_pair(DB_BLOCK_INDEX, uint256()));
 
+    int n_untrimmed = 0;
+    int n_total = 0;
+
     // Load m_block_index
     while (pcursor->Valid()) {
         if (ShutdownRequested()) return false;
@@ -332,19 +367,27 @@ bool CBlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams,
                 pindexNew->nTime          = diskindex.nTime;
                 pindexNew->nBits          = diskindex.nBits;
                 pindexNew->nNonce         = diskindex.nNonce;
-                pindexNew->proof          = diskindex.proof;
                 pindexNew->nStatus        = diskindex.nStatus;
                 pindexNew->nTx            = diskindex.nTx;
-                pindexNew->m_dynafed_params       = diskindex.m_dynafed_params;
-                pindexNew->m_signblock_witness = diskindex.m_signblock_witness;
-
-                const uint256 block_hash = pindexNew->GetBlockHash();
-                // Only validate one of every 1000 block header for sanity check
-                if (pindexNew->nHeight % 1000 == 0 &&
-                        block_hash != consensusParams.hashGenesisBlock &&
-                        !CheckProof(pindexNew->GetBlockHeader(), consensusParams)) {
-                    return error("%s: CheckProof: %s, %s", __func__, block_hash.ToString(), pindexNew->ToString());
+
+                n_total++;
+                if (diskindex.nHeight >= trim_below_height) {
+                    n_untrimmed++;
+                    pindexNew->proof          = diskindex.proof;
+                    pindexNew->m_dynafed_params       = diskindex.m_dynafed_params;
+                    pindexNew->m_signblock_witness = diskindex.m_signblock_witness;
+
+                    const uint256 block_hash = pindexNew->GetBlockHash();
+                    // Only validate one of every 1000 block header for sanity check
+                    if (pindexNew->nHeight % 1000 == 0 &&
+                            block_hash != consensusParams.hashGenesisBlock &&
+                            !CheckProof(pindexNew->GetBlockHeader(), consensusParams)) {
+                        return error("%s: CheckProof: %s, %s", __func__, block_hash.ToString(), pindexNew->ToString());
+                    }
+                } else {
+                    pindexNew->m_trimmed = true;
                 }
+
                 pcursor->Next();
             } else {
                 return error("%s: failed to read value", __func__);
@@ -354,6 +397,7 @@ bool CBlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams,
         }
     }
 
+    LogPrintf("LoadBlockIndexGuts: loaded %d total / %d untrimmed (fully in-memory) headers\n", n_total, n_untrimmed);
     return true;
 }
 

src/txdb.h

@@ -85,8 +85,9 @@ class CBlockTreeDB : public CDBWrapper
     void ReadReindexing(bool &fReindexing);
     bool WriteFlag(const std::string &name, bool fValue);
     bool ReadFlag(const std::string &name, bool &fValue);
-    bool LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex);
+    bool LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex, int trim_below_height);
     // ELEMENTS:
+    bool WalkBlockIndexGutsForMaxHeight(int* nHeight);
     bool ReadPAKList(std::vector<std::vector<unsigned char> >& offline_list, std::vector<std::vector<unsigned char> >& online_list, bool& reject);
     bool WritePAKList(const std::vector<std::vector<unsigned char> >& offline_list, const std::vector<std::vector<unsigned char> >& online_list, bool reject);
 };

src/validation.cpp

@@ -4110,7 +4110,19 @@ bool BlockManager::LoadBlockIndex(
     CBlockTreeDB& blocktree,
     std::set<CBlockIndex*, CBlockIndexWorkComparator>& block_index_candidates)
 {
-    if (!blocktree.LoadBlockIndexGuts(consensus_params, [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); }))
+    int trim_below_height = 0;
+    if (fTrimHeaders) {
+        int max_height = 0;
+        if (!blocktree.WalkBlockIndexGutsForMaxHeight(&max_height)) {
+            LogPrintf("LoadBlockIndex: Somehow failed to WalkBlockIndexGutsForMaxHeight.\n");
+            return false;
+        }
+
+        int must_keep_headers = (consensus_params.total_valid_epochs + 2) * consensus_params.dynamic_epoch_length;
+        int extra_headers_buffer = consensus_params.dynamic_epoch_length * 2; // XXX arbitrary
+        trim_below_height = max_height - must_keep_headers - extra_headers_buffer;
+    }
+    if (!blocktree.LoadBlockIndexGuts(consensus_params, [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); }, trim_below_height))
         return false;
 
     // Calculate nChainWork