LCOV - code coverage report
Current view: top level - src/node - blockstorage.h (source / functions) Hit Total Coverage
Test: fuzz_coverage.info Lines: 18 20 90.0 %
Date: 2023-10-05 12:38:51 Functions: 10 11 90.9 %
Branches: 2 4 50.0 %

           Branch data     Line data    Source code
       1                 :            : // Copyright (c) 2011-2022 The Bitcoin Core developers
       2                 :            : // Distributed under the MIT software license, see the accompanying
       3                 :            : // file COPYING or http://www.opensource.org/licenses/mit-license.php.
       4                 :            : 
       5                 :            : #ifndef BITCOIN_NODE_BLOCKSTORAGE_H
       6                 :            : #define BITCOIN_NODE_BLOCKSTORAGE_H
       7                 :            : 
       8                 :            : #include <attributes.h>
       9                 :            : #include <chain.h>
      10                 :            : #include <dbwrapper.h>
      11                 :            : #include <kernel/blockmanager_opts.h>
      12                 :            : #include <kernel/chain.h>
      13                 :            : #include <kernel/chainparams.h>
      14                 :            : #include <kernel/cs_main.h>
      15                 :            : #include <kernel/messagestartchars.h>
      16                 :            : #include <sync.h>
      17                 :            : #include <util/fs.h>
      18                 :            : #include <util/hasher.h>
      19                 :            : 
      20                 :            : #include <atomic>
      21                 :            : #include <cstdint>
      22                 :            : #include <functional>
      23                 :            : #include <limits>
      24                 :            : #include <map>
      25                 :            : #include <memory>
      26                 :            : #include <set>
      27                 :            : #include <string>
      28                 :            : #include <unordered_map>
      29                 :            : #include <utility>
      30                 :            : #include <vector>
      31                 :            : 
      32                 :            : class BlockValidationState;
      33                 :            : class CAutoFile;
      34                 :            : class CBlock;
      35                 :            : class CBlockFileInfo;
      36                 :            : class CBlockUndo;
      37                 :            : class CChainParams;
      38                 :            : class Chainstate;
      39                 :            : class ChainstateManager;
      40                 :            : struct CCheckpointData;
      41                 :            : struct FlatFilePos;
      42                 :            : namespace Consensus {
      43                 :            : struct Params;
      44                 :            : }
      45                 :            : namespace util {
      46                 :            : class SignalInterrupt;
      47                 :            : } // namespace util
      48                 :            : 
      49                 :            : namespace kernel {
      50                 :            : /** Access to the block database (blocks/index/) */
      51                 :            : class BlockTreeDB : public CDBWrapper
      52                 :            : {
      53                 :            : public:
      54                 :            :     using CDBWrapper::CDBWrapper;
      55                 :            :     bool WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*>>& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo);
      56                 :            :     bool ReadBlockFileInfo(int nFile, CBlockFileInfo& info);
      57                 :            :     bool ReadLastBlockFile(int& nFile);
      58                 :            :     bool WriteReindexing(bool fReindexing);
      59                 :            :     void ReadReindexing(bool& fReindexing);
      60                 :            :     bool WriteFlag(const std::string& name, bool fValue);
      61                 :            :     bool ReadFlag(const std::string& name, bool& fValue);
      62                 :            :     bool LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex, const util::SignalInterrupt& interrupt)
      63                 :            :         EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
      64                 :            : };
      65                 :            : } // namespace kernel
      66                 :            : 
      67                 :            : namespace node {
      68                 :            : using kernel::BlockTreeDB;
      69                 :            : 
      70                 :            : /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */
      71                 :            : static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB
      72                 :            : /** The pre-allocation chunk size for rev?????.dat files (since 0.8) */
      73                 :            : static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB
      74                 :            : /** The maximum size of a blk?????.dat file (since 0.8) */
      75                 :            : static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB
      76                 :            : 
      77                 :            : /** Size of header written by WriteBlockToDisk before a serialized CBlock */
      78                 :            : static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE = std::tuple_size_v<MessageStartChars> + sizeof(unsigned int);
      79                 :            : 
      80                 :            : extern std::atomic_bool fReindex;
      81                 :            : 
      82                 :            : // Because validation code takes pointers to the map's CBlockIndex objects, if
      83                 :            : // we ever switch to another associative container, we need to either use a
      84                 :            : // container that has stable addressing (true of all std associative
      85                 :            : // containers), or make the key a `std::unique_ptr<CBlockIndex>`
      86                 :            : using BlockMap = std::unordered_map<uint256, CBlockIndex, BlockHasher>;
      87                 :            : 
      88                 :            : struct CBlockIndexWorkComparator {
      89                 :            :     bool operator()(const CBlockIndex* pa, const CBlockIndex* pb) const;
      90                 :            : };
      91                 :            : 
      92                 :            : struct CBlockIndexHeightOnlyComparator {
      93                 :            :     /* Only compares the height of two block indices, doesn't try to tie-break */
      94                 :            :     bool operator()(const CBlockIndex* pa, const CBlockIndex* pb) const;
      95                 :            : };
      96                 :            : 
      97                 :          0 : struct PruneLockInfo {
      98                 :          0 :     int height_first{std::numeric_limits<int>::max()}; //! Height of earliest block that should be kept and not pruned
      99                 :            : };
     100                 :            : 
     101                 :            : enum BlockfileType {
     102                 :            :     // Values used as array indexes - do not change carelessly.
     103                 :            :     NORMAL = 0,
     104                 :            :     ASSUMED = 1,
     105                 :            :     NUM_TYPES = 2,
     106                 :            : };
     107                 :            : 
     108                 :            : std::ostream& operator<<(std::ostream& os, const BlockfileType& type);
     109                 :            : 
     110                 :            : struct BlockfileCursor {
     111                 :            :     // The latest blockfile number.
     112                 :            :     int file_num{0};
     113                 :            : 
     114                 :            :     // Track the height of the highest block in file_num whose undo
     115                 :            :     // data has been written. Block data is written to block files in download
     116                 :            :     // order, but is written to undo files in validation order, which is
     117                 :            :     // usually in order by height. To avoid wasting disk space, undo files will
     118                 :            :     // be trimmed whenever the corresponding block file is finalized and
     119                 :            :     // the height of the highest block written to the block file equals the
     120                 :            :     // height of the highest block written to the undo file. This is a
     121                 :            :     // heuristic and can sometimes preemptively trim undo files that will write
     122                 :            :     // more data later, and sometimes fail to trim undo files that can't have
     123                 :            :     // more data written later.
     124                 :            :     int undo_height{0};
     125                 :            : };
     126                 :            : 
     127                 :            : std::ostream& operator<<(std::ostream& os, const BlockfileCursor& cursor);
     128                 :            : 
     129                 :            : 
     130                 :            : /**
     131                 :            :  * Maintains a tree of blocks (stored in `m_block_index`) which is consulted
     132                 :            :  * to determine where the most-work tip is.
     133                 :            :  *
     134                 :            :  * This data is used mostly in `Chainstate` - information about, e.g.,
     135                 :            :  * candidate tips is not maintained here.
     136                 :            :  */
     137                 :            : class BlockManager
     138                 :            : {
     139                 :            :     friend Chainstate;
     140                 :            :     friend ChainstateManager;
     141                 :            : 
     142                 :            : private:
     143                 :          1 :     const CChainParams& GetParams() const { return m_opts.chainparams; }
     144                 :          3 :     const Consensus::Params& GetConsensus() const { return m_opts.chainparams.GetConsensus(); }
     145                 :            :     /**
     146                 :            :      * Load the blocktree off disk and into memory. Populate certain metadata
     147                 :            :      * per index entry (nStatus, nChainWork, nTimeMax, etc.) as well as peripheral
     148                 :            :      * collections like m_dirty_blockindex.
     149                 :            :      */
     150                 :            :     bool LoadBlockIndex(const std::optional<uint256>& snapshot_blockhash)
     151                 :            :         EXCLUSIVE_LOCKS_REQUIRED(cs_main);
     152                 :            : 
     153                 :            :     /** Return false if block file or undo file flushing fails. */
     154                 :            :     [[nodiscard]] bool FlushBlockFile(int blockfile_num, bool fFinalize, bool finalize_undo);
     155                 :            : 
     156                 :            :     /** Return false if undo file flushing fails. */
     157                 :            :     [[nodiscard]] bool FlushUndoFile(int block_file, bool finalize = false);
     158                 :            : 
     159                 :            :     [[nodiscard]] bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown);
     160                 :            :     [[nodiscard]] bool FlushChainstateBlockFile(int tip_height);
     161                 :            :     bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize);
     162                 :            : 
     163                 :            :     FlatFileSeq BlockFileSeq() const;
     164                 :            :     FlatFileSeq UndoFileSeq() const;
     165                 :            : 
     166                 :            :     CAutoFile OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false) const;
     167                 :            : 
     168                 :            :     bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const;
     169                 :            :     bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock) const;
     170                 :            : 
     171                 :            :     /* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */
     172                 :            :     void FindFilesToPruneManual(
     173                 :            :         std::set<int>& setFilesToPrune,
     174                 :            :         int nManualPruneHeight,
     175                 :            :         const Chainstate& chain,
     176                 :            :         ChainstateManager& chainman);
     177                 :            : 
     178                 :            :     /**
     179                 :            :      * Prune block and undo files (blk???.dat and rev???.dat) so that the disk space used is less than a user-defined target.
     180                 :            :      * The user sets the target (in MB) on the command line or in config file.  This will be run on startup and whenever new
     181                 :            :      * space is allocated in a block or undo file, staying below the target. Changing back to unpruned requires a reindex
     182                 :            :      * (which in this case means the blockchain must be re-downloaded.)
     183                 :            :      *
     184                 :            :      * Pruning functions are called from FlushStateToDisk when the m_check_for_pruning flag has been set.
     185                 :            :      * Block and undo files are deleted in lock-step (when blk00003.dat is deleted, so is rev00003.dat.)
     186                 :            :      * Pruning cannot take place until the longest chain is at least a certain length (CChainParams::nPruneAfterHeight).
     187                 :            :      * Pruning will never delete a block within a defined distance (currently 288) from the active chain's tip.
     188                 :            :      * The block index is updated by unsetting HAVE_DATA and HAVE_UNDO for any blocks that were stored in the deleted files.
     189                 :            :      * A db flag records the fact that at least some block files have been pruned.
     190                 :            :      *
     191                 :            :      * @param[out]   setFilesToPrune   The set of file indices that can be unlinked will be returned
     192                 :            :      * @param        last_prune        The last height we're able to prune, according to the prune locks
     193                 :            :      */
     194                 :            :     void FindFilesToPrune(
     195                 :            :         std::set<int>& setFilesToPrune,
     196                 :            :         int last_prune,
     197                 :            :         const Chainstate& chain,
     198                 :            :         ChainstateManager& chainman);
     199                 :            : 
     200                 :            :     RecursiveMutex cs_LastBlockFile;
     201                 :            :     std::vector<CBlockFileInfo> m_blockfile_info;
     202                 :            : 
     203                 :            :     //! Since assumedvalid chainstates may be syncing a range of the chain that is very
     204                 :            :     //! far away from the normal/background validation process, we should segment blockfiles
     205                 :            :     //! for assumed chainstates. Otherwise, we might have wildly different height ranges
     206                 :            :     //! mixed into the same block files, which would impair our ability to prune
     207                 :            :     //! effectively.
     208                 :            :     //!
     209                 :            :     //! This data structure maintains separate blockfile number cursors for each
     210                 :            :     //! BlockfileType. The ASSUMED state is initialized, when necessary, in FindBlockPos().
     211                 :            :     //!
     212                 :            :     //! The first element is the NORMAL cursor, second is ASSUMED.
     213                 :            :     std::array<std::optional<BlockfileCursor>, BlockfileType::NUM_TYPES>
     214                 :          1 :         m_blockfile_cursors GUARDED_BY(cs_LastBlockFile) = {
     215                 :          1 :             BlockfileCursor{},
     216                 :            :             std::nullopt,
     217                 :            :     };
     218                 :          1 :     int MaxBlockfileNum() const EXCLUSIVE_LOCKS_REQUIRED(cs_LastBlockFile)
     219                 :            :     {
     220                 :            :         static const BlockfileCursor empty_cursor;
     221                 :          1 :         const auto& normal = m_blockfile_cursors[BlockfileType::NORMAL].value_or(empty_cursor);
     222                 :          1 :         const auto& assumed = m_blockfile_cursors[BlockfileType::ASSUMED].value_or(empty_cursor);
     223                 :          1 :         return std::max(normal.file_num, assumed.file_num);
     224                 :            :     }
     225                 :            : 
     226                 :            :     /** Global flag to indicate we should check to see if there are
     227                 :            :      *  block/undo files that should be deleted.  Set on startup
     228                 :            :      *  or if we allocate more file space when we're in prune mode
     229                 :            :      */
     230                 :          1 :     bool m_check_for_pruning = false;
     231                 :            : 
     232                 :            :     const bool m_prune_mode;
     233                 :            : 
     234                 :            :     /** Dirty block index entries. */
     235                 :            :     std::set<CBlockIndex*> m_dirty_blockindex;
     236                 :            : 
     237                 :            :     /** Dirty block file entries. */
     238                 :            :     std::set<int> m_dirty_fileinfo;
     239                 :            : 
     240                 :            :     /**
     241                 :            :      * Map from external index name to oldest block that must not be pruned.
     242                 :            :      *
     243                 :            :      * @note Internally, only blocks at height (height_first - PRUNE_LOCK_BUFFER - 1) and
     244                 :            :      * below will be pruned, but callers should avoid assuming any particular buffer size.
     245                 :            :      */
     246                 :            :     std::unordered_map<std::string, PruneLockInfo> m_prune_locks GUARDED_BY(::cs_main);
     247                 :            : 
     248                 :            :     BlockfileType BlockfileTypeForHeight(int height);
     249                 :            : 
     250                 :            :     const kernel::BlockManagerOpts m_opts;
     251                 :            : 
     252                 :            : public:
     253                 :            :     using Options = kernel::BlockManagerOpts;
     254                 :            : 
     255                 :          2 :     explicit BlockManager(const util::SignalInterrupt& interrupt, Options opts)
     256                 :          1 :         : m_prune_mode{opts.prune_target > 0},
     257         [ +  - ]:          1 :           m_opts{std::move(opts)},
     258                 :          2 :           m_interrupt{interrupt} {};
     259                 :            : 
     260                 :            :     const util::SignalInterrupt& m_interrupt;
     261                 :          1 :     std::atomic<bool> m_importing{false};
     262                 :            : 
     263                 :            :     BlockMap m_block_index GUARDED_BY(cs_main);
     264                 :            : 
     265                 :            :     /**
     266                 :            :      * The height of the base block of an assumeutxo snapshot, if one is in use.
     267                 :            :      *
     268                 :            :      * This controls how blockfiles are segmented by chainstate type to avoid
     269                 :            :      * comingling different height regions of the chain when an assumedvalid chainstate
     270                 :            :      * is in use. If heights are drastically different in the same blockfile, pruning
     271                 :            :      * suffers.
     272                 :            :      *
     273                 :            :      * This is set during ActivateSnapshot() or upon LoadBlockIndex() if a snapshot
     274                 :            :      * had been previously loaded. After the snapshot is validated, this is unset to
     275                 :            :      * restore normal LoadBlockIndex behavior.
     276                 :            :      */
     277                 :            :     std::optional<int> m_snapshot_height;
     278                 :            : 
     279                 :            :     std::vector<CBlockIndex*> GetAllBlockIndices() EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
     280                 :            : 
     281                 :            :     /**
     282                 :            :      * All pairs A->B, where A (or one of its ancestors) misses transactions, but B has transactions.
     283                 :            :      * Pruned nodes may have entries where B is missing data.
     284                 :            :      */
     285                 :            :     std::multimap<CBlockIndex*, CBlockIndex*> m_blocks_unlinked;
     286                 :            : 
     287                 :            :     std::unique_ptr<BlockTreeDB> m_block_tree_db GUARDED_BY(::cs_main);
     288                 :            : 
     289                 :            :     bool WriteBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
     290                 :            :     bool LoadBlockIndexDB(const std::optional<uint256>& snapshot_blockhash)
     291                 :            :         EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
     292                 :            : 
     293                 :            :     /**
     294                 :            :      * Remove any pruned block & undo files that are still on disk.
     295                 :            :      * This could happen on some systems if the file was still being read while unlinked,
     296                 :            :      * or if we crash before unlinking.
     297                 :            :      */
     298                 :            :     void ScanAndUnlinkAlreadyPrunedFiles() EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
     299                 :            : 
     300                 :            :     CBlockIndex* AddToBlockIndex(const CBlockHeader& block, CBlockIndex*& best_header) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
     301                 :            :     /** Create a new block index entry for a given block hash */
     302                 :            :     CBlockIndex* InsertBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
     303                 :            : 
     304                 :            :     //! Mark one block file as pruned (modify associated database entries)
     305                 :            :     void PruneOneBlockFile(const int fileNumber) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
     306                 :            : 
     307                 :            :     CBlockIndex* LookupBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
     308                 :            :     const CBlockIndex* LookupBlockIndex(const uint256& hash) const EXCLUSIVE_LOCKS_REQUIRED(cs_main);
     309                 :            : 
     310                 :            :     /** Get block file info entry for one block file */
     311                 :            :     CBlockFileInfo* GetBlockFileInfo(size_t n);
     312                 :            : 
     313                 :            :     bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block)
     314                 :            :         EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
     315                 :            : 
     316                 :            :     /** Store block on disk. If dbp is not nullptr, then it provides the known position of the block within a block file on disk. */
     317                 :            :     FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, const FlatFilePos* dbp);
     318                 :            : 
     319                 :            :     /** Whether running in -prune mode. */
     320                 :          5 :     [[nodiscard]] bool IsPruneMode() const { return m_prune_mode; }
     321                 :            : 
     322                 :            :     /** Attempt to stay below this number of bytes of block files. */
     323                 :          2 :     [[nodiscard]] uint64_t GetPruneTarget() const { return m_opts.prune_target; }
     324                 :            :     static constexpr auto PRUNE_TARGET_MANUAL{std::numeric_limits<uint64_t>::max()};
     325                 :            : 
     326         [ -  + ]:          4 :     [[nodiscard]] bool LoadingBlocks() const { return m_importing || fReindex; }
     327                 :            : 
     328                 :            :     /** Calculate the amount of disk space the block & undo files currently use */
     329                 :            :     uint64_t CalculateCurrentUsage();
     330                 :            : 
     331                 :            :     //! Returns last CBlockIndex* that is a checkpoint
     332                 :            :     const CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
     333                 :            : 
     334                 :            :     //! Check if all blocks in the [upper_block, lower_block] range have data available.
     335                 :            :     //! The caller is responsible for ensuring that lower_block is an ancestor of upper_block
     336                 :            :     //! (part of the same chain).
     337                 :            :     bool CheckBlockDataAvailability(const CBlockIndex& upper_block LIFETIMEBOUND, const CBlockIndex& lower_block LIFETIMEBOUND) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
     338                 :            : 
     339                 :            :     //! Find the first stored ancestor of start_block immediately after the last
     340                 :            :     //! pruned ancestor. Return value will never be null. Caller is responsible
     341                 :            :     //! for ensuring that start_block has data is not pruned.
     342                 :            :     const CBlockIndex* GetFirstStoredBlock(const CBlockIndex& start_block LIFETIMEBOUND, const CBlockIndex* lower_block=nullptr) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
     343                 :            : 
     344                 :            :     /** True if any block files have ever been pruned. */
     345                 :          1 :     bool m_have_pruned = false;
     346                 :            : 
     347                 :            :     //! Check whether the block associated with this index entry is pruned or not.
     348                 :            :     bool IsBlockPruned(const CBlockIndex* pblockindex) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
     349                 :            : 
     350                 :            :     //! Create or update a prune lock identified by its name
     351                 :            :     void UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
     352                 :            : 
     353                 :            :     /** Open a block file (blk?????.dat) */
     354                 :            :     CAutoFile OpenBlockFile(const FlatFilePos& pos, bool fReadOnly = false) const;
     355                 :            : 
     356                 :            :     /** Translation to a filesystem path */
     357                 :            :     fs::path GetBlockPosFilename(const FlatFilePos& pos) const;
     358                 :            : 
     359                 :            :     /**
     360                 :            :      *  Actually unlink the specified files
     361                 :            :      */
     362                 :            :     void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) const;
     363                 :            : 
     364                 :            :     /** Functions for disk access for blocks */
     365                 :            :     bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos) const;
     366                 :            :     bool ReadBlockFromDisk(CBlock& block, const CBlockIndex& index) const;
     367                 :            :     bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos) const;
     368                 :            : 
     369                 :            :     bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& index) const;
     370                 :            : 
     371                 :            :     void CleanupBlockRevFiles() const;
     372                 :            : };
     373                 :            : 
     374                 :            : void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFiles);
     375                 :            : } // namespace node
     376                 :            : 
     377                 :            : #endif // BITCOIN_NODE_BLOCKSTORAGE_H

Generated by: LCOV version 1.14