Line data Source code
1 : // Copyright (c) 2011-2022 The Bitcoin Core developers
2 : // Distributed under the MIT software license, see the accompanying
3 : // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 :
5 : #ifndef BITCOIN_NODE_BLOCKSTORAGE_H
6 : #define BITCOIN_NODE_BLOCKSTORAGE_H
7 :
8 : #include <attributes.h>
9 : #include <chain.h>
10 : #include <dbwrapper.h>
11 : #include <kernel/blockmanager_opts.h>
12 : #include <kernel/chainparams.h>
13 : #include <kernel/cs_main.h>
14 : #include <kernel/messagestartchars.h>
15 : #include <sync.h>
16 : #include <util/fs.h>
17 : #include <util/hasher.h>
18 :
19 : #include <atomic>
20 : #include <cstdint>
21 : #include <functional>
22 : #include <limits>
23 : #include <map>
24 : #include <memory>
25 : #include <set>
26 : #include <string>
27 : #include <unordered_map>
28 : #include <utility>
29 : #include <vector>
30 :
31 : class BlockValidationState;
32 : class CBlock;
33 : class CBlockFileInfo;
34 : class CBlockUndo;
35 : class CChainParams;
36 : class Chainstate;
37 : class ChainstateManager;
38 : struct CCheckpointData;
39 : struct FlatFilePos;
40 : namespace Consensus {
41 : struct Params;
42 : }
43 : namespace util {
44 : class SignalInterrupt;
45 : } // namespace util
46 :
47 : namespace kernel {
48 : /** Access to the block database (blocks/index/) */
49 : class BlockTreeDB : public CDBWrapper
50 : {
51 : public:
52 : using CDBWrapper::CDBWrapper;
53 : bool WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*>>& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo);
54 : bool ReadBlockFileInfo(int nFile, CBlockFileInfo& info);
55 : bool ReadLastBlockFile(int& nFile);
56 : bool WriteReindexing(bool fReindexing);
57 : void ReadReindexing(bool& fReindexing);
58 : bool WriteFlag(const std::string& name, bool fValue);
59 : bool ReadFlag(const std::string& name, bool& fValue);
60 : bool LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex, const util::SignalInterrupt& interrupt)
61 : EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
62 : };
63 : } // namespace kernel
64 :
65 : namespace node {
66 : using kernel::BlockTreeDB;
67 :
68 : /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */
69 : static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB
70 : /** The pre-allocation chunk size for rev?????.dat files (since 0.8) */
71 : static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB
72 : /** The maximum size of a blk?????.dat file (since 0.8) */
73 : static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB
74 :
75 : /** Size of header written by WriteBlockToDisk before a serialized CBlock */
76 : static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE = std::tuple_size_v<MessageStartChars> + sizeof(unsigned int);
77 :
78 : extern std::atomic_bool fReindex;
79 :
80 : // Because validation code takes pointers to the map's CBlockIndex objects, if
81 : // we ever switch to another associative container, we need to either use a
82 : // container that has stable addressing (true of all std associative
83 : // containers), or make the key a `std::unique_ptr<CBlockIndex>`
84 : using BlockMap = std::unordered_map<uint256, CBlockIndex, BlockHasher>;
85 :
86 : struct CBlockIndexWorkComparator {
87 : bool operator()(const CBlockIndex* pa, const CBlockIndex* pb) const;
88 : };
89 :
90 : struct CBlockIndexHeightOnlyComparator {
91 : /* Only compares the height of two block indices, doesn't try to tie-break */
92 : bool operator()(const CBlockIndex* pa, const CBlockIndex* pb) const;
93 : };
94 :
95 0 : struct PruneLockInfo {
96 0 : int height_first{std::numeric_limits<int>::max()}; //! Height of earliest block that should be kept and not pruned
97 : };
98 :
99 : /**
100 : * Maintains a tree of blocks (stored in `m_block_index`) which is consulted
101 : * to determine where the most-work tip is.
102 : *
103 : * This data is used mostly in `Chainstate` - information about, e.g.,
104 : * candidate tips is not maintained here.
105 : */
106 : class BlockManager
107 : {
108 : friend Chainstate;
109 : friend ChainstateManager;
110 :
111 : private:
112 401 : const CChainParams& GetParams() const { return m_opts.chainparams; }
113 3 : const Consensus::Params& GetConsensus() const { return m_opts.chainparams.GetConsensus(); }
114 : /**
115 : * Load the blocktree off disk and into memory. Populate certain metadata
116 : * per index entry (nStatus, nChainWork, nTimeMax, etc.) as well as peripheral
117 : * collections like m_dirty_blockindex.
118 : */
119 : bool LoadBlockIndex()
120 : EXCLUSIVE_LOCKS_REQUIRED(cs_main);
121 : void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false);
122 : void FlushUndoFile(int block_file, bool finalize = false);
123 : bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown);
124 : bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize);
125 :
126 : FlatFileSeq BlockFileSeq() const;
127 : FlatFileSeq UndoFileSeq() const;
128 :
129 : FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false) const;
130 :
131 : bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const;
132 : bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock) const;
133 :
134 : /* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */
135 : void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight, int chain_tip_height);
136 :
137 : /**
138 : * Prune block and undo files (blk???.dat and rev???.dat) so that the disk space used is less than a user-defined target.
139 : * The user sets the target (in MB) on the command line or in config file. This will be run on startup and whenever new
140 : * space is allocated in a block or undo file, staying below the target. Changing back to unpruned requires a reindex
141 : * (which in this case means the blockchain must be re-downloaded.)
142 : *
143 : * Pruning functions are called from FlushStateToDisk when the m_check_for_pruning flag has been set.
144 : * Block and undo files are deleted in lock-step (when blk00003.dat is deleted, so is rev00003.dat.)
145 : * Pruning cannot take place until the longest chain is at least a certain length (CChainParams::nPruneAfterHeight).
146 : * Pruning will never delete a block within a defined distance (currently 288) from the active chain's tip.
147 : * The block index is updated by unsetting HAVE_DATA and HAVE_UNDO for any blocks that were stored in the deleted files.
148 : * A db flag records the fact that at least some block files have been pruned.
149 : *
150 : * @param[out] setFilesToPrune The set of file indices that can be unlinked will be returned
151 : */
152 : void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight, int chain_tip_height, int prune_height, bool is_ibd);
153 :
154 : RecursiveMutex cs_LastBlockFile;
155 : std::vector<CBlockFileInfo> m_blockfile_info;
156 1 : int m_last_blockfile = 0;
157 :
158 : // Track the height of the highest block in m_last_blockfile whose undo
159 : // data has been written. Block data is written to block files in download
160 : // order, but is written to undo files in validation order, which is
161 : // usually in order by height. To avoid wasting disk space, undo files will
162 : // be trimmed whenever the corresponding block file is finalized and
163 : // the height of the highest block written to the block file equals the
164 : // height of the highest block written to the undo file. This is a
165 : // heuristic and can sometimes preemptively trim undo files that will write
166 : // more data later, and sometimes fail to trim undo files that can't have
167 : // more data written later.
168 1 : unsigned int m_undo_height_in_last_blockfile = 0;
169 :
170 : /** Global flag to indicate we should check to see if there are
171 : * block/undo files that should be deleted. Set on startup
172 : * or if we allocate more file space when we're in prune mode
173 : */
174 1 : bool m_check_for_pruning = false;
175 :
176 : const bool m_prune_mode;
177 :
178 : /** Dirty block index entries. */
179 : std::set<CBlockIndex*> m_dirty_blockindex;
180 :
181 : /** Dirty block file entries. */
182 : std::set<int> m_dirty_fileinfo;
183 :
184 : /**
185 : * Map from external index name to oldest block that must not be pruned.
186 : *
187 : * @note Internally, only blocks at height (height_first - PRUNE_LOCK_BUFFER - 1) and
188 : * below will be pruned, but callers should avoid assuming any particular buffer size.
189 : */
190 : std::unordered_map<std::string, PruneLockInfo> m_prune_locks GUARDED_BY(::cs_main);
191 :
192 : const kernel::BlockManagerOpts m_opts;
193 :
194 : public:
195 : using Options = kernel::BlockManagerOpts;
196 :
197 2 : explicit BlockManager(const util::SignalInterrupt& interrupt, Options opts)
198 1 : : m_prune_mode{opts.prune_target > 0},
199 1 : m_opts{std::move(opts)},
200 2 : m_interrupt{interrupt} {};
201 :
202 : const util::SignalInterrupt& m_interrupt;
203 1 : std::atomic<bool> m_importing{false};
204 :
205 : BlockMap m_block_index GUARDED_BY(cs_main);
206 :
207 : std::vector<CBlockIndex*> GetAllBlockIndices() EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
208 :
209 : /**
210 : * All pairs A->B, where A (or one of its ancestors) misses transactions, but B has transactions.
211 : * Pruned nodes may have entries where B is missing data.
212 : */
213 : std::multimap<CBlockIndex*, CBlockIndex*> m_blocks_unlinked;
214 :
215 : std::unique_ptr<BlockTreeDB> m_block_tree_db GUARDED_BY(::cs_main);
216 :
217 : bool WriteBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
218 : bool LoadBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
219 :
220 : /**
221 : * Remove any pruned block & undo files that are still on disk.
222 : * This could happen on some systems if the file was still being read while unlinked,
223 : * or if we crash before unlinking.
224 : */
225 : void ScanAndUnlinkAlreadyPrunedFiles() EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
226 :
227 : CBlockIndex* AddToBlockIndex(const CBlockHeader& block, CBlockIndex*& best_header) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
228 : /** Create a new block index entry for a given block hash */
229 : CBlockIndex* InsertBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
230 :
231 : //! Mark one block file as pruned (modify associated database entries)
232 : void PruneOneBlockFile(const int fileNumber) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
233 :
234 : CBlockIndex* LookupBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
235 : const CBlockIndex* LookupBlockIndex(const uint256& hash) const EXCLUSIVE_LOCKS_REQUIRED(cs_main);
236 :
237 : /** Get block file info entry for one block file */
238 : CBlockFileInfo* GetBlockFileInfo(size_t n);
239 :
240 : bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block)
241 : EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
242 :
243 : /** Store block on disk. If dbp is not nullptr, then it provides the known position of the block within a block file on disk. */
244 : FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, const FlatFilePos* dbp);
245 :
246 : /** Whether running in -prune mode. */
247 15950 : [[nodiscard]] bool IsPruneMode() const { return m_prune_mode; }
248 :
249 : /** Attempt to stay below this number of bytes of block files. */
250 2 : [[nodiscard]] uint64_t GetPruneTarget() const { return m_opts.prune_target; }
251 : static constexpr auto PRUNE_TARGET_MANUAL{std::numeric_limits<uint64_t>::max()};
252 :
253 1205 : [[nodiscard]] bool LoadingBlocks() const { return m_importing || fReindex; }
254 :
255 : /** Calculate the amount of disk space the block & undo files currently use */
256 : uint64_t CalculateCurrentUsage();
257 :
258 : //! Returns last CBlockIndex* that is a checkpoint
259 : const CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
260 :
261 : //! Check if all blocks in the [upper_block, lower_block] range have data available.
262 : //! The caller is responsible for ensuring that lower_block is an ancestor of upper_block
263 : //! (part of the same chain).
264 : bool CheckBlockDataAvailability(const CBlockIndex& upper_block LIFETIMEBOUND, const CBlockIndex& lower_block LIFETIMEBOUND) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
265 :
266 : //! Find the first stored ancestor of start_block immediately after the last
267 : //! pruned ancestor. Return value will never be null. Caller is responsible
268 : //! for ensuring that start_block has data is not pruned.
269 : const CBlockIndex* GetFirstStoredBlock(const CBlockIndex& start_block LIFETIMEBOUND, const CBlockIndex* lower_block=nullptr) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
270 :
271 : /** True if any block files have ever been pruned. */
272 1 : bool m_have_pruned = false;
273 :
274 : //! Check whether the block associated with this index entry is pruned or not.
275 : bool IsBlockPruned(const CBlockIndex* pblockindex) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
276 :
277 : //! Create or update a prune lock identified by its name
278 : void UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
279 :
280 : /** Open a block file (blk?????.dat) */
281 : FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly = false) const;
282 :
283 : /** Translation to a filesystem path */
284 : fs::path GetBlockPosFilename(const FlatFilePos& pos) const;
285 :
286 : /**
287 : * Actually unlink the specified files
288 : */
289 : void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) const;
290 :
291 : /** Functions for disk access for blocks */
292 : bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos) const;
293 : bool ReadBlockFromDisk(CBlock& block, const CBlockIndex& index) const;
294 : bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos) const;
295 :
296 : bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& index) const;
297 :
298 : void CleanupBlockRevFiles() const;
299 : };
300 :
301 : void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFiles);
302 : } // namespace node
303 :
304 : #endif // BITCOIN_NODE_BLOCKSTORAGE_H
|
