Branch data Line data Source code
1 : : // Copyright (c) 2011-2022 The Bitcoin Core developers
2 : : // Distributed under the MIT software license, see the accompanying
3 : : // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 : :
5 : : #ifndef BITCOIN_NODE_BLOCKSTORAGE_H
6 : : #define BITCOIN_NODE_BLOCKSTORAGE_H
7 : :
8 : : #include <attributes.h>
9 : : #include <chain.h>
10 : : #include <dbwrapper.h>
11 : : #include <kernel/blockmanager_opts.h>
12 : : #include <kernel/chain.h>
13 : : #include <kernel/chainparams.h>
14 : : #include <kernel/cs_main.h>
15 : : #include <kernel/messagestartchars.h>
16 : : #include <sync.h>
17 : : #include <util/fs.h>
18 : : #include <util/hasher.h>
19 : :
20 : : #include <atomic>
21 : : #include <cstdint>
22 : : #include <functional>
23 : : #include <limits>
24 : : #include <map>
25 : : #include <memory>
26 : : #include <set>
27 : : #include <string>
28 : : #include <unordered_map>
29 : : #include <utility>
30 : : #include <vector>
31 : :
32 : : class BlockValidationState;
33 : : class CAutoFile;
34 : : class CBlock;
35 : : class CBlockFileInfo;
36 : : class CBlockUndo;
37 : : class CChainParams;
38 : : class Chainstate;
39 : : class ChainstateManager;
40 : : struct CCheckpointData;
41 : : struct FlatFilePos;
42 : : namespace Consensus {
43 : : struct Params;
44 : : }
45 : : namespace util {
46 : : class SignalInterrupt;
47 : : } // namespace util
48 : :
49 : : namespace kernel {
50 : : /** Access to the block database (blocks/index/) */
51 : : class BlockTreeDB : public CDBWrapper
52 : : {
53 : : public:
54 : : using CDBWrapper::CDBWrapper;
55 : : bool WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*>>& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo);
56 : : bool ReadBlockFileInfo(int nFile, CBlockFileInfo& info);
57 : : bool ReadLastBlockFile(int& nFile);
58 : : bool WriteReindexing(bool fReindexing);
59 : : void ReadReindexing(bool& fReindexing);
60 : : bool WriteFlag(const std::string& name, bool fValue);
61 : : bool ReadFlag(const std::string& name, bool& fValue);
62 : : bool LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex, const util::SignalInterrupt& interrupt)
63 : : EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
64 : : };
65 : : } // namespace kernel
66 : :
67 : : namespace node {
68 : : using kernel::BlockTreeDB;
69 : :
70 : : /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */
71 : : static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB
72 : : /** The pre-allocation chunk size for rev?????.dat files (since 0.8) */
73 : : static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB
74 : : /** The maximum size of a blk?????.dat file (since 0.8) */
75 : : static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB
76 : :
77 : : /** Size of header written by WriteBlockToDisk before a serialized CBlock */
78 : : static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE = std::tuple_size_v<MessageStartChars> + sizeof(unsigned int);
79 : :
80 : : extern std::atomic_bool fReindex;
81 : :
82 : : // Because validation code takes pointers to the map's CBlockIndex objects, if
83 : : // we ever switch to another associative container, we need to either use a
84 : : // container that has stable addressing (true of all std associative
85 : : // containers), or make the key a `std::unique_ptr<CBlockIndex>`
86 : : using BlockMap = std::unordered_map<uint256, CBlockIndex, BlockHasher>;
87 : :
88 : : struct CBlockIndexWorkComparator {
89 : : bool operator()(const CBlockIndex* pa, const CBlockIndex* pb) const;
90 : : };
91 : :
92 : : struct CBlockIndexHeightOnlyComparator {
93 : : /* Only compares the height of two block indices, doesn't try to tie-break */
94 : : bool operator()(const CBlockIndex* pa, const CBlockIndex* pb) const;
95 : : };
96 : :
97 : 0 : struct PruneLockInfo {
98 : 0 : int height_first{std::numeric_limits<int>::max()}; //! Height of earliest block that should be kept and not pruned
99 : : };
100 : :
101 : : enum BlockfileType {
102 : : // Values used as array indexes - do not change carelessly.
103 : : NORMAL = 0,
104 : : ASSUMED = 1,
105 : : NUM_TYPES = 2,
106 : : };
107 : :
108 : : std::ostream& operator<<(std::ostream& os, const BlockfileType& type);
109 : :
110 : : struct BlockfileCursor {
111 : : // The latest blockfile number.
112 : : int file_num{0};
113 : :
114 : : // Track the height of the highest block in file_num whose undo
115 : : // data has been written. Block data is written to block files in download
116 : : // order, but is written to undo files in validation order, which is
117 : : // usually in order by height. To avoid wasting disk space, undo files will
118 : : // be trimmed whenever the corresponding block file is finalized and
119 : : // the height of the highest block written to the block file equals the
120 : : // height of the highest block written to the undo file. This is a
121 : : // heuristic and can sometimes preemptively trim undo files that will write
122 : : // more data later, and sometimes fail to trim undo files that can't have
123 : : // more data written later.
124 : : int undo_height{0};
125 : : };
126 : :
127 : : std::ostream& operator<<(std::ostream& os, const BlockfileCursor& cursor);
128 : :
129 : :
130 : : /**
131 : : * Maintains a tree of blocks (stored in `m_block_index`) which is consulted
132 : : * to determine where the most-work tip is.
133 : : *
134 : : * This data is used mostly in `Chainstate` - information about, e.g.,
135 : : * candidate tips is not maintained here.
136 : : */
137 : : class BlockManager
138 : : {
139 : : friend Chainstate;
140 : : friend ChainstateManager;
141 : :
142 : : private:
143 : 90296 : const CChainParams& GetParams() const { return m_opts.chainparams; }
144 : 74236 : const Consensus::Params& GetConsensus() const { return m_opts.chainparams.GetConsensus(); }
145 : : /**
146 : : * Load the blocktree off disk and into memory. Populate certain metadata
147 : : * per index entry (nStatus, nChainWork, nTimeMax, etc.) as well as peripheral
148 : : * collections like m_dirty_blockindex.
149 : : */
150 : : bool LoadBlockIndex(const std::optional<uint256>& snapshot_blockhash)
151 : : EXCLUSIVE_LOCKS_REQUIRED(cs_main);
152 : :
153 : : /** Return false if block file or undo file flushing fails. */
154 : : [[nodiscard]] bool FlushBlockFile(int blockfile_num, bool fFinalize, bool finalize_undo);
155 : :
156 : : /** Return false if undo file flushing fails. */
157 : : [[nodiscard]] bool FlushUndoFile(int block_file, bool finalize = false);
158 : :
159 : : [[nodiscard]] bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown);
160 : : [[nodiscard]] bool FlushChainstateBlockFile(int tip_height);
161 : : bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize);
162 : :
163 : : FlatFileSeq BlockFileSeq() const;
164 : : FlatFileSeq UndoFileSeq() const;
165 : :
166 : : CAutoFile OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false) const;
167 : :
168 : : bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const;
169 : : bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock) const;
170 : :
171 : : /* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */
172 : : void FindFilesToPruneManual(
173 : : std::set<int>& setFilesToPrune,
174 : : int nManualPruneHeight,
175 : : const Chainstate& chain,
176 : : ChainstateManager& chainman);
177 : :
178 : : /**
179 : : * Prune block and undo files (blk???.dat and rev???.dat) so that the disk space used is less than a user-defined target.
180 : : * The user sets the target (in MB) on the command line or in config file. This will be run on startup and whenever new
181 : : * space is allocated in a block or undo file, staying below the target. Changing back to unpruned requires a reindex
182 : : * (which in this case means the blockchain must be re-downloaded.)
183 : : *
184 : : * Pruning functions are called from FlushStateToDisk when the m_check_for_pruning flag has been set.
185 : : * Block and undo files are deleted in lock-step (when blk00003.dat is deleted, so is rev00003.dat.)
186 : : * Pruning cannot take place until the longest chain is at least a certain length (CChainParams::nPruneAfterHeight).
187 : : * Pruning will never delete a block within a defined distance (currently 288) from the active chain's tip.
188 : : * The block index is updated by unsetting HAVE_DATA and HAVE_UNDO for any blocks that were stored in the deleted files.
189 : : * A db flag records the fact that at least some block files have been pruned.
190 : : *
191 : : * @param[out] setFilesToPrune The set of file indices that can be unlinked will be returned
192 : : * @param last_prune The last height we're able to prune, according to the prune locks
193 : : */
194 : : void FindFilesToPrune(
195 : : std::set<int>& setFilesToPrune,
196 : : int last_prune,
197 : : const Chainstate& chain,
198 : : ChainstateManager& chainman);
199 : :
200 : : RecursiveMutex cs_LastBlockFile;
201 : : std::vector<CBlockFileInfo> m_blockfile_info;
202 : :
203 : : //! Since assumedvalid chainstates may be syncing a range of the chain that is very
204 : : //! far away from the normal/background validation process, we should segment blockfiles
205 : : //! for assumed chainstates. Otherwise, we might have wildly different height ranges
206 : : //! mixed into the same block files, which would impair our ability to prune
207 : : //! effectively.
208 : : //!
209 : : //! This data structure maintains separate blockfile number cursors for each
210 : : //! BlockfileType. The ASSUMED state is initialized, when necessary, in FindBlockPos().
211 : : //!
212 : : //! The first element is the NORMAL cursor, second is ASSUMED.
213 : : std::array<std::optional<BlockfileCursor>, BlockfileType::NUM_TYPES>
214 : 836 : m_blockfile_cursors GUARDED_BY(cs_LastBlockFile) = {
215 : 836 : BlockfileCursor{},
216 : : std::nullopt,
217 : : };
218 : 50872 : int MaxBlockfileNum() const EXCLUSIVE_LOCKS_REQUIRED(cs_LastBlockFile)
219 : : {
220 : : static const BlockfileCursor empty_cursor;
221 : 50872 : const auto& normal = m_blockfile_cursors[BlockfileType::NORMAL].value_or(empty_cursor);
222 : 50872 : const auto& assumed = m_blockfile_cursors[BlockfileType::ASSUMED].value_or(empty_cursor);
223 : 50872 : return std::max(normal.file_num, assumed.file_num);
224 : : }
225 : :
226 : : /** Global flag to indicate we should check to see if there are
227 : : * block/undo files that should be deleted. Set on startup
228 : : * or if we allocate more file space when we're in prune mode
229 : : */
230 : 836 : bool m_check_for_pruning = false;
231 : :
232 : : const bool m_prune_mode;
233 : :
234 : : /** Dirty block index entries. */
235 : : std::set<CBlockIndex*> m_dirty_blockindex;
236 : :
237 : : /** Dirty block file entries. */
238 : : std::set<int> m_dirty_fileinfo;
239 : :
240 : : /**
241 : : * Map from external index name to oldest block that must not be pruned.
242 : : *
243 : : * @note Internally, only blocks at height (height_first - PRUNE_LOCK_BUFFER - 1) and
244 : : * below will be pruned, but callers should avoid assuming any particular buffer size.
245 : : */
246 : : std::unordered_map<std::string, PruneLockInfo> m_prune_locks GUARDED_BY(::cs_main);
247 : :
248 : : BlockfileType BlockfileTypeForHeight(int height);
249 : :
250 : : const kernel::BlockManagerOpts m_opts;
251 : :
252 : : public:
253 : : using Options = kernel::BlockManagerOpts;
254 : :
255 : 1672 : explicit BlockManager(const util::SignalInterrupt& interrupt, Options opts)
256 : 836 : : m_prune_mode{opts.prune_target > 0},
257 [ + - ]: 836 : m_opts{std::move(opts)},
258 : 1672 : m_interrupt{interrupt} {};
259 : :
260 : : const util::SignalInterrupt& m_interrupt;
261 : 836 : std::atomic<bool> m_importing{false};
262 : :
263 : : BlockMap m_block_index GUARDED_BY(cs_main);
264 : :
265 : : /**
266 : : * The height of the base block of an assumeutxo snapshot, if one is in use.
267 : : *
268 : : * This controls how blockfiles are segmented by chainstate type to avoid
269 : : * comingling different height regions of the chain when an assumedvalid chainstate
270 : : * is in use. If heights are drastically different in the same blockfile, pruning
271 : : * suffers.
272 : : *
273 : : * This is set during ActivateSnapshot() or upon LoadBlockIndex() if a snapshot
274 : : * had been previously loaded. After the snapshot is validated, this is unset to
275 : : * restore normal LoadBlockIndex behavior.
276 : : */
277 : : std::optional<int> m_snapshot_height;
278 : :
279 : : std::vector<CBlockIndex*> GetAllBlockIndices() EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
280 : :
281 : : /**
282 : : * All pairs A->B, where A (or one of its ancestors) misses transactions, but B has transactions.
283 : : * Pruned nodes may have entries where B is missing data.
284 : : */
285 : : std::multimap<CBlockIndex*, CBlockIndex*> m_blocks_unlinked;
286 : :
287 : : std::unique_ptr<BlockTreeDB> m_block_tree_db GUARDED_BY(::cs_main);
288 : :
289 : : bool WriteBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
290 : : bool LoadBlockIndexDB(const std::optional<uint256>& snapshot_blockhash)
291 : : EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
292 : :
293 : : /**
294 : : * Remove any pruned block & undo files that are still on disk.
295 : : * This could happen on some systems if the file was still being read while unlinked,
296 : : * or if we crash before unlinking.
297 : : */
298 : : void ScanAndUnlinkAlreadyPrunedFiles() EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
299 : :
300 : : CBlockIndex* AddToBlockIndex(const CBlockHeader& block, CBlockIndex*& best_header) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
301 : : /** Create a new block index entry for a given block hash */
302 : : CBlockIndex* InsertBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
303 : :
304 : : //! Mark one block file as pruned (modify associated database entries)
305 : : void PruneOneBlockFile(const int fileNumber) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
306 : :
307 : : CBlockIndex* LookupBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
308 : : const CBlockIndex* LookupBlockIndex(const uint256& hash) const EXCLUSIVE_LOCKS_REQUIRED(cs_main);
309 : :
310 : : /** Get block file info entry for one block file */
311 : : CBlockFileInfo* GetBlockFileInfo(size_t n);
312 : :
313 : : bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block)
314 : : EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
315 : :
316 : : /** Store block on disk. If dbp is not nullptr, then it provides the known position of the block within a block file on disk. */
317 : : FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, const FlatFilePos* dbp);
318 : :
319 : : /** Whether running in -prune mode. */
320 : 4732061 : [[nodiscard]] bool IsPruneMode() const { return m_prune_mode; }
321 : :
322 : : /** Attempt to stay below this number of bytes of block files. */
323 : 1672 : [[nodiscard]] uint64_t GetPruneTarget() const { return m_opts.prune_target; }
324 : : static constexpr auto PRUNE_TARGET_MANUAL{std::numeric_limits<uint64_t>::max()};
325 : :
326 [ - + ]: 843226 : [[nodiscard]] bool LoadingBlocks() const { return m_importing || fReindex; }
327 : :
328 : : /** Calculate the amount of disk space the block & undo files currently use */
329 : : uint64_t CalculateCurrentUsage();
330 : :
331 : : //! Returns last CBlockIndex* that is a checkpoint
332 : : const CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
333 : :
334 : : //! Check if all blocks in the [upper_block, lower_block] range have data available.
335 : : //! The caller is responsible for ensuring that lower_block is an ancestor of upper_block
336 : : //! (part of the same chain).
337 : : bool CheckBlockDataAvailability(const CBlockIndex& upper_block LIFETIMEBOUND, const CBlockIndex& lower_block LIFETIMEBOUND) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
338 : :
339 : : //! Find the first stored ancestor of start_block immediately after the last
340 : : //! pruned ancestor. Return value will never be null. Caller is responsible
341 : : //! for ensuring that start_block has data is not pruned.
342 : : const CBlockIndex* GetFirstStoredBlock(const CBlockIndex& start_block LIFETIMEBOUND, const CBlockIndex* lower_block=nullptr) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
343 : :
344 : : /** True if any block files have ever been pruned. */
345 : 836 : bool m_have_pruned = false;
346 : :
347 : : //! Check whether the block associated with this index entry is pruned or not.
348 : : bool IsBlockPruned(const CBlockIndex* pblockindex) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
349 : :
350 : : //! Create or update a prune lock identified by its name
351 : : void UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
352 : :
353 : : /** Open a block file (blk?????.dat) */
354 : : CAutoFile OpenBlockFile(const FlatFilePos& pos, bool fReadOnly = false) const;
355 : :
356 : : /** Translation to a filesystem path */
357 : : fs::path GetBlockPosFilename(const FlatFilePos& pos) const;
358 : :
359 : : /**
360 : : * Actually unlink the specified files
361 : : */
362 : : void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) const;
363 : :
364 : : /** Functions for disk access for blocks */
365 : : bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos) const;
366 : : bool ReadBlockFromDisk(CBlock& block, const CBlockIndex& index) const;
367 : : bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos) const;
368 : :
369 : : bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& index) const;
370 : :
371 : : void CleanupBlockRevFiles() const;
372 : : };
373 : :
374 : : void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFiles);
375 : : } // namespace node
376 : :
377 : : #endif // BITCOIN_NODE_BLOCKSTORAGE_H
|
