Branch data Line data Source code
1 : : // Copyright (c) 2011-2022 The Bitcoin Core developers
2 : : // Distributed under the MIT software license, see the accompanying
3 : : // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 : :
5 : : #include <node/blockstorage.h>
6 : :
7 : : #include <arith_uint256.h>
8 : : #include <chain.h>
9 : : #include <consensus/params.h>
10 : : #include <consensus/validation.h>
11 : : #include <dbwrapper.h>
12 : : #include <flatfile.h>
13 : : #include <hash.h>
14 : : #include <kernel/blockmanager_opts.h>
15 : : #include <kernel/chainparams.h>
16 : : #include <kernel/messagestartchars.h>
17 : : #include <kernel/notifications_interface.h>
18 : : #include <logging.h>
19 : : #include <pow.h>
20 : : #include <primitives/block.h>
21 : : #include <primitives/transaction.h>
22 : : #include <reverse_iterator.h>
23 : : #include <serialize.h>
24 : : #include <signet.h>
25 : : #include <span.h>
26 : : #include <streams.h>
27 : : #include <sync.h>
28 : : #include <tinyformat.h>
29 : : #include <uint256.h>
30 : : #include <undo.h>
31 : : #include <util/batchpriority.h>
32 : : #include <util/check.h>
33 : : #include <util/fs.h>
34 : : #include <util/signalinterrupt.h>
35 : : #include <util/strencodings.h>
36 : : #include <util/translation.h>
37 : : #include <validation.h>
38 : :
39 : : #include <map>
40 : : #include <unordered_map>
41 : :
42 : : namespace kernel {
43 : : static constexpr uint8_t DB_BLOCK_FILES{'f'};
44 : : static constexpr uint8_t DB_BLOCK_INDEX{'b'};
45 : : static constexpr uint8_t DB_FLAG{'F'};
46 : : static constexpr uint8_t DB_REINDEX_FLAG{'R'};
47 : : static constexpr uint8_t DB_LAST_BLOCK{'l'};
48 : : // Keys used in previous version that might still be found in the DB:
49 : : // BlockTreeDB::DB_TXINDEX_BLOCK{'T'};
50 : : // BlockTreeDB::DB_TXINDEX{'t'}
51 : : // BlockTreeDB::ReadFlag("txindex")
52 : :
53 : 0 : bool BlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo& info)
54 : : {
55 : 0 : return Read(std::make_pair(DB_BLOCK_FILES, nFile), info);
56 : : }
57 : :
58 : 0 : bool BlockTreeDB::WriteReindexing(bool fReindexing)
59 : : {
60 [ # # ]: 0 : if (fReindexing) {
61 : 0 : return Write(DB_REINDEX_FLAG, uint8_t{'1'});
62 : : } else {
63 : 0 : return Erase(DB_REINDEX_FLAG);
64 : : }
65 : 0 : }
66 : :
67 : 0 : void BlockTreeDB::ReadReindexing(bool& fReindexing)
68 : : {
69 : 0 : fReindexing = Exists(DB_REINDEX_FLAG);
70 : 0 : }
71 : :
72 : 0 : bool BlockTreeDB::ReadLastBlockFile(int& nFile)
73 : : {
74 : 0 : return Read(DB_LAST_BLOCK, nFile);
75 : : }
76 : :
77 : 0 : bool BlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*>>& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo)
78 : : {
79 : 0 : CDBBatch batch(*this);
80 [ # # ]: 0 : for (const auto& [file, info] : fileInfo) {
81 [ # # ][ # # ]: 0 : batch.Write(std::make_pair(DB_BLOCK_FILES, file), *info);
[ # # ]
82 : : }
83 [ # # ]: 0 : batch.Write(DB_LAST_BLOCK, nLastFile);
84 [ # # ]: 0 : for (const CBlockIndex* bi : blockinfo) {
85 [ # # ][ # # ]: 0 : batch.Write(std::make_pair(DB_BLOCK_INDEX, bi->GetBlockHash()), CDiskBlockIndex{bi});
[ # # ][ # # ]
86 : : }
87 [ # # ]: 0 : return WriteBatch(batch, true);
88 : 0 : }
89 : :
90 : 0 : bool BlockTreeDB::WriteFlag(const std::string& name, bool fValue)
91 : : {
92 [ # # ]: 0 : return Write(std::make_pair(DB_FLAG, name), fValue ? uint8_t{'1'} : uint8_t{'0'});
93 : 0 : }
94 : :
95 : 0 : bool BlockTreeDB::ReadFlag(const std::string& name, bool& fValue)
96 : : {
97 : : uint8_t ch;
98 [ # # ][ # # ]: 0 : if (!Read(std::make_pair(DB_FLAG, name), ch)) {
99 : 0 : return false;
100 : : }
101 : 0 : fValue = ch == uint8_t{'1'};
102 : 0 : return true;
103 : 0 : }
104 : :
105 : 0 : bool BlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex, const util::SignalInterrupt& interrupt)
106 : : {
107 : 0 : AssertLockHeld(::cs_main);
108 : 0 : std::unique_ptr<CDBIterator> pcursor(NewIterator());
109 [ # # ][ # # ]: 0 : pcursor->Seek(std::make_pair(DB_BLOCK_INDEX, uint256()));
[ # # ]
110 : :
111 : : // Load m_block_index
112 [ # # ][ # # ]: 0 : while (pcursor->Valid()) {
113 [ # # ][ # # ]: 0 : if (interrupt) return false;
114 [ # # ]: 0 : std::pair<uint8_t, uint256> key;
115 [ # # ][ # # ]: 0 : if (pcursor->GetKey(key) && key.first == DB_BLOCK_INDEX) {
[ # # ]
116 [ # # ]: 0 : CDiskBlockIndex diskindex;
117 [ # # ][ # # ]: 0 : if (pcursor->GetValue(diskindex)) {
118 : : // Construct block index object
119 [ # # ][ # # ]: 0 : CBlockIndex* pindexNew = insertBlockIndex(diskindex.ConstructBlockHash());
120 [ # # ]: 0 : pindexNew->pprev = insertBlockIndex(diskindex.hashPrev);
121 : 0 : pindexNew->nHeight = diskindex.nHeight;
122 : 0 : pindexNew->nFile = diskindex.nFile;
123 : 0 : pindexNew->nDataPos = diskindex.nDataPos;
124 : 0 : pindexNew->nUndoPos = diskindex.nUndoPos;
125 : 0 : pindexNew->nVersion = diskindex.nVersion;
126 : 0 : pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot;
127 : 0 : pindexNew->nTime = diskindex.nTime;
128 : 0 : pindexNew->nBits = diskindex.nBits;
129 : 0 : pindexNew->nNonce = diskindex.nNonce;
130 : 0 : pindexNew->nStatus = diskindex.nStatus;
131 : 0 : pindexNew->nTx = diskindex.nTx;
132 : :
133 [ # # ][ # # ]: 0 : if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, consensusParams)) {
[ # # ]
134 [ # # ][ # # ]: 0 : return error("%s: CheckProofOfWork failed: %s", __func__, pindexNew->ToString());
135 : : }
136 : :
137 [ # # ]: 0 : pcursor->Next();
138 : 0 : } else {
139 [ # # ]: 0 : return error("%s: failed to read value", __func__);
140 : : }
141 : 0 : } else {
142 : 0 : break;
143 : : }
144 : : }
145 : :
146 : 0 : return true;
147 : 0 : }
148 : : } // namespace kernel
149 : :
150 : : namespace node {
151 : : std::atomic_bool fReindex(false);
152 : :
153 : 0 : bool CBlockIndexWorkComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const
154 : : {
155 : : // First sort by most total work, ...
156 [ # # ]: 0 : if (pa->nChainWork > pb->nChainWork) return false;
157 [ # # ]: 0 : if (pa->nChainWork < pb->nChainWork) return true;
158 : :
159 : : // ... then by earliest time received, ...
160 [ # # ]: 0 : if (pa->nSequenceId < pb->nSequenceId) return false;
161 [ # # ]: 0 : if (pa->nSequenceId > pb->nSequenceId) return true;
162 : :
163 : : // Use pointer address as tie breaker (should only happen with blocks
164 : : // loaded from disk, as those all have id 0).
165 [ # # ]: 0 : if (pa < pb) return false;
166 [ # # ]: 0 : if (pa > pb) return true;
167 : :
168 : : // Identical blocks.
169 : 0 : return false;
170 : 0 : }
171 : :
172 : 0 : bool CBlockIndexHeightOnlyComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const
173 : : {
174 : 0 : return pa->nHeight < pb->nHeight;
175 : : }
176 : :
177 : 0 : std::vector<CBlockIndex*> BlockManager::GetAllBlockIndices()
178 : : {
179 : 0 : AssertLockHeld(cs_main);
180 : 0 : std::vector<CBlockIndex*> rv;
181 [ # # ]: 0 : rv.reserve(m_block_index.size());
182 [ # # ]: 0 : for (auto& [_, block_index] : m_block_index) {
183 [ # # ]: 0 : rv.push_back(&block_index);
184 : : }
185 : 0 : return rv;
186 [ # # ]: 0 : }
187 : :
188 : 0 : CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash)
189 : : {
190 : 0 : AssertLockHeld(cs_main);
191 : 0 : BlockMap::iterator it = m_block_index.find(hash);
192 [ # # ]: 0 : return it == m_block_index.end() ? nullptr : &it->second;
193 : : }
194 : :
195 : 0 : const CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) const
196 : : {
197 : 0 : AssertLockHeld(cs_main);
198 : 0 : BlockMap::const_iterator it = m_block_index.find(hash);
199 [ # # ]: 0 : return it == m_block_index.end() ? nullptr : &it->second;
200 : : }
201 : :
202 : 0 : CBlockIndex* BlockManager::AddToBlockIndex(const CBlockHeader& block, CBlockIndex*& best_header)
203 : : {
204 : 0 : AssertLockHeld(cs_main);
205 : :
206 : 0 : auto [mi, inserted] = m_block_index.try_emplace(block.GetHash(), block);
207 [ # # ]: 0 : if (!inserted) {
208 : 0 : return &mi->second;
209 : : }
210 : 0 : CBlockIndex* pindexNew = &(*mi).second;
211 : :
212 : : // We assign the sequence id to blocks only when the full data is available,
213 : : // to avoid miners withholding blocks but broadcasting headers, to get a
214 : : // competitive advantage.
215 : 0 : pindexNew->nSequenceId = 0;
216 : :
217 : 0 : pindexNew->phashBlock = &((*mi).first);
218 : 0 : BlockMap::iterator miPrev = m_block_index.find(block.hashPrevBlock);
219 [ # # ]: 0 : if (miPrev != m_block_index.end()) {
220 : 0 : pindexNew->pprev = &(*miPrev).second;
221 : 0 : pindexNew->nHeight = pindexNew->pprev->nHeight + 1;
222 : 0 : pindexNew->BuildSkip();
223 : 0 : }
224 [ # # ]: 0 : pindexNew->nTimeMax = (pindexNew->pprev ? std::max(pindexNew->pprev->nTimeMax, pindexNew->nTime) : pindexNew->nTime);
225 [ # # ]: 0 : pindexNew->nChainWork = (pindexNew->pprev ? pindexNew->pprev->nChainWork : 0) + GetBlockProof(*pindexNew);
226 : 0 : pindexNew->RaiseValidity(BLOCK_VALID_TREE);
227 [ # # ][ # # ]: 0 : if (best_header == nullptr || best_header->nChainWork < pindexNew->nChainWork) {
228 : 0 : best_header = pindexNew;
229 : 0 : }
230 : :
231 : 0 : m_dirty_blockindex.insert(pindexNew);
232 : :
233 : 0 : return pindexNew;
234 : 0 : }
235 : :
236 : 0 : void BlockManager::PruneOneBlockFile(const int fileNumber)
237 : : {
238 : 0 : AssertLockHeld(cs_main);
239 : 0 : LOCK(cs_LastBlockFile);
240 : :
241 [ # # ]: 0 : for (auto& entry : m_block_index) {
242 : 0 : CBlockIndex* pindex = &entry.second;
243 [ # # ]: 0 : if (pindex->nFile == fileNumber) {
244 : 0 : pindex->nStatus &= ~BLOCK_HAVE_DATA;
245 : 0 : pindex->nStatus &= ~BLOCK_HAVE_UNDO;
246 : 0 : pindex->nFile = 0;
247 : 0 : pindex->nDataPos = 0;
248 : 0 : pindex->nUndoPos = 0;
249 [ # # ]: 0 : m_dirty_blockindex.insert(pindex);
250 : :
251 : : // Prune from m_blocks_unlinked -- any block we prune would have
252 : : // to be downloaded again in order to consider its chain, at which
253 : : // point it would be considered as a candidate for
254 : : // m_blocks_unlinked or setBlockIndexCandidates.
255 [ # # ]: 0 : auto range = m_blocks_unlinked.equal_range(pindex->pprev);
256 [ # # ]: 0 : while (range.first != range.second) {
257 : 0 : std::multimap<CBlockIndex*, CBlockIndex*>::iterator _it = range.first;
258 : 0 : range.first++;
259 [ # # ]: 0 : if (_it->second == pindex) {
260 [ # # ]: 0 : m_blocks_unlinked.erase(_it);
261 : 0 : }
262 : : }
263 : 0 : }
264 : : }
265 : :
266 [ # # ][ # # ]: 0 : m_blockfile_info.at(fileNumber) = CBlockFileInfo{};
267 [ # # ]: 0 : m_dirty_fileinfo.insert(fileNumber);
268 : 0 : }
269 : :
270 : 0 : void BlockManager::FindFilesToPruneManual(
271 : : std::set<int>& setFilesToPrune,
272 : : int nManualPruneHeight,
273 : : const Chainstate& chain,
274 : : ChainstateManager& chainman)
275 : : {
276 [ # # ][ # # ]: 0 : assert(IsPruneMode() && nManualPruneHeight > 0);
277 : :
278 [ # # ][ # # ]: 0 : LOCK2(cs_main, cs_LastBlockFile);
279 [ # # ][ # # ]: 0 : if (chain.m_chain.Height() < 0) {
280 : 0 : return;
281 : : }
282 : :
283 [ # # ]: 0 : const auto [min_block_to_prune, last_block_can_prune] = chainman.GetPruneRange(chain, nManualPruneHeight);
284 : :
285 : 0 : int count = 0;
286 [ # # ][ # # ]: 0 : for (int fileNumber = 0; fileNumber < this->MaxBlockfileNum(); fileNumber++) {
287 : 0 : const auto& fileinfo = m_blockfile_info[fileNumber];
288 [ # # ][ # # ]: 0 : if (fileinfo.nSize == 0 || fileinfo.nHeightLast > (unsigned)last_block_can_prune || fileinfo.nHeightFirst < (unsigned)min_block_to_prune) {
[ # # ][ # # ]
[ # # ]
289 : 0 : continue;
290 : : }
291 : :
292 [ # # ]: 0 : PruneOneBlockFile(fileNumber);
293 [ # # ]: 0 : setFilesToPrune.insert(fileNumber);
294 : 0 : count++;
295 : 0 : }
296 [ # # ][ # # ]: 0 : LogPrintf("[%s] Prune (Manual): prune_height=%d removed %d blk/rev pairs\n",
[ # # ][ # # ]
[ # # ]
297 : : chain.GetRole(), last_block_can_prune, count);
298 [ # # ]: 0 : }
299 : :
300 : 0 : void BlockManager::FindFilesToPrune(
301 : : std::set<int>& setFilesToPrune,
302 : : int last_prune,
303 : : const Chainstate& chain,
304 : : ChainstateManager& chainman)
305 : : {
306 [ # # ][ # # ]: 0 : LOCK2(cs_main, cs_LastBlockFile);
307 : : // Distribute our -prune budget over all chainstates.
308 [ # # ]: 0 : const auto target = std::max(
309 [ # # ][ # # ]: 0 : MIN_DISK_SPACE_FOR_BLOCK_FILES, GetPruneTarget() / chainman.GetAll().size());
310 : :
311 [ # # ][ # # ]: 0 : if (chain.m_chain.Height() < 0 || target == 0) {
[ # # ]
312 : 0 : return;
313 : : }
314 [ # # ][ # # ]: 0 : if (static_cast<uint64_t>(chain.m_chain.Height()) <= chainman.GetParams().PruneAfterHeight()) {
[ # # ][ # # ]
315 : 0 : return;
316 : : }
317 : :
318 [ # # ]: 0 : const auto [min_block_to_prune, last_block_can_prune] = chainman.GetPruneRange(chain, last_prune);
319 : :
320 [ # # ]: 0 : uint64_t nCurrentUsage = CalculateCurrentUsage();
321 : : // We don't check to prune until after we've allocated new space for files
322 : : // So we should leave a buffer under our target to account for another allocation
323 : : // before the next pruning.
324 : 0 : uint64_t nBuffer = BLOCKFILE_CHUNK_SIZE + UNDOFILE_CHUNK_SIZE;
325 : : uint64_t nBytesToPrune;
326 : 0 : int count = 0;
327 : :
328 [ # # ]: 0 : if (nCurrentUsage + nBuffer >= target) {
329 : : // On a prune event, the chainstate DB is flushed.
330 : : // To avoid excessive prune events negating the benefit of high dbcache
331 : : // values, we should not prune too rapidly.
332 : : // So when pruning in IBD, increase the buffer a bit to avoid a re-prune too soon.
333 [ # # ][ # # ]: 0 : if (chainman.IsInitialBlockDownload()) {
334 : : // Since this is only relevant during IBD, we use a fixed 10%
335 : 0 : nBuffer += target / 10;
336 : 0 : }
337 : :
338 [ # # ][ # # ]: 0 : for (int fileNumber = 0; fileNumber < this->MaxBlockfileNum(); fileNumber++) {
339 : 0 : const auto& fileinfo = m_blockfile_info[fileNumber];
340 : 0 : nBytesToPrune = fileinfo.nSize + fileinfo.nUndoSize;
341 : :
342 [ # # ]: 0 : if (fileinfo.nSize == 0) {
343 : 0 : continue;
344 : : }
345 : :
346 [ # # ]: 0 : if (nCurrentUsage + nBuffer < target) { // are we below our target?
347 : 0 : break;
348 : : }
349 : :
350 : : // don't prune files that could have a block that's not within the allowable
351 : : // prune range for the chain being pruned.
352 [ # # ][ # # ]: 0 : if (fileinfo.nHeightLast > (unsigned)last_block_can_prune || fileinfo.nHeightFirst < (unsigned)min_block_to_prune) {
[ # # ][ # # ]
353 : 0 : continue;
354 : : }
355 : :
356 [ # # ]: 0 : PruneOneBlockFile(fileNumber);
357 : : // Queue up the files for removal
358 [ # # ]: 0 : setFilesToPrune.insert(fileNumber);
359 : 0 : nCurrentUsage -= nBytesToPrune;
360 : 0 : count++;
361 : 0 : }
362 : 0 : }
363 : :
364 [ # # ][ # # ]: 0 : LogPrint(BCLog::PRUNE, "[%s] target=%dMiB actual=%dMiB diff=%dMiB min_height=%d max_prune_height=%d removed %d blk/rev pairs\n",
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ]
365 : : chain.GetRole(), target / 1024 / 1024, nCurrentUsage / 1024 / 1024,
366 : : (int64_t(target) - int64_t(nCurrentUsage)) / 1024 / 1024,
367 : : min_block_to_prune, last_block_can_prune, count);
368 [ # # ]: 0 : }
369 : :
370 : 0 : void BlockManager::UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) {
371 : 0 : AssertLockHeld(::cs_main);
372 : 0 : m_prune_locks[name] = lock_info;
373 : 0 : }
374 : :
375 : 0 : CBlockIndex* BlockManager::InsertBlockIndex(const uint256& hash)
376 : : {
377 : 0 : AssertLockHeld(cs_main);
378 : :
379 [ # # ]: 0 : if (hash.IsNull()) {
380 : 0 : return nullptr;
381 : : }
382 : :
383 : 0 : const auto [mi, inserted]{m_block_index.try_emplace(hash)};
384 : 0 : CBlockIndex* pindex = &(*mi).second;
385 [ # # ]: 0 : if (inserted) {
386 : 0 : pindex->phashBlock = &((*mi).first);
387 : 0 : }
388 : 0 : return pindex;
389 : 0 : }
390 : :
391 : 0 : bool BlockManager::LoadBlockIndex(const std::optional<uint256>& snapshot_blockhash)
392 : : {
393 [ # # ][ # # ]: 0 : if (!m_block_tree_db->LoadBlockIndexGuts(
[ # # ]
394 : 0 : GetConsensus(), [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); }, m_interrupt)) {
395 : 0 : return false;
396 : : }
397 : :
398 [ # # ]: 0 : if (snapshot_blockhash) {
399 : 0 : const std::optional<AssumeutxoData> maybe_au_data = GetParams().AssumeutxoForBlockhash(*snapshot_blockhash);
400 [ # # ]: 0 : if (!maybe_au_data) {
401 [ # # ][ # # ]: 0 : m_opts.notifications.fatalError(strprintf("Assumeutxo data not found for the given blockhash '%s'.", snapshot_blockhash->ToString()));
402 : 0 : return false;
403 : : }
404 : 0 : const AssumeutxoData& au_data = *Assert(maybe_au_data);
405 : 0 : m_snapshot_height = au_data.height;
406 : 0 : CBlockIndex* base{LookupBlockIndex(*snapshot_blockhash)};
407 : :
408 : : // Since nChainTx (responsible for estimated progress) isn't persisted
409 : : // to disk, we must bootstrap the value for assumedvalid chainstates
410 : : // from the hardcoded assumeutxo chainparams.
411 : 0 : base->nChainTx = au_data.nChainTx;
412 [ # # ][ # # ]: 0 : LogPrintf("[snapshot] set nChainTx=%d for %s\n", au_data.nChainTx, snapshot_blockhash->ToString());
[ # # ][ # # ]
413 : 0 : } else {
414 : : // If this isn't called with a snapshot blockhash, make sure the cached snapshot height
415 : : // is null. This is relevant during snapshot completion, when the blockman may be loaded
416 : : // with a height that then needs to be cleared after the snapshot is fully validated.
417 : 0 : m_snapshot_height.reset();
418 : : }
419 : :
420 : 0 : Assert(m_snapshot_height.has_value() == snapshot_blockhash.has_value());
421 : :
422 : : // Calculate nChainWork
423 : 0 : std::vector<CBlockIndex*> vSortedByHeight{GetAllBlockIndices()};
424 [ # # ]: 0 : std::sort(vSortedByHeight.begin(), vSortedByHeight.end(),
425 : : CBlockIndexHeightOnlyComparator());
426 : :
427 : 0 : CBlockIndex* previous_index{nullptr};
428 [ # # ]: 0 : for (CBlockIndex* pindex : vSortedByHeight) {
429 [ # # ][ # # ]: 0 : if (m_interrupt) return false;
430 [ # # ][ # # ]: 0 : if (previous_index && pindex->nHeight > previous_index->nHeight + 1) {
431 [ # # ]: 0 : return error("%s: block index is non-contiguous, index of height %d missing", __func__, previous_index->nHeight + 1);
432 : : }
433 : 0 : previous_index = pindex;
434 [ # # ][ # # ]: 0 : pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex);
[ # # ][ # # ]
[ # # ][ # # ]
[ # # ]
435 [ # # ][ # # ]: 0 : pindex->nTimeMax = (pindex->pprev ? std::max(pindex->pprev->nTimeMax, pindex->nTime) : pindex->nTime);
436 : :
437 : : // We can link the chain of blocks for which we've received transactions at some point, or
438 : : // blocks that are assumed-valid on the basis of snapshot load (see
439 : : // PopulateAndValidateSnapshot()).
440 : : // Pruned nodes may have deleted the block.
441 [ # # ]: 0 : if (pindex->nTx > 0) {
442 [ # # ]: 0 : if (pindex->pprev) {
443 [ # # ][ # # ]: 0 : if (m_snapshot_height && pindex->nHeight == *m_snapshot_height &&
[ # # ]
444 [ # # ][ # # ]: 0 : pindex->GetBlockHash() == *snapshot_blockhash) {
445 : : // Should have been set above; don't disturb it with code below.
446 [ # # ]: 0 : Assert(pindex->nChainTx > 0);
447 [ # # ]: 0 : } else if (pindex->pprev->nChainTx > 0) {
448 : 0 : pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx;
449 : 0 : } else {
450 : 0 : pindex->nChainTx = 0;
451 [ # # ][ # # ]: 0 : m_blocks_unlinked.insert(std::make_pair(pindex->pprev, pindex));
452 : : }
453 : 0 : } else {
454 : 0 : pindex->nChainTx = pindex->nTx;
455 : : }
456 : 0 : }
457 [ # # ][ # # ]: 0 : if (!(pindex->nStatus & BLOCK_FAILED_MASK) && pindex->pprev && (pindex->pprev->nStatus & BLOCK_FAILED_MASK)) {
[ # # ]
458 : 0 : pindex->nStatus |= BLOCK_FAILED_CHILD;
459 [ # # ]: 0 : m_dirty_blockindex.insert(pindex);
460 : 0 : }
461 [ # # ]: 0 : if (pindex->pprev) {
462 [ # # ]: 0 : pindex->BuildSkip();
463 : 0 : }
464 : : }
465 : :
466 : 0 : return true;
467 : 0 : }
468 : :
469 : 0 : bool BlockManager::WriteBlockIndexDB()
470 : : {
471 : 0 : AssertLockHeld(::cs_main);
472 : 0 : std::vector<std::pair<int, const CBlockFileInfo*>> vFiles;
473 [ # # ]: 0 : vFiles.reserve(m_dirty_fileinfo.size());
474 [ # # ]: 0 : for (std::set<int>::iterator it = m_dirty_fileinfo.begin(); it != m_dirty_fileinfo.end();) {
475 [ # # ]: 0 : vFiles.emplace_back(*it, &m_blockfile_info[*it]);
476 [ # # ]: 0 : m_dirty_fileinfo.erase(it++);
477 : : }
478 : 0 : std::vector<const CBlockIndex*> vBlocks;
479 [ # # ]: 0 : vBlocks.reserve(m_dirty_blockindex.size());
480 [ # # ]: 0 : for (std::set<CBlockIndex*>::iterator it = m_dirty_blockindex.begin(); it != m_dirty_blockindex.end();) {
481 [ # # ]: 0 : vBlocks.push_back(*it);
482 [ # # ]: 0 : m_dirty_blockindex.erase(it++);
483 : : }
484 [ # # ][ # # ]: 0 : int max_blockfile = WITH_LOCK(cs_LastBlockFile, return this->MaxBlockfileNum());
[ # # ]
485 [ # # ][ # # ]: 0 : if (!m_block_tree_db->WriteBatchSync(vFiles, max_blockfile, vBlocks)) {
486 : 0 : return false;
487 : : }
488 : 0 : return true;
489 : 0 : }
490 : :
491 : 0 : bool BlockManager::LoadBlockIndexDB(const std::optional<uint256>& snapshot_blockhash)
492 : : {
493 [ # # ]: 0 : if (!LoadBlockIndex(snapshot_blockhash)) {
494 : 0 : return false;
495 : : }
496 : 0 : int max_blockfile_num{0};
497 : :
498 : : // Load block file info
499 : 0 : m_block_tree_db->ReadLastBlockFile(max_blockfile_num);
500 : 0 : m_blockfile_info.resize(max_blockfile_num + 1);
501 [ # # ][ # # ]: 0 : LogPrintf("%s: last block file = %i\n", __func__, max_blockfile_num);
[ # # ]
502 [ # # ]: 0 : for (int nFile = 0; nFile <= max_blockfile_num; nFile++) {
503 : 0 : m_block_tree_db->ReadBlockFileInfo(nFile, m_blockfile_info[nFile]);
504 : 0 : }
505 [ # # ][ # # ]: 0 : LogPrintf("%s: last block file info: %s\n", __func__, m_blockfile_info[max_blockfile_num].ToString());
[ # # ][ # # ]
506 [ # # ]: 0 : for (int nFile = max_blockfile_num + 1; true; nFile++) {
507 : 0 : CBlockFileInfo info;
508 [ # # ]: 0 : if (m_block_tree_db->ReadBlockFileInfo(nFile, info)) {
509 : 0 : m_blockfile_info.push_back(info);
510 : 0 : } else {
511 : 0 : break;
512 : : }
513 : 0 : }
514 : :
515 : : // Check presence of blk files
516 [ # # ][ # # ]: 0 : LogPrintf("Checking all blk files are present...\n");
[ # # ]
517 : 0 : std::set<int> setBlkDataFiles;
518 [ # # ]: 0 : for (const auto& [_, block_index] : m_block_index) {
519 [ # # ]: 0 : if (block_index.nStatus & BLOCK_HAVE_DATA) {
520 [ # # ]: 0 : setBlkDataFiles.insert(block_index.nFile);
521 : 0 : }
522 : : }
523 [ # # ]: 0 : for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) {
524 [ # # ]: 0 : FlatFilePos pos(*it, 0);
525 [ # # ][ # # ]: 0 : if (OpenBlockFile(pos, true).IsNull()) {
[ # # ]
526 : 0 : return false;
527 : : }
528 : 0 : }
529 : :
530 : : {
531 : : // Initialize the blockfile cursors.
532 [ # # ][ # # ]: 0 : LOCK(cs_LastBlockFile);
533 [ # # ]: 0 : for (size_t i = 0; i < m_blockfile_info.size(); ++i) {
534 : 0 : const auto last_height_in_file = m_blockfile_info[i].nHeightLast;
535 [ # # ]: 0 : m_blockfile_cursors[BlockfileTypeForHeight(last_height_in_file)] = {static_cast<int>(i), 0};
536 : 0 : }
537 : 0 : }
538 : :
539 : : // Check whether we have ever pruned block & undo files
540 [ # # ][ # # ]: 0 : m_block_tree_db->ReadFlag("prunedblockfiles", m_have_pruned);
541 [ # # ]: 0 : if (m_have_pruned) {
542 [ # # ][ # # ]: 0 : LogPrintf("LoadBlockIndexDB(): Block files have previously been pruned\n");
[ # # ]
543 : 0 : }
544 : :
545 : : // Check whether we need to continue reindexing
546 : 0 : bool fReindexing = false;
547 [ # # ]: 0 : m_block_tree_db->ReadReindexing(fReindexing);
548 [ # # ]: 0 : if (fReindexing) fReindex = true;
549 : :
550 : 0 : return true;
551 : 0 : }
552 : :
553 : 0 : void BlockManager::ScanAndUnlinkAlreadyPrunedFiles()
554 : : {
555 : 0 : AssertLockHeld(::cs_main);
556 [ # # ]: 0 : int max_blockfile = WITH_LOCK(cs_LastBlockFile, return this->MaxBlockfileNum());
557 [ # # ]: 0 : if (!m_have_pruned) {
558 : 0 : return;
559 : : }
560 : :
561 : 0 : std::set<int> block_files_to_prune;
562 [ # # ]: 0 : for (int file_number = 0; file_number < max_blockfile; file_number++) {
563 [ # # ]: 0 : if (m_blockfile_info[file_number].nSize == 0) {
564 [ # # ]: 0 : block_files_to_prune.insert(file_number);
565 : 0 : }
566 : 0 : }
567 : :
568 [ # # ]: 0 : UnlinkPrunedFiles(block_files_to_prune);
569 : 0 : }
570 : :
571 : 0 : const CBlockIndex* BlockManager::GetLastCheckpoint(const CCheckpointData& data)
572 : : {
573 : 0 : const MapCheckpoints& checkpoints = data.mapCheckpoints;
574 : :
575 [ # # ]: 0 : for (const MapCheckpoints::value_type& i : reverse_iterate(checkpoints)) {
576 : 0 : const uint256& hash = i.second;
577 : 0 : const CBlockIndex* pindex = LookupBlockIndex(hash);
578 [ # # ]: 0 : if (pindex) {
579 : 0 : return pindex;
580 : : }
581 : : }
582 : 0 : return nullptr;
583 : 0 : }
584 : :
585 : 0 : bool BlockManager::IsBlockPruned(const CBlockIndex& block)
586 : : {
587 : 0 : AssertLockHeld(::cs_main);
588 [ # # ][ # # ]: 0 : return m_have_pruned && !(block.nStatus & BLOCK_HAVE_DATA) && (block.nTx > 0);
589 : : }
590 : :
591 : 0 : const CBlockIndex* BlockManager::GetFirstStoredBlock(const CBlockIndex& upper_block, const CBlockIndex* lower_block)
592 : : {
593 : 0 : AssertLockHeld(::cs_main);
594 : 0 : const CBlockIndex* last_block = &upper_block;
595 [ # # ]: 0 : assert(last_block->nStatus & BLOCK_HAVE_DATA); // 'upper_block' must have data
596 [ # # ][ # # ]: 0 : while (last_block->pprev && (last_block->pprev->nStatus & BLOCK_HAVE_DATA)) {
597 [ # # ]: 0 : if (lower_block) {
598 : : // Return if we reached the lower_block
599 [ # # ]: 0 : if (last_block == lower_block) return lower_block;
600 : : // if range was surpassed, means that 'lower_block' is not part of the 'upper_block' chain
601 : : // and so far this is not allowed.
602 [ # # ]: 0 : assert(last_block->nHeight >= lower_block->nHeight);
603 : 0 : }
604 : 0 : last_block = last_block->pprev;
605 : : }
606 [ # # ]: 0 : assert(last_block != nullptr);
607 : 0 : return last_block;
608 : 0 : }
609 : :
610 : 0 : bool BlockManager::CheckBlockDataAvailability(const CBlockIndex& upper_block, const CBlockIndex& lower_block)
611 : : {
612 [ # # ]: 0 : if (!(upper_block.nStatus & BLOCK_HAVE_DATA)) return false;
613 : 0 : return GetFirstStoredBlock(upper_block, &lower_block) == &lower_block;
614 : 0 : }
615 : :
616 : : // If we're using -prune with -reindex, then delete block files that will be ignored by the
617 : : // reindex. Since reindexing works by starting at block file 0 and looping until a blockfile
618 : : // is missing, do the same here to delete any later block files after a gap. Also delete all
619 : : // rev files since they'll be rewritten by the reindex anyway. This ensures that m_blockfile_info
620 : : // is in sync with what's actually on disk by the time we start downloading, so that pruning
621 : : // works correctly.
622 : 0 : void BlockManager::CleanupBlockRevFiles() const
623 : : {
624 : 0 : std::map<std::string, fs::path> mapBlockFiles;
625 : :
626 : : // Glob all blk?????.dat and rev?????.dat files from the blocks directory.
627 : : // Remove the rev files immediately and insert the blk file paths into an
628 : : // ordered map keyed by block file index.
629 [ # # ][ # # ]: 0 : LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n");
[ # # ]
630 [ # # ][ # # ]: 0 : for (fs::directory_iterator it(m_opts.blocks_dir); it != fs::directory_iterator(); it++) {
[ # # ]
631 [ # # ][ # # ]: 0 : const std::string path = fs::PathToString(it->path().filename());
[ # # ]
632 [ # # ][ # # ]: 0 : if (fs::is_regular_file(*it) &&
[ # # ][ # # ]
[ # # ]
633 [ # # ]: 0 : path.length() == 12 &&
634 [ # # ][ # # ]: 0 : path.substr(8,4) == ".dat")
635 : : {
636 [ # # ][ # # ]: 0 : if (path.substr(0, 3) == "blk") {
[ # # ]
637 [ # # ][ # # ]: 0 : mapBlockFiles[path.substr(3, 5)] = it->path();
[ # # ][ # # ]
638 [ # # ][ # # ]: 0 : } else if (path.substr(0, 3) == "rev") {
[ # # ]
639 [ # # ]: 0 : remove(it->path());
640 : 0 : }
641 : 0 : }
642 : 0 : }
643 : :
644 : : // Remove all block files that aren't part of a contiguous set starting at
645 : : // zero by walking the ordered map (keys are block file indices) by
646 : : // keeping a separate counter. Once we hit a gap (or if 0 doesn't exist)
647 : : // start removing block files.
648 : 0 : int nContigCounter = 0;
649 [ # # ]: 0 : for (const std::pair<const std::string, fs::path>& item : mapBlockFiles) {
650 [ # # ][ # # ]: 0 : if (LocaleIndependentAtoi<int>(item.first) == nContigCounter) {
651 : 0 : nContigCounter++;
652 : 0 : continue;
653 : : }
654 [ # # ]: 0 : remove(item.second);
655 : : }
656 : 0 : }
657 : :
658 : 0 : CBlockFileInfo* BlockManager::GetBlockFileInfo(size_t n)
659 : : {
660 : 0 : LOCK(cs_LastBlockFile);
661 : :
662 [ # # ]: 0 : return &m_blockfile_info.at(n);
663 : 0 : }
664 : :
665 : 0 : bool BlockManager::UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock) const
666 : : {
667 : : // Open history file to append
668 : 0 : AutoFile fileout{OpenUndoFile(pos)};
669 [ # # ][ # # ]: 0 : if (fileout.IsNull()) {
670 [ # # ]: 0 : return error("%s: OpenUndoFile failed", __func__);
671 : : }
672 : :
673 : : // Write index header
674 [ # # ]: 0 : unsigned int nSize = GetSerializeSize(blockundo);
675 [ # # ][ # # ]: 0 : fileout << GetParams().MessageStart() << nSize;
[ # # ][ # # ]
676 : :
677 : : // Write undo data
678 [ # # ][ # # ]: 0 : long fileOutPos = ftell(fileout.Get());
679 [ # # ]: 0 : if (fileOutPos < 0) {
680 [ # # ]: 0 : return error("%s: ftell failed", __func__);
681 : : }
682 : 0 : pos.nPos = (unsigned int)fileOutPos;
683 [ # # ]: 0 : fileout << blockundo;
684 : :
685 : : // calculate & write checksum
686 [ # # ]: 0 : HashWriter hasher{};
687 [ # # ]: 0 : hasher << hashBlock;
688 [ # # ]: 0 : hasher << blockundo;
689 [ # # ][ # # ]: 0 : fileout << hasher.GetHash();
690 : :
691 : 0 : return true;
692 : 0 : }
693 : :
694 : 0 : bool BlockManager::UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& index) const
695 : : {
696 [ # # ]: 0 : const FlatFilePos pos{WITH_LOCK(::cs_main, return index.GetUndoPos())};
697 : :
698 [ # # ]: 0 : if (pos.IsNull()) {
699 : 0 : return error("%s: no undo data available", __func__);
700 : : }
701 : :
702 : : // Open history file to read
703 : 0 : AutoFile filein{OpenUndoFile(pos, true)};
704 [ # # ][ # # ]: 0 : if (filein.IsNull()) {
705 [ # # ]: 0 : return error("%s: OpenUndoFile failed", __func__);
706 : : }
707 : :
708 : : // Read block
709 [ # # ]: 0 : uint256 hashChecksum;
710 [ # # ]: 0 : HashVerifier verifier{filein}; // Use HashVerifier as reserializing may lose data, c.f. commit d342424301013ec47dc146a4beb49d5c9319d80a
711 : : try {
712 [ # # ][ # # ]: 0 : verifier << index.pprev->GetBlockHash();
713 [ # # ]: 0 : verifier >> blockundo;
714 [ # # ]: 0 : filein >> hashChecksum;
715 [ # # ]: 0 : } catch (const std::exception& e) {
716 [ # # ]: 0 : return error("%s: Deserialize or I/O error - %s", __func__, e.what());
717 [ # # ][ # # ]: 0 : }
718 : :
719 : : // Verify checksum
720 [ # # ][ # # ]: 0 : if (hashChecksum != verifier.GetHash()) {
[ # # ]
721 [ # # ]: 0 : return error("%s: Checksum mismatch", __func__);
722 : : }
723 : :
724 : 0 : return true;
725 : 0 : }
726 : :
727 : 0 : bool BlockManager::FlushUndoFile(int block_file, bool finalize)
728 : : {
729 : 0 : FlatFilePos undo_pos_old(block_file, m_blockfile_info[block_file].nUndoSize);
730 [ # # ][ # # ]: 0 : if (!UndoFileSeq().Flush(undo_pos_old, finalize)) {
731 [ # # ][ # # ]: 0 : m_opts.notifications.flushError("Flushing undo file to disk failed. This is likely the result of an I/O error.");
732 : 0 : return false;
733 : : }
734 : 0 : return true;
735 : 0 : }
736 : :
737 : 0 : bool BlockManager::FlushBlockFile(int blockfile_num, bool fFinalize, bool finalize_undo)
738 : : {
739 : 0 : bool success = true;
740 : 0 : LOCK(cs_LastBlockFile);
741 : :
742 [ # # ]: 0 : if (m_blockfile_info.size() < 1) {
743 : : // Return if we haven't loaded any blockfiles yet. This happens during
744 : : // chainstate init, when we call ChainstateManager::MaybeRebalanceCaches() (which
745 : : // then calls FlushStateToDisk()), resulting in a call to this function before we
746 : : // have populated `m_blockfile_info` via LoadBlockIndexDB().
747 : 0 : return true;
748 : : }
749 [ # # ]: 0 : assert(static_cast<int>(m_blockfile_info.size()) > blockfile_num);
750 : :
751 [ # # ]: 0 : FlatFilePos block_pos_old(blockfile_num, m_blockfile_info[blockfile_num].nSize);
752 [ # # ][ # # ]: 0 : if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) {
[ # # ]
753 [ # # ][ # # ]: 0 : m_opts.notifications.flushError("Flushing block file to disk failed. This is likely the result of an I/O error.");
754 : 0 : success = false;
755 : 0 : }
756 : : // we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks,
757 : : // e.g. during IBD or a sync after a node going offline
758 [ # # ][ # # ]: 0 : if (!fFinalize || finalize_undo) {
759 [ # # ][ # # ]: 0 : if (!FlushUndoFile(blockfile_num, finalize_undo)) {
760 : 0 : success = false;
761 : 0 : }
762 : 0 : }
763 : 0 : return success;
764 : 0 : }
765 : :
766 : 0 : BlockfileType BlockManager::BlockfileTypeForHeight(int height)
767 : : {
768 [ # # ]: 0 : if (!m_snapshot_height) {
769 : 0 : return BlockfileType::NORMAL;
770 : : }
771 : 0 : return (height >= *m_snapshot_height) ? BlockfileType::ASSUMED : BlockfileType::NORMAL;
772 : 0 : }
773 : :
774 : 0 : bool BlockManager::FlushChainstateBlockFile(int tip_height)
775 : : {
776 : 0 : LOCK(cs_LastBlockFile);
777 : 0 : auto& cursor = m_blockfile_cursors[BlockfileTypeForHeight(tip_height)];
778 : : // If the cursor does not exist, it means an assumeutxo snapshot is loaded,
779 : : // but no blocks past the snapshot height have been written yet, so there
780 : : // is no data associated with the chainstate, and it is safe not to flush.
781 [ # # ]: 0 : if (cursor) {
782 [ # # ]: 0 : return FlushBlockFile(cursor->file_num, /*fFinalize=*/false, /*finalize_undo=*/false);
783 : : }
784 : : // No need to log warnings in this case.
785 : 0 : return true;
786 : 0 : }
787 : :
788 : 0 : uint64_t BlockManager::CalculateCurrentUsage()
789 : : {
790 : 0 : LOCK(cs_LastBlockFile);
791 : :
792 : 0 : uint64_t retval = 0;
793 [ # # ]: 0 : for (const CBlockFileInfo& file : m_blockfile_info) {
794 : 0 : retval += file.nSize + file.nUndoSize;
795 : : }
796 : 0 : return retval;
797 : 0 : }
798 : :
799 : 0 : void BlockManager::UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) const
800 : : {
801 : 0 : std::error_code ec;
802 [ # # ]: 0 : for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) {
803 : 0 : FlatFilePos pos(*it, 0);
804 [ # # ]: 0 : const bool removed_blockfile{fs::remove(BlockFileSeq().FileName(pos), ec)};
805 [ # # ]: 0 : const bool removed_undofile{fs::remove(UndoFileSeq().FileName(pos), ec)};
806 [ # # ][ # # ]: 0 : if (removed_blockfile || removed_undofile) {
807 [ # # ][ # # ]: 0 : LogPrint(BCLog::BLOCKSTORAGE, "Prune: %s deleted blk/rev (%05u)\n", __func__, *it);
[ # # ][ # # ]
808 : 0 : }
809 : 0 : }
810 : 0 : }
811 : :
812 : 0 : FlatFileSeq BlockManager::BlockFileSeq() const
813 : : {
814 [ # # ]: 0 : return FlatFileSeq(m_opts.blocks_dir, "blk", m_opts.fast_prune ? 0x4000 /* 16kb */ : BLOCKFILE_CHUNK_SIZE);
815 : 0 : }
816 : :
817 : 0 : FlatFileSeq BlockManager::UndoFileSeq() const
818 : : {
819 [ # # ]: 0 : return FlatFileSeq(m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE);
820 : 0 : }
821 : :
822 : 0 : AutoFile BlockManager::OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) const
823 : : {
824 [ # # ][ # # ]: 0 : return AutoFile{BlockFileSeq().Open(pos, fReadOnly)};
825 : 0 : }
826 : :
827 : : /** Open an undo file (rev?????.dat) */
828 : 0 : AutoFile BlockManager::OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) const
829 : : {
830 [ # # ][ # # ]: 0 : return AutoFile{UndoFileSeq().Open(pos, fReadOnly)};
831 : 0 : }
832 : :
833 : 0 : fs::path BlockManager::GetBlockPosFilename(const FlatFilePos& pos) const
834 : : {
835 [ # # ]: 0 : return BlockFileSeq().FileName(pos);
836 : 0 : }
837 : :
838 : 0 : bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown)
839 : : {
840 : 0 : LOCK(cs_LastBlockFile);
841 : :
842 : 0 : const BlockfileType chain_type = BlockfileTypeForHeight(nHeight);
843 : :
844 [ # # ]: 0 : if (!m_blockfile_cursors[chain_type]) {
845 : : // If a snapshot is loaded during runtime, we may not have initialized this cursor yet.
846 [ # # ]: 0 : assert(chain_type == BlockfileType::ASSUMED);
847 [ # # ]: 0 : const auto new_cursor = BlockfileCursor{this->MaxBlockfileNum() + 1};
848 : 0 : m_blockfile_cursors[chain_type] = new_cursor;
849 [ # # ][ # # ]: 0 : LogPrint(BCLog::BLOCKSTORAGE, "[%s] initializing blockfile cursor to %s\n", chain_type, new_cursor);
[ # # ][ # # ]
[ # # ]
850 : 0 : }
851 : 0 : const int last_blockfile = m_blockfile_cursors[chain_type]->file_num;
852 : :
853 [ # # ]: 0 : int nFile = fKnown ? pos.nFile : last_blockfile;
854 [ # # ]: 0 : if (static_cast<int>(m_blockfile_info.size()) <= nFile) {
855 [ # # ]: 0 : m_blockfile_info.resize(nFile + 1);
856 : 0 : }
857 : :
858 : 0 : bool finalize_undo = false;
859 [ # # ]: 0 : if (!fKnown) {
860 : 0 : unsigned int max_blockfile_size{MAX_BLOCKFILE_SIZE};
861 : : // Use smaller blockfiles in test-only -fastprune mode - but avoid
862 : : // the possibility of having a block not fit into the block file.
863 [ # # ]: 0 : if (m_opts.fast_prune) {
864 : 0 : max_blockfile_size = 0x10000; // 64kiB
865 [ # # ]: 0 : if (nAddSize >= max_blockfile_size) {
866 : : // dynamically adjust the blockfile size to be larger than the added size
867 : 0 : max_blockfile_size = nAddSize + 1;
868 : 0 : }
869 : 0 : }
870 [ # # ]: 0 : assert(nAddSize < max_blockfile_size);
871 : :
872 [ # # ]: 0 : while (m_blockfile_info[nFile].nSize + nAddSize >= max_blockfile_size) {
873 : : // when the undo file is keeping up with the block file, we want to flush it explicitly
874 : : // when it is lagging behind (more blocks arrive than are being connected), we let the
875 : : // undo block write case handle it
876 : 0 : finalize_undo = (static_cast<int>(m_blockfile_info[nFile].nHeightLast) ==
877 [ # # ]: 0 : Assert(m_blockfile_cursors[chain_type])->undo_height);
878 : :
879 : : // Try the next unclaimed blockfile number
880 [ # # ]: 0 : nFile = this->MaxBlockfileNum() + 1;
881 : : // Set to increment MaxBlockfileNum() for next iteration
882 : 0 : m_blockfile_cursors[chain_type] = BlockfileCursor{nFile};
883 : :
884 [ # # ]: 0 : if (static_cast<int>(m_blockfile_info.size()) <= nFile) {
885 [ # # ]: 0 : m_blockfile_info.resize(nFile + 1);
886 : 0 : }
887 : : }
888 : 0 : pos.nFile = nFile;
889 : 0 : pos.nPos = m_blockfile_info[nFile].nSize;
890 : 0 : }
891 : :
892 [ # # ]: 0 : if (nFile != last_blockfile) {
893 [ # # ]: 0 : if (!fKnown) {
894 [ # # ][ # # ]: 0 : LogPrint(BCLog::BLOCKSTORAGE, "Leaving block file %i: %s (onto %i) (height %i)\n",
[ # # ][ # # ]
[ # # ][ # # ]
895 : : last_blockfile, m_blockfile_info[last_blockfile].ToString(), nFile, nHeight);
896 : 0 : }
897 : :
898 : : // Do not propagate the return code. The flush concerns a previous block
899 : : // and undo file that has already been written to. If a flush fails
900 : : // here, and we crash, there is no expected additional block data
901 : : // inconsistency arising from the flush failure here. However, the undo
902 : : // data may be inconsistent after a crash if the flush is called during
903 : : // a reindex. A flush error might also leave some of the data files
904 : : // untrimmed.
905 [ # # ][ # # ]: 0 : if (!FlushBlockFile(last_blockfile, !fKnown, finalize_undo)) {
906 [ # # ][ # # ]: 0 : LogPrintLevel(BCLog::BLOCKSTORAGE, BCLog::Level::Warning,
[ # # ][ # # ]
[ # # ]
907 : : "Failed to flush previous block file %05i (finalize=%i, finalize_undo=%i) before opening new block file %05i\n",
908 : : last_blockfile, !fKnown, finalize_undo, nFile);
909 : 0 : }
910 : : // No undo data yet in the new file, so reset our undo-height tracking.
911 : 0 : m_blockfile_cursors[chain_type] = BlockfileCursor{nFile};
912 : 0 : }
913 : :
914 [ # # ]: 0 : m_blockfile_info[nFile].AddBlock(nHeight, nTime);
915 [ # # ]: 0 : if (fKnown) {
916 [ # # ]: 0 : m_blockfile_info[nFile].nSize = std::max(pos.nPos + nAddSize, m_blockfile_info[nFile].nSize);
917 : 0 : } else {
918 : 0 : m_blockfile_info[nFile].nSize += nAddSize;
919 : : }
920 : :
921 [ # # ]: 0 : if (!fKnown) {
922 : : bool out_of_space;
923 [ # # ][ # # ]: 0 : size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space);
924 [ # # ]: 0 : if (out_of_space) {
925 [ # # ][ # # ]: 0 : m_opts.notifications.fatalError("Disk space is too low!", _("Disk space is too low!"));
[ # # ]
926 : 0 : return false;
927 : : }
928 [ # # ][ # # ]: 0 : if (bytes_allocated != 0 && IsPruneMode()) {
[ # # ]
929 : 0 : m_check_for_pruning = true;
930 : 0 : }
931 : 0 : }
932 : :
933 [ # # ]: 0 : m_dirty_fileinfo.insert(nFile);
934 : 0 : return true;
935 : 0 : }
936 : :
937 : 0 : bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize)
938 : : {
939 : 0 : pos.nFile = nFile;
940 : :
941 : 0 : LOCK(cs_LastBlockFile);
942 : :
943 : 0 : pos.nPos = m_blockfile_info[nFile].nUndoSize;
944 : 0 : m_blockfile_info[nFile].nUndoSize += nAddSize;
945 [ # # ]: 0 : m_dirty_fileinfo.insert(nFile);
946 : :
947 : : bool out_of_space;
948 [ # # ][ # # ]: 0 : size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space);
949 [ # # ]: 0 : if (out_of_space) {
950 [ # # ][ # # ]: 0 : return FatalError(m_opts.notifications, state, "Disk space is too low!", _("Disk space is too low!"));
[ # # ]
951 : : }
952 [ # # ][ # # ]: 0 : if (bytes_allocated != 0 && IsPruneMode()) {
[ # # ]
953 : 0 : m_check_for_pruning = true;
954 : 0 : }
955 : :
956 : 0 : return true;
957 : 0 : }
958 : :
959 : 0 : bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const
960 : : {
961 : : // Open history file to append
962 : 0 : AutoFile fileout{OpenBlockFile(pos)};
963 [ # # ][ # # ]: 0 : if (fileout.IsNull()) {
964 [ # # ]: 0 : return error("WriteBlockToDisk: OpenBlockFile failed");
965 : : }
966 : :
967 : : // Write index header
968 [ # # ][ # # ]: 0 : unsigned int nSize = GetSerializeSize(TX_WITH_WITNESS(block));
969 [ # # ][ # # ]: 0 : fileout << GetParams().MessageStart() << nSize;
[ # # ][ # # ]
970 : :
971 : : // Write block
972 [ # # ][ # # ]: 0 : long fileOutPos = ftell(fileout.Get());
973 [ # # ]: 0 : if (fileOutPos < 0) {
974 [ # # ]: 0 : return error("WriteBlockToDisk: ftell failed");
975 : : }
976 : 0 : pos.nPos = (unsigned int)fileOutPos;
977 [ # # ][ # # ]: 0 : fileout << TX_WITH_WITNESS(block);
978 : :
979 : 0 : return true;
980 : 0 : }
981 : :
982 : 0 : bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block)
983 : : {
984 : 0 : AssertLockHeld(::cs_main);
985 : 0 : const BlockfileType type = BlockfileTypeForHeight(block.nHeight);
986 : 0 : auto& cursor = *Assert(WITH_LOCK(cs_LastBlockFile, return m_blockfile_cursors[type]));
987 : :
988 : : // Write undo information to disk
989 [ # # ]: 0 : if (block.GetUndoPos().IsNull()) {
990 : 0 : FlatFilePos _pos;
991 [ # # ]: 0 : if (!FindUndoPos(state, block.nFile, _pos, ::GetSerializeSize(blockundo) + 40)) {
992 : 0 : return error("ConnectBlock(): FindUndoPos failed");
993 : : }
994 [ # # ]: 0 : if (!UndoWriteToDisk(blockundo, _pos, block.pprev->GetBlockHash())) {
995 [ # # ][ # # ]: 0 : return FatalError(m_opts.notifications, state, "Failed to write undo data");
996 : : }
997 : : // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order)
998 : : // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height
999 : : // in the block file info as below; note that this does not catch the case where the undo writes are keeping up
1000 : : // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in
1001 : : // the FindBlockPos function
1002 [ # # ][ # # ]: 0 : if (_pos.nFile < cursor.file_num && static_cast<uint32_t>(block.nHeight) == m_blockfile_info[_pos.nFile].nHeightLast) {
1003 : : // Do not propagate the return code, a failed flush here should not
1004 : : // be an indication for a failed write. If it were propagated here,
1005 : : // the caller would assume the undo data not to be written, when in
1006 : : // fact it is. Note though, that a failed flush might leave the data
1007 : : // file untrimmed.
1008 [ # # ]: 0 : if (!FlushUndoFile(_pos.nFile, true)) {
1009 [ # # ][ # # ]: 0 : LogPrintLevel(BCLog::BLOCKSTORAGE, BCLog::Level::Warning, "Failed to flush undo file %05i\n", _pos.nFile);
[ # # ][ # # ]
1010 : 0 : }
1011 [ # # ][ # # ]: 0 : } else if (_pos.nFile == cursor.file_num && block.nHeight > cursor.undo_height) {
1012 : 0 : cursor.undo_height = block.nHeight;
1013 : 0 : }
1014 : : // update nUndoPos in block index
1015 : 0 : block.nUndoPos = _pos.nPos;
1016 : 0 : block.nStatus |= BLOCK_HAVE_UNDO;
1017 : 0 : m_dirty_blockindex.insert(&block);
1018 : 0 : }
1019 : :
1020 : 0 : return true;
1021 : 0 : }
1022 : :
1023 : 0 : bool BlockManager::ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos) const
1024 : : {
1025 : 0 : block.SetNull();
1026 : :
1027 : : // Open history file to read
1028 : 0 : AutoFile filein{OpenBlockFile(pos, true)};
1029 [ # # ][ # # ]: 0 : if (filein.IsNull()) {
1030 [ # # ][ # # ]: 0 : return error("ReadBlockFromDisk: OpenBlockFile failed for %s", pos.ToString());
1031 : : }
1032 : :
1033 : : // Read block
1034 : : try {
1035 [ # # ][ # # ]: 0 : filein >> TX_WITH_WITNESS(block);
1036 [ # # ]: 0 : } catch (const std::exception& e) {
1037 [ # # ][ # # ]: 0 : return error("%s: Deserialize or I/O error - %s at %s", __func__, e.what(), pos.ToString());
1038 [ # # ][ # # ]: 0 : }
1039 : :
1040 : : // Check the header
1041 [ # # ][ # # ]: 0 : if (!CheckProofOfWork(block.GetHash(), block.nBits, GetConsensus())) {
[ # # ][ # # ]
1042 [ # # ][ # # ]: 0 : return error("ReadBlockFromDisk: Errors in block header at %s", pos.ToString());
1043 : : }
1044 : :
1045 : : // Signet only: check block solution
1046 [ # # ][ # # ]: 0 : if (GetConsensus().signet_blocks && !CheckSignetBlockSolution(block, GetConsensus())) {
[ # # ][ # # ]
[ # # ]
1047 [ # # ][ # # ]: 0 : return error("ReadBlockFromDisk: Errors in block solution at %s", pos.ToString());
1048 : : }
1049 : :
1050 : 0 : return true;
1051 : 0 : }
1052 : :
1053 : 0 : bool BlockManager::ReadBlockFromDisk(CBlock& block, const CBlockIndex& index) const
1054 : : {
1055 [ # # ]: 0 : const FlatFilePos block_pos{WITH_LOCK(cs_main, return index.GetBlockPos())};
1056 : :
1057 [ # # ]: 0 : if (!ReadBlockFromDisk(block, block_pos)) {
1058 : 0 : return false;
1059 : : }
1060 [ # # ]: 0 : if (block.GetHash() != index.GetBlockHash()) {
1061 [ # # ]: 0 : return error("ReadBlockFromDisk(CBlock&, CBlockIndex*): GetHash() doesn't match index for %s at %s",
1062 [ # # ]: 0 : index.ToString(), block_pos.ToString());
1063 : : }
1064 : 0 : return true;
1065 : 0 : }
1066 : :
1067 : 0 : bool BlockManager::ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos) const
1068 : : {
1069 : 0 : FlatFilePos hpos = pos;
1070 : 0 : hpos.nPos -= 8; // Seek back 8 bytes for meta header
1071 : 0 : AutoFile filein{OpenBlockFile(hpos, true)};
1072 [ # # ][ # # ]: 0 : if (filein.IsNull()) {
1073 [ # # ][ # # ]: 0 : return error("%s: OpenBlockFile failed for %s", __func__, pos.ToString());
1074 : : }
1075 : :
1076 : : try {
1077 : : MessageStartChars blk_start;
1078 : : unsigned int blk_size;
1079 : :
1080 [ # # ][ # # ]: 0 : filein >> blk_start >> blk_size;
1081 : :
1082 [ # # ][ # # ]: 0 : if (blk_start != GetParams().MessageStart()) {
[ # # ][ # # ]
1083 [ # # ][ # # ]: 0 : return error("%s: Block magic mismatch for %s: %s versus expected %s", __func__, pos.ToString(),
1084 [ # # ][ # # ]: 0 : HexStr(blk_start),
1085 [ # # ][ # # ]: 0 : HexStr(GetParams().MessageStart()));
[ # # ][ # # ]
1086 : : }
1087 : :
1088 [ # # ]: 0 : if (blk_size > MAX_SIZE) {
1089 [ # # ][ # # ]: 0 : return error("%s: Block data is larger than maximum deserialization size for %s: %s versus %s", __func__, pos.ToString(),
1090 : : blk_size, MAX_SIZE);
1091 : : }
1092 : :
1093 [ # # ]: 0 : block.resize(blk_size); // Zeroing of memory is intentional here
1094 [ # # ]: 0 : filein.read(MakeWritableByteSpan(block));
1095 [ # # ]: 0 : } catch (const std::exception& e) {
1096 [ # # ][ # # ]: 0 : return error("%s: Read from block file failed: %s for %s", __func__, e.what(), pos.ToString());
1097 [ # # ][ # # ]: 0 : }
1098 : :
1099 : 0 : return true;
1100 : 0 : }
1101 : :
1102 : 0 : FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, const FlatFilePos* dbp)
1103 : : {
1104 : 0 : unsigned int nBlockSize = ::GetSerializeSize(TX_WITH_WITNESS(block));
1105 : 0 : FlatFilePos blockPos;
1106 : 0 : const auto position_known {dbp != nullptr};
1107 [ # # ]: 0 : if (position_known) {
1108 : 0 : blockPos = *dbp;
1109 : 0 : } else {
1110 : : // when known, blockPos.nPos points at the offset of the block data in the blk file. that already accounts for
1111 : : // the serialization header present in the file (the 4 magic message start bytes + the 4 length bytes = 8 bytes = BLOCK_SERIALIZATION_HEADER_SIZE).
1112 : : // we add BLOCK_SERIALIZATION_HEADER_SIZE only for new blocks since they will have the serialization header added when written to disk.
1113 : 0 : nBlockSize += static_cast<unsigned int>(BLOCK_SERIALIZATION_HEADER_SIZE);
1114 : : }
1115 [ # # ]: 0 : if (!FindBlockPos(blockPos, nBlockSize, nHeight, block.GetBlockTime(), position_known)) {
1116 : 0 : error("%s: FindBlockPos failed", __func__);
1117 : 0 : return FlatFilePos();
1118 : : }
1119 [ # # ]: 0 : if (!position_known) {
1120 [ # # ]: 0 : if (!WriteBlockToDisk(block, blockPos)) {
1121 [ # # ][ # # ]: 0 : m_opts.notifications.fatalError("Failed to write block");
1122 : 0 : return FlatFilePos();
1123 : : }
1124 : 0 : }
1125 : 0 : return blockPos;
1126 : 0 : }
1127 : :
1128 : : class ImportingNow
1129 : : {
1130 : : std::atomic<bool>& m_importing;
1131 : :
1132 : : public:
1133 : 0 : ImportingNow(std::atomic<bool>& importing) : m_importing{importing}
1134 : : {
1135 [ # # ]: 0 : assert(m_importing == false);
1136 : 0 : m_importing = true;
1137 : 0 : }
1138 : 0 : ~ImportingNow()
1139 : : {
1140 [ # # ]: 0 : assert(m_importing == true);
1141 : 0 : m_importing = false;
1142 : 0 : }
1143 : : };
1144 : :
1145 : 0 : void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFiles)
1146 : : {
1147 : 0 : ScheduleBatchPriority();
1148 : :
1149 : : {
1150 : 0 : ImportingNow imp{chainman.m_blockman.m_importing};
1151 : :
1152 : : // -reindex
1153 [ # # ]: 0 : if (fReindex) {
1154 : 0 : int nFile = 0;
1155 : : // Map of disk positions for blocks with unknown parent (only used for reindex);
1156 : : // parent hash -> child disk position, multiple children can have the same parent.
1157 : 0 : std::multimap<uint256, FlatFilePos> blocks_with_unknown_parent;
1158 : 0 : while (true) {
1159 [ # # ]: 0 : FlatFilePos pos(nFile, 0);
1160 [ # # ][ # # ]: 0 : if (!fs::exists(chainman.m_blockman.GetBlockPosFilename(pos))) {
[ # # ]
1161 : 0 : break; // No block files left to reindex
1162 : : }
1163 [ # # ]: 0 : AutoFile file{chainman.m_blockman.OpenBlockFile(pos, true)};
1164 [ # # ][ # # ]: 0 : if (file.IsNull()) {
1165 : 0 : break; // This error is logged in OpenBlockFile
1166 : : }
1167 [ # # ][ # # ]: 0 : LogPrintf("Reindexing block file blk%05u.dat...\n", (unsigned int)nFile);
[ # # ]
1168 [ # # ]: 0 : chainman.LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent);
1169 [ # # ][ # # ]: 0 : if (chainman.m_interrupt) {
1170 [ # # ][ # # ]: 0 : LogPrintf("Interrupt requested. Exit %s\n", __func__);
[ # # ]
1171 : 0 : return;
1172 : : }
1173 : 0 : nFile++;
1174 [ # # # ]: 0 : }
1175 [ # # ][ # # ]: 0 : WITH_LOCK(::cs_main, chainman.m_blockman.m_block_tree_db->WriteReindexing(false));
[ # # ]
1176 : 0 : fReindex = false;
1177 [ # # ][ # # ]: 0 : LogPrintf("Reindexing finished\n");
[ # # ]
1178 : : // To avoid ending up in a situation without genesis block, re-try initializing (no-op if reindexing worked):
1179 [ # # ][ # # ]: 0 : chainman.ActiveChainstate().LoadGenesisBlock();
1180 [ # # ]: 0 : }
1181 : :
1182 : : // -loadblock=
1183 [ # # ]: 0 : for (const fs::path& path : vImportFiles) {
1184 [ # # ][ # # ]: 0 : AutoFile file{fsbridge::fopen(path, "rb")};
1185 [ # # ][ # # ]: 0 : if (!file.IsNull()) {
1186 [ # # ][ # # ]: 0 : LogPrintf("Importing blocks file %s...\n", fs::PathToString(path));
[ # # ][ # # ]
1187 [ # # ]: 0 : chainman.LoadExternalBlockFile(file);
1188 [ # # ][ # # ]: 0 : if (chainman.m_interrupt) {
1189 [ # # ][ # # ]: 0 : LogPrintf("Interrupt requested. Exit %s\n", __func__);
[ # # ]
1190 : 0 : return;
1191 : : }
1192 : 0 : } else {
1193 [ # # ][ # # ]: 0 : LogPrintf("Warning: Could not open blocks file %s\n", fs::PathToString(path));
[ # # ][ # # ]
1194 : : }
1195 [ # # ]: 0 : }
1196 : :
1197 : : // scan for better chains in the block chain database, that are not yet connected in the active best chain
1198 : :
1199 : : // We can't hold cs_main during ActivateBestChain even though we're accessing
1200 : : // the chainman unique_ptrs since ABC requires us not to be holding cs_main, so retrieve
1201 : : // the relevant pointers before the ABC call.
1202 [ # # ][ # # ]: 0 : for (Chainstate* chainstate : WITH_LOCK(::cs_main, return chainman.GetAll())) {
[ # # ][ # # ]
[ # # ]
1203 : 0 : BlockValidationState state;
1204 [ # # ][ # # ]: 0 : if (!chainstate->ActivateBestChain(state, nullptr)) {
1205 [ # # ][ # # ]: 0 : chainman.GetNotifications().fatalError(strprintf("Failed to connect best block (%s)", state.ToString()));
[ # # ][ # # ]
1206 : 0 : return;
1207 : : }
1208 [ # # ]: 0 : }
1209 [ # # ]: 0 : } // End scope of ImportingNow
1210 : 0 : }
1211 : :
1212 : 0 : std::ostream& operator<<(std::ostream& os, const BlockfileType& type) {
1213 [ # # # ]: 0 : switch(type) {
1214 : 0 : case BlockfileType::NORMAL: os << "normal"; break;
1215 : 0 : case BlockfileType::ASSUMED: os << "assumed"; break;
1216 : 0 : default: os.setstate(std::ios_base::failbit);
1217 : 0 : }
1218 : 0 : return os;
1219 : : }
1220 : :
1221 : 0 : std::ostream& operator<<(std::ostream& os, const BlockfileCursor& cursor) {
1222 [ # # ]: 0 : os << strprintf("BlockfileCursor(file_num=%d, undo_height=%d)", cursor.file_num, cursor.undo_height);
1223 : 0 : return os;
1224 : 0 : }
1225 : : } // namespace node
|
