Line data Source code
1 : // Copyright (c) 2011-2021 The Bitcoin Core developers
2 : // Distributed under the MIT software license, see the accompanying
3 : // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 :
5 : #include <node/blockstorage.h>
6 :
7 : #include <chain.h>
8 : #include <chainparams.h>
9 : #include <clientversion.h>
10 : #include <consensus/validation.h>
11 : #include <flatfile.h>
12 : #include <fs.h>
13 : #include <hash.h>
14 : #include <pow.h>
15 : #include <shutdown.h>
16 : #include <streams.h>
17 : #include <undo.h>
18 : #include <util/system.h>
19 : #include <validation.h>
20 : #include <walletinitinterface.h>
21 :
22 : #include <map>
23 : #include <ranges>
24 : #include <unordered_map>
25 :
26 : namespace node {
27 : std::atomic_bool fImporting(false);
28 : std::atomic_bool fReindex(false);
29 : bool fPruneMode = false;
30 : uint64_t nPruneTarget = 0;
31 :
32 66404402 : bool CBlockIndexWorkComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const
33 : {
34 : // First sort by most total work, ...
35 66404402 : if (pa->nChainWork > pb->nChainWork) return false;
36 66187212 : if (pa->nChainWork < pb->nChainWork) return true;
37 :
38 : // ... then by earliest time received, ...
39 329256 : if (pa->nSequenceId < pb->nSequenceId) return false;
40 327543 : if (pa->nSequenceId > pb->nSequenceId) return true;
41 :
42 : // Use pointer address as tie breaker (should only happen with blocks
43 : // loaded from disk, as those all have id 0).
44 326615 : if (pa < pb) return false;
45 326615 : if (pa > pb) return true;
46 :
47 : // Identical blocks.
48 326615 : return false;
49 66404402 : }
50 :
51 2824 : bool CBlockIndexHeightOnlyComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const
52 : {
53 2824 : return pa->nHeight < pb->nHeight;
54 : }
55 :
56 : static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false);
57 : static FlatFileSeq BlockFileSeq();
58 : static FlatFileSeq UndoFileSeq();
59 :
60 360 : std::vector<CBlockIndex*> BlockManager::GetAllBlockIndices()
61 : {
62 360 : AssertLockHeld(cs_main);
63 360 : std::vector<CBlockIndex*> rv;
64 360 : rv.reserve(m_block_index.size());
65 764 : for (auto& [_, block_index] : m_block_index) {
66 404 : rv.push_back(&block_index);
67 : }
68 360 : return rv;
69 360 : }
70 :
71 59596 : CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash)
72 : {
73 59596 : AssertLockHeld(cs_main);
74 59596 : BlockMap::iterator it = m_block_index.find(hash);
75 59596 : return it == m_block_index.end() ? nullptr : &it->second;
76 : }
77 :
78 29055 : const CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) const
79 : {
80 29055 : AssertLockHeld(cs_main);
81 29055 : BlockMap::const_iterator it = m_block_index.find(hash);
82 29055 : return it == m_block_index.end() ? nullptr : &it->second;
83 : }
84 :
85 24717 : CBlockIndex* BlockManager::AddToBlockIndex(const CBlockHeader& block, const uint256& hash, CBlockIndex*& best_header,
86 : enum BlockStatus nStatus)
87 : {
88 24717 : assert(!(nStatus & BLOCK_FAILED_MASK)); // no failed blocks allowed
89 24717 : AssertLockHeld(cs_main);
90 :
91 24717 : auto [mi, inserted] = m_block_index.try_emplace(hash, block);
92 24717 : if (!inserted) {
93 3 : return &mi->second;
94 : }
95 24714 : CBlockIndex* pindexNew = &(*mi).second;
96 :
97 : // We assign the sequence id to blocks only when the full data is available,
98 : // to avoid miners withholding blocks but broadcasting headers, to get a
99 : // competitive advantage.
100 24714 : pindexNew->nSequenceId = 0;
101 :
102 24714 : pindexNew->phashBlock = &((*mi).first);
103 24714 : BlockMap::iterator miPrev = m_block_index.find(block.hashPrevBlock);
104 24714 : if (miPrev != m_block_index.end()) {
105 24533 : pindexNew->pprev = &(*miPrev).second;
106 24533 : pindexNew->nHeight = pindexNew->pprev->nHeight + 1;
107 24533 : pindexNew->BuildSkip();
108 24533 : }
109 24714 : pindexNew->nTimeMax = (pindexNew->pprev ? std::max(pindexNew->pprev->nTimeMax, pindexNew->nTime) : pindexNew->nTime);
110 24714 : pindexNew->nChainWork = (pindexNew->pprev ? pindexNew->pprev->nChainWork : 0) + GetBlockProof(*pindexNew);
111 24714 : if (nStatus & BLOCK_VALID_MASK) {
112 24714 : pindexNew->RaiseValidity(nStatus);
113 24714 : if (best_header == nullptr || best_header->nChainWork < pindexNew->nChainWork) {
114 24309 : best_header = pindexNew;
115 24309 : }
116 24714 : } else {
117 0 : pindexNew->RaiseValidity(BLOCK_VALID_TREE); // required validity level
118 0 : pindexNew->nStatus |= nStatus;
119 : }
120 :
121 24714 : m_dirty_blockindex.insert(pindexNew);
122 :
123 : // track prevBlockHash -> pindex (multimap)
124 24714 : if (pindexNew->pprev) {
125 24533 : m_prev_block_index.emplace(pindexNew->pprev->GetBlockHash(), pindexNew);
126 24533 : }
127 :
128 24714 : return pindexNew;
129 24717 : }
130 :
131 3 : void BlockManager::PruneOneBlockFile(const int fileNumber)
132 : {
133 3 : AssertLockHeld(cs_main);
134 3 : LOCK(cs_LastBlockFile);
135 :
136 309 : for (auto& entry : m_block_index) {
137 306 : CBlockIndex* pindex = &entry.second;
138 306 : if (pindex->nFile == fileNumber) {
139 203 : pindex->nStatus &= ~BLOCK_HAVE_DATA;
140 203 : pindex->nStatus &= ~BLOCK_HAVE_UNDO;
141 203 : pindex->nFile = 0;
142 203 : pindex->nDataPos = 0;
143 203 : pindex->nUndoPos = 0;
144 203 : m_dirty_blockindex.insert(pindex);
145 :
146 : // Prune from m_blocks_unlinked -- any block we prune would have
147 : // to be downloaded again in order to consider its chain, at which
148 : // point it would be considered as a candidate for
149 : // m_blocks_unlinked or setBlockIndexCandidates.
150 203 : auto range = m_blocks_unlinked.equal_range(pindex->pprev);
151 203 : while (range.first != range.second) {
152 0 : std::multimap<CBlockIndex*, CBlockIndex*>::iterator _it = range.first;
153 0 : range.first++;
154 0 : if (_it->second == pindex) {
155 0 : m_blocks_unlinked.erase(_it);
156 0 : }
157 : }
158 203 : }
159 : }
160 :
161 3 : m_blockfile_info.at(fileNumber) = CBlockFileInfo{};
162 3 : m_dirty_fileinfo.insert(fileNumber);
163 3 : }
164 :
165 0 : void BlockManager::FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight, int chain_tip_height)
166 : {
167 0 : assert(fPruneMode && nManualPruneHeight > 0);
168 :
169 0 : LOCK2(cs_main, cs_LastBlockFile);
170 0 : if (chain_tip_height < 0) {
171 0 : return;
172 : }
173 :
174 : // last block to prune is the lesser of (user-specified height, MIN_BLOCKS_TO_KEEP from the tip)
175 0 : unsigned int nLastBlockWeCanPrune = std::min((unsigned)nManualPruneHeight, chain_tip_height - MIN_BLOCKS_TO_KEEP);
176 0 : int count = 0;
177 0 : for (int fileNumber = 0; fileNumber < m_last_blockfile; fileNumber++) {
178 0 : if (m_blockfile_info[fileNumber].nSize == 0 || m_blockfile_info[fileNumber].nHeightLast > nLastBlockWeCanPrune) {
179 0 : continue;
180 : }
181 0 : PruneOneBlockFile(fileNumber);
182 0 : setFilesToPrune.insert(fileNumber);
183 0 : count++;
184 0 : }
185 0 : LogPrintf("Prune (Manual): prune_height=%d removed %d blk/rev pairs\n", nLastBlockWeCanPrune, count);
186 0 : }
187 :
188 0 : void BlockManager::FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight, int chain_tip_height, int prune_height, bool is_ibd)
189 : {
190 0 : LOCK2(cs_main, cs_LastBlockFile);
191 0 : if (chain_tip_height < 0 || nPruneTarget == 0) {
192 0 : return;
193 : }
194 0 : if ((uint64_t)chain_tip_height <= nPruneAfterHeight) {
195 0 : return;
196 : }
197 :
198 0 : unsigned int nLastBlockWeCanPrune{(unsigned)std::min(prune_height, chain_tip_height - static_cast<int>(MIN_BLOCKS_TO_KEEP))};
199 0 : uint64_t nCurrentUsage = CalculateCurrentUsage();
200 : // We don't check to prune until after we've allocated new space for files
201 : // So we should leave a buffer under our target to account for another allocation
202 : // before the next pruning.
203 0 : uint64_t nBuffer = BLOCKFILE_CHUNK_SIZE + UNDOFILE_CHUNK_SIZE;
204 : uint64_t nBytesToPrune;
205 0 : int count = 0;
206 :
207 0 : if (nCurrentUsage + nBuffer >= nPruneTarget) {
208 : // On a prune event, the chainstate DB is flushed.
209 : // To avoid excessive prune events negating the benefit of high dbcache
210 : // values, we should not prune too rapidly.
211 : // So when pruning in IBD, increase the buffer a bit to avoid a re-prune too soon.
212 0 : if (is_ibd) {
213 : // Since this is only relevant during IBD, we use a fixed 10%
214 0 : nBuffer += nPruneTarget / 10;
215 0 : }
216 :
217 0 : for (int fileNumber = 0; fileNumber < m_last_blockfile; fileNumber++) {
218 0 : nBytesToPrune = m_blockfile_info[fileNumber].nSize + m_blockfile_info[fileNumber].nUndoSize;
219 :
220 0 : if (m_blockfile_info[fileNumber].nSize == 0) {
221 0 : continue;
222 : }
223 :
224 0 : if (nCurrentUsage + nBuffer < nPruneTarget) { // are we below our target?
225 0 : break;
226 : }
227 :
228 : // don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the main chain's tip but keep scanning
229 0 : if (m_blockfile_info[fileNumber].nHeightLast > nLastBlockWeCanPrune) {
230 0 : continue;
231 : }
232 :
233 0 : PruneOneBlockFile(fileNumber);
234 : // Queue up the files for removal
235 0 : setFilesToPrune.insert(fileNumber);
236 0 : nCurrentUsage -= nBytesToPrune;
237 0 : count++;
238 0 : }
239 0 : }
240 :
241 0 : LogPrint(BCLog::PRUNE, "target=%dMiB actual=%dMiB diff=%dMiB max_prune_height=%d removed %d blk/rev pairs\n",
242 : nPruneTarget/1024/1024, nCurrentUsage/1024/1024,
243 : ((int64_t)nPruneTarget - (int64_t)nCurrentUsage)/1024/1024,
244 : nLastBlockWeCanPrune, count);
245 0 : }
246 :
247 23 : void BlockManager::UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) {
248 23 : AssertLockHeld(::cs_main);
249 23 : m_prune_locks[name] = lock_info;
250 23 : }
251 :
252 0 : CBlockIndex* BlockManager::InsertBlockIndex(const uint256& hash)
253 : {
254 0 : AssertLockHeld(cs_main);
255 :
256 0 : if (hash.IsNull()) {
257 0 : return nullptr;
258 : }
259 :
260 0 : const auto [mi, inserted]{m_block_index.try_emplace(hash)};
261 0 : CBlockIndex* pindex = &(*mi).second;
262 0 : if (inserted) {
263 0 : pindex->phashBlock = &((*mi).first);
264 0 : }
265 0 : return pindex;
266 0 : }
267 :
268 180 : bool BlockManager::LoadBlockIndex()
269 : {
270 180 : if (!m_block_tree_db->LoadBlockIndexGuts(GetConsensus(), [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); })) {
271 0 : return false;
272 : }
273 :
274 382 : for (auto& [_, block_index] : m_block_index) {
275 : // build m_blockman.m_prev_block_index
276 202 : if (block_index.pprev) {
277 600 : m_prev_block_index.emplace(block_index.pprev->GetBlockHash(), &block_index);
278 200 : }
279 : }
280 :
281 : // Calculate nChainWork
282 180 : std::vector<CBlockIndex*> vSortedByHeight{GetAllBlockIndices()};
283 180 : std::sort(vSortedByHeight.begin(), vSortedByHeight.end(),
284 : CBlockIndexHeightOnlyComparator());
285 :
286 180 : CBlockIndex* previous_index{nullptr};
287 382 : for (CBlockIndex* pindex : vSortedByHeight) {
288 202 : if (ShutdownRequested()) return false;
289 202 : if (previous_index && pindex->nHeight > previous_index->nHeight + 1) {
290 0 : return error("%s: block index is non-contiguous, index of height %d missing", __func__, previous_index->nHeight + 1);
291 : }
292 202 : previous_index = pindex;
293 202 : pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex);
294 202 : pindex->nTimeMax = (pindex->pprev ? std::max(pindex->pprev->nTimeMax, pindex->nTime) : pindex->nTime);
295 :
296 : // We can link the chain of blocks for which we've received transactions at some point, or
297 : // blocks that are assumed-valid on the basis of snapshot load (see
298 : // PopulateAndValidateSnapshot()).
299 : // Pruned nodes may have deleted the block.
300 202 : if (pindex->nTx > 0) {
301 202 : if (pindex->pprev) {
302 200 : if (pindex->pprev->nChainTx > 0) {
303 200 : pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx;
304 200 : } else {
305 0 : pindex->nChainTx = 0;
306 0 : m_blocks_unlinked.insert(std::make_pair(pindex->pprev, pindex));
307 : }
308 200 : } else {
309 2 : pindex->nChainTx = pindex->nTx;
310 : }
311 202 : }
312 202 : if (!(pindex->nStatus & BLOCK_FAILED_MASK) && pindex->pprev && (pindex->pprev->nStatus & BLOCK_FAILED_MASK)) {
313 0 : pindex->nStatus |= BLOCK_FAILED_CHILD;
314 0 : m_dirty_blockindex.insert(pindex);
315 0 : }
316 202 : if (pindex->pprev) {
317 200 : pindex->BuildSkip();
318 200 : }
319 : }
320 :
321 180 : return true;
322 180 : }
323 :
324 23 : bool BlockManager::WriteBlockIndexDB()
325 : {
326 23 : AssertLockHeld(::cs_main);
327 23 : std::vector<std::pair<int, const CBlockFileInfo*>> vFiles;
328 23 : vFiles.reserve(m_dirty_fileinfo.size());
329 29 : for (std::set<int>::iterator it = m_dirty_fileinfo.begin(); it != m_dirty_fileinfo.end();) {
330 6 : vFiles.emplace_back(*it, &m_blockfile_info[*it]);
331 6 : m_dirty_fileinfo.erase(it++);
332 : }
333 23 : std::vector<const CBlockIndex*> vBlocks;
334 23 : vBlocks.reserve(m_dirty_blockindex.size());
335 567 : for (std::set<CBlockIndex*>::iterator it = m_dirty_blockindex.begin(); it != m_dirty_blockindex.end();) {
336 544 : vBlocks.push_back(*it);
337 544 : m_dirty_blockindex.erase(it++);
338 : }
339 23 : if (!m_block_tree_db->WriteBatchSync(vFiles, m_last_blockfile, vBlocks)) {
340 0 : return false;
341 : }
342 23 : return true;
343 23 : }
344 :
345 180 : bool BlockManager::LoadBlockIndexDB()
346 : {
347 180 : if (!LoadBlockIndex()) {
348 0 : return false;
349 : }
350 :
351 : // Load block file info
352 180 : m_block_tree_db->ReadLastBlockFile(m_last_blockfile);
353 180 : m_blockfile_info.resize(m_last_blockfile + 1);
354 180 : LogPrintf("%s: last block file = %i\n", __func__, m_last_blockfile);
355 360 : for (int nFile = 0; nFile <= m_last_blockfile; nFile++) {
356 180 : m_block_tree_db->ReadBlockFileInfo(nFile, m_blockfile_info[nFile]);
357 180 : }
358 180 : LogPrintf("%s: last block file info: %s\n", __func__, m_blockfile_info[m_last_blockfile].ToString());
359 180 : for (int nFile = m_last_blockfile + 1; true; nFile++) {
360 180 : CBlockFileInfo info;
361 180 : if (m_block_tree_db->ReadBlockFileInfo(nFile, info)) {
362 0 : m_blockfile_info.push_back(info);
363 0 : } else {
364 180 : break;
365 : }
366 0 : }
367 :
368 : // Check presence of blk files
369 180 : LogPrintf("Checking all blk files are present...\n");
370 180 : std::set<int> setBlkDataFiles;
371 382 : for (const auto& [_, block_index] : m_block_index) {
372 202 : if (block_index.nStatus & BLOCK_HAVE_DATA) {
373 182 : setBlkDataFiles.insert(block_index.nFile);
374 182 : }
375 : }
376 182 : for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) {
377 2 : FlatFilePos pos(*it, 0);
378 2 : if (CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION).IsNull()) {
379 0 : return false;
380 : }
381 2 : }
382 :
383 : // Check whether we have ever pruned block & undo files
384 180 : m_block_tree_db->ReadFlag("prunedblockfiles", m_have_pruned);
385 180 : if (m_have_pruned) {
386 0 : LogPrintf("LoadBlockIndexDB(): Block files have previously been pruned\n");
387 0 : }
388 :
389 : // Check whether we need to continue reindexing
390 180 : bool fReindexing = false;
391 180 : m_block_tree_db->ReadReindexing(fReindexing);
392 180 : if (fReindexing) fReindex = true;
393 :
394 : // Migrate old synchronous index data from block index database
395 : // to async indexes in separate databases
396 180 : if (!m_block_tree_db->MigrateOldIndexData()) {
397 0 : return false;
398 : }
399 :
400 180 : return true;
401 180 : }
402 :
403 49050 : const CBlockIndex* BlockManager::GetLastCheckpoint(const CCheckpointData& data)
404 : {
405 49050 : const MapCheckpoints& checkpoints = data.mapCheckpoints;
406 :
407 54294 : for (const MapCheckpoints::value_type& i : checkpoints | std::views::reverse) {
408 54156 : const uint256& hash = i.second;
409 54156 : const CBlockIndex* pindex = LookupBlockIndex(hash);
410 54156 : if (pindex) {
411 48912 : return pindex;
412 : }
413 : }
414 138 : return nullptr;
415 49050 : }
416 :
417 0 : bool BlockManager::IsBlockPruned(const CBlockIndex* pblockindex)
418 : {
419 0 : AssertLockHeld(::cs_main);
420 0 : return (m_have_pruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0);
421 : }
422 :
423 72 : const CBlockIndex* BlockManager::GetFirstStoredBlock(const CBlockIndex& start_block)
424 : {
425 72 : AssertLockHeld(::cs_main);
426 72 : const CBlockIndex* last_block = &start_block;
427 11595 : while (last_block->pprev && (last_block->pprev->nStatus & BLOCK_HAVE_DATA)) {
428 11523 : last_block = last_block->pprev;
429 : }
430 72 : return last_block;
431 : }
432 :
433 : // If we're using -prune with -reindex, then delete block files that will be ignored by the
434 : // reindex. Since reindexing works by starting at block file 0 and looping until a blockfile
435 : // is missing, do the same here to delete any later block files after a gap. Also delete all
436 : // rev files since they'll be rewritten by the reindex anyway. This ensures that m_blockfile_info
437 : // is in sync with what's actually on disk by the time we start downloading, so that pruning
438 : // works correctly.
439 0 : void CleanupBlockRevFiles()
440 : {
441 0 : std::map<std::string, fs::path> mapBlockFiles;
442 :
443 : // Glob all blk?????.dat and rev?????.dat files from the blocks directory.
444 : // Remove the rev files immediately and insert the blk file paths into an
445 : // ordered map keyed by block file index.
446 0 : LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n");
447 0 : const fs::path& blocksdir = gArgs.GetBlocksDirPath();
448 0 : for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) {
449 0 : const std::string path = fs::PathToString(it->path().filename());
450 0 : if (fs::is_regular_file(*it) &&
451 0 : path.length() == 12 &&
452 0 : path.substr(8,4) == ".dat")
453 : {
454 0 : if (path.substr(0, 3) == "blk") {
455 0 : mapBlockFiles[path.substr(3, 5)] = it->path();
456 0 : } else if (path.substr(0, 3) == "rev") {
457 0 : remove(it->path());
458 0 : }
459 0 : }
460 0 : }
461 :
462 : // Remove all block files that aren't part of a contiguous set starting at
463 : // zero by walking the ordered map (keys are block file indices) by
464 : // keeping a separate counter. Once we hit a gap (or if 0 doesn't exist)
465 : // start removing block files.
466 0 : int nContigCounter = 0;
467 0 : for (const std::pair<const std::string, fs::path>& item : mapBlockFiles) {
468 0 : if (LocaleIndependentAtoi<int>(item.first) == nContigCounter) {
469 0 : nContigCounter++;
470 0 : continue;
471 : }
472 0 : remove(item.second);
473 : }
474 0 : }
475 :
476 2 : CBlockFileInfo* BlockManager::GetBlockFileInfo(size_t n)
477 : {
478 2 : LOCK(cs_LastBlockFile);
479 :
480 2 : return &m_blockfile_info.at(n);
481 2 : }
482 :
483 24493 : static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart)
484 : {
485 : // Open history file to append
486 24493 : CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION);
487 24493 : if (fileout.IsNull()) {
488 0 : return error("%s: OpenUndoFile failed", __func__);
489 : }
490 :
491 : // Write index header
492 24493 : unsigned int nSize = GetSerializeSize(blockundo, fileout.GetVersion());
493 24493 : fileout << messageStart << nSize;
494 :
495 : // Write undo data
496 24493 : long fileOutPos = ftell(fileout.Get());
497 24493 : if (fileOutPos < 0) {
498 0 : return error("%s: ftell failed", __func__);
499 : }
500 24493 : pos.nPos = (unsigned int)fileOutPos;
501 24493 : fileout << blockundo;
502 :
503 : // calculate & write checksum
504 24493 : HashWriter hasher{};
505 24493 : hasher << hashBlock;
506 24493 : hasher << blockundo;
507 24493 : fileout << hasher.GetHash();
508 :
509 24493 : return true;
510 24493 : }
511 :
512 815 : bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex)
513 : {
514 1630 : const FlatFilePos pos{WITH_LOCK(::cs_main, return pindex->GetUndoPos())};
515 :
516 815 : if (pos.IsNull()) {
517 0 : return error("%s: no undo data available", __func__);
518 : }
519 :
520 : // Open history file to read
521 815 : CAutoFile filein(OpenUndoFile(pos, true), SER_DISK, CLIENT_VERSION);
522 815 : if (filein.IsNull()) {
523 0 : return error("%s: OpenUndoFile failed", __func__);
524 : }
525 :
526 : // Read block
527 815 : uint256 hashChecksum;
528 815 : CHashVerifier<CAutoFile> verifier(&filein); // We need a CHashVerifier as reserializing may lose data
529 : try {
530 815 : verifier << pindex->pprev->GetBlockHash();
531 815 : verifier >> blockundo;
532 815 : filein >> hashChecksum;
533 815 : } catch (const std::exception& e) {
534 0 : return error("%s: Deserialize or I/O error - %s", __func__, e.what());
535 0 : }
536 :
537 : // Verify checksum
538 815 : if (hashChecksum != verifier.GetHash()) {
539 0 : return error("%s: Checksum mismatch", __func__);
540 : }
541 :
542 815 : return true;
543 815 : }
544 :
545 25 : void BlockManager::FlushUndoFile(int block_file, bool finalize)
546 : {
547 25 : FlatFilePos undo_pos_old(block_file, m_blockfile_info[block_file].nUndoSize);
548 25 : if (!UndoFileSeq().Flush(undo_pos_old, finalize)) {
549 0 : AbortNode("Flushing undo file to disk failed. This is likely the result of an I/O error.");
550 0 : }
551 25 : }
552 :
553 25 : void BlockManager::FlushBlockFile(bool fFinalize, bool finalize_undo)
554 : {
555 25 : LOCK(cs_LastBlockFile);
556 25 : FlatFilePos block_pos_old(m_last_blockfile, m_blockfile_info[m_last_blockfile].nSize);
557 25 : if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) {
558 0 : AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error.");
559 0 : }
560 : // we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks,
561 : // e.g. during IBD or a sync after a node going offline
562 25 : if (!fFinalize || finalize_undo) FlushUndoFile(m_last_blockfile, finalize_undo);
563 25 : }
564 :
565 0 : uint64_t BlockManager::CalculateCurrentUsage()
566 : {
567 0 : LOCK(cs_LastBlockFile);
568 :
569 0 : uint64_t retval = 0;
570 0 : for (const CBlockFileInfo& file : m_blockfile_info) {
571 0 : retval += file.nSize + file.nUndoSize;
572 : }
573 0 : return retval;
574 0 : }
575 :
576 3 : void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune)
577 : {
578 6 : for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) {
579 3 : FlatFilePos pos(*it, 0);
580 3 : fs::remove(BlockFileSeq().FileName(pos));
581 3 : fs::remove(UndoFileSeq().FileName(pos));
582 3 : LogPrint(BCLog::BLOCKSTORE, "Prune: %s deleted blk/rev (%05u)\n", __func__, *it);
583 3 : }
584 3 : }
585 :
586 94567 : static FlatFileSeq BlockFileSeq()
587 : {
588 189134 : return FlatFileSeq(gArgs.GetBlocksDirPath(), "blk",
589 189134 : gArgs.GetBoolArg("-fastprune", false) ? 0x4000 /* 16kb */ :
590 94567 : (gArgs.GetBoolArg("-tinyblk", false) ? 0x10000 /* 64kb */ : BLOCKFILE_CHUNK_SIZE));
591 0 : }
592 :
593 49829 : static FlatFileSeq UndoFileSeq()
594 : {
595 49829 : return FlatFileSeq(gArgs.GetBlocksDirPath(), "rev", gArgs.GetBoolArg("-tinyblk", false) ? 0x10000 /* 64kb */ : UNDOFILE_CHUNK_SIZE);
596 0 : }
597 :
598 69837 : FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly)
599 : {
600 69837 : return BlockFileSeq().Open(pos, fReadOnly);
601 0 : }
602 :
603 : /** Open an undo file (rev?????.dat) */
604 25308 : static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly)
605 : {
606 25308 : return UndoFileSeq().Open(pos, fReadOnly);
607 0 : }
608 :
609 0 : fs::path GetBlockPosFilename(const FlatFilePos& pos)
610 : {
611 0 : return BlockFileSeq().FileName(pos);
612 0 : }
613 :
614 24703 : bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown)
615 : {
616 24703 : LOCK(cs_LastBlockFile);
617 :
618 24703 : unsigned int nFile = fKnown ? pos.nFile : m_last_blockfile;
619 24703 : if (m_blockfile_info.size() <= nFile) {
620 4 : m_blockfile_info.resize(nFile + 1);
621 4 : }
622 :
623 24703 : bool finalize_undo = false;
624 24703 : if (!fKnown) {
625 24702 : unsigned int max_blockfile_size{MAX_BLOCKFILE_SIZE};
626 : // Use smaller blockfiles in test-only -fastprune mode - but avoid
627 : // the possibility of having a block not fit into the block file.
628 24702 : if (gArgs.GetBoolArg("-fastprune", false)) {
629 0 : max_blockfile_size = 0x10000; // 64kiB
630 0 : if (nAddSize >= max_blockfile_size) {
631 : // dynamically adjust the blockfile size to be larger than the added size
632 0 : max_blockfile_size = nAddSize + 1;
633 0 : }
634 0 : }
635 24702 : assert(nAddSize < max_blockfile_size);
636 24704 : while (m_blockfile_info[nFile].nSize + nAddSize >= max_blockfile_size) {
637 : // when the undo file is keeping up with the block file, we want to flush it explicitly
638 : // when it is lagging behind (more blocks arrive than are being connected), we let the
639 : // undo block write case handle it
640 2 : finalize_undo = (m_blockfile_info[nFile].nHeightLast == (unsigned int)active_chain.Tip()->nHeight);
641 2 : nFile++;
642 2 : if (m_blockfile_info.size() <= nFile) {
643 2 : m_blockfile_info.resize(nFile + 1);
644 2 : }
645 : }
646 24702 : pos.nFile = nFile;
647 24702 : pos.nPos = m_blockfile_info[nFile].nSize;
648 24702 : }
649 :
650 24703 : if ((int)nFile != m_last_blockfile) {
651 2 : if (!fKnown) {
652 2 : LogPrint(BCLog::BLOCKSTORE, "Leaving block file %i: %s\n", m_last_blockfile, m_blockfile_info[m_last_blockfile].ToString());
653 2 : }
654 2 : FlushBlockFile(!fKnown, finalize_undo);
655 2 : m_last_blockfile = nFile;
656 2 : }
657 :
658 24703 : m_blockfile_info[nFile].AddBlock(nHeight, nTime);
659 24703 : if (fKnown) {
660 1 : m_blockfile_info[nFile].nSize = std::max(pos.nPos + nAddSize, m_blockfile_info[nFile].nSize);
661 1 : } else {
662 24702 : m_blockfile_info[nFile].nSize += nAddSize;
663 : }
664 :
665 24703 : if (!fKnown) {
666 : bool out_of_space;
667 24702 : size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space);
668 24702 : if (out_of_space) {
669 0 : return AbortNode("Disk space is too low!", _("Disk space is too low!"));
670 : }
671 24702 : if (bytes_allocated != 0 && fPruneMode) {
672 0 : m_check_for_pruning = true;
673 0 : }
674 24702 : }
675 :
676 24703 : m_dirty_fileinfo.insert(nFile);
677 24703 : return true;
678 24703 : }
679 :
680 24493 : bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize)
681 : {
682 24493 : pos.nFile = nFile;
683 :
684 24493 : LOCK(cs_LastBlockFile);
685 :
686 24493 : pos.nPos = m_blockfile_info[nFile].nUndoSize;
687 24493 : m_blockfile_info[nFile].nUndoSize += nAddSize;
688 24493 : m_dirty_fileinfo.insert(nFile);
689 :
690 : bool out_of_space;
691 24493 : size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space);
692 24493 : if (out_of_space) {
693 0 : return AbortNode(state, "Disk space is too low!", _("Disk space is too low!"));
694 : }
695 24493 : if (bytes_allocated != 0 && fPruneMode) {
696 0 : m_check_for_pruning = true;
697 0 : }
698 :
699 24493 : return true;
700 24493 : }
701 :
702 24702 : static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart)
703 : {
704 : // Open history file to append
705 24702 : CAutoFile fileout(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION);
706 24702 : if (fileout.IsNull()) {
707 0 : return error("WriteBlockToDisk: OpenBlockFile failed");
708 : }
709 :
710 : // Write index header
711 24702 : unsigned int nSize = GetSerializeSize(block, fileout.GetVersion());
712 24702 : fileout << messageStart << nSize;
713 :
714 : // Write block
715 24702 : long fileOutPos = ftell(fileout.Get());
716 24702 : if (fileOutPos < 0) {
717 0 : return error("WriteBlockToDisk: ftell failed");
718 : }
719 24702 : pos.nPos = (unsigned int)fileOutPos;
720 24702 : fileout << block;
721 :
722 24702 : return true;
723 24702 : }
724 :
725 24497 : bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block)
726 : {
727 24497 : AssertLockHeld(::cs_main);
728 : // Write undo information to disk
729 24497 : if (block.GetUndoPos().IsNull()) {
730 24493 : FlatFilePos _pos;
731 24493 : if (!FindUndoPos(state, block.nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) {
732 0 : return error("ConnectBlock(): FindUndoPos failed");
733 : }
734 24493 : if (!UndoWriteToDisk(blockundo, _pos, block.pprev->GetBlockHash(), GetParams().MessageStart())) {
735 0 : return AbortNode(state, "Failed to write undo data");
736 : }
737 : // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order)
738 : // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height
739 : // in the block file info as below; note that this does not catch the case where the undo writes are keeping up
740 : // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in
741 : // the FindBlockPos function
742 24493 : if (_pos.nFile < m_last_blockfile && static_cast<uint32_t>(block.nHeight) == m_blockfile_info[_pos.nFile].nHeightLast) {
743 0 : FlushUndoFile(_pos.nFile, true);
744 0 : }
745 :
746 : // update nUndoPos in block index
747 24493 : block.nUndoPos = _pos.nPos;
748 24493 : block.nStatus |= BLOCK_HAVE_UNDO;
749 24493 : m_dirty_blockindex.insert(&block);
750 24493 : }
751 :
752 24497 : return true;
753 24497 : }
754 :
755 44843 : std::optional<uint256> ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams)
756 : {
757 44843 : block.SetNull();
758 :
759 : // Open history file to read
760 44843 : CAutoFile filein(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION);
761 44843 : if (filein.IsNull()) {
762 104 : error("ReadBlockFromDisk: OpenBlockFile failed for %s", pos.ToString());
763 104 : return std::nullopt;
764 : }
765 :
766 : // Read block
767 : try {
768 44739 : filein >> block;
769 44739 : } catch (const std::exception& e) {
770 0 : error("%s: Deserialize or I/O error - %s at %s", __func__, e.what(), pos.ToString());
771 0 : return std::nullopt;
772 0 : }
773 :
774 : // Check the header
775 44739 : const uint256 hash{block.GetHash()};
776 44739 : if (!CheckProofOfWork(hash, block.nBits, consensusParams)) {
777 0 : error("ReadBlockFromDisk: Errors in block header at %s", pos.ToString());
778 0 : return std::nullopt;
779 : }
780 :
781 44739 : return hash;
782 44843 : }
783 :
784 44729 : bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams)
785 : {
786 89458 : const FlatFilePos block_pos{WITH_LOCK(cs_main, return pindex->GetBlockPos())};
787 :
788 44729 : const auto hash{ReadBlockFromDisk(block, block_pos, consensusParams)};
789 44729 : if (!hash) {
790 104 : return false;
791 : }
792 44625 : if (*hash != pindex->GetBlockHash()) {
793 0 : return error("ReadBlockFromDisk(CBlock&, CBlockIndex*): GetHash() doesn't match index for %s at %s",
794 0 : pindex->ToString(), block_pos.ToString());
795 : }
796 44625 : return true;
797 44729 : }
798 :
799 24703 : FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const FlatFilePos* dbp)
800 : {
801 24703 : unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION);
802 24703 : FlatFilePos blockPos;
803 24703 : const auto position_known {dbp != nullptr};
804 24703 : if (position_known) {
805 1 : blockPos = *dbp;
806 1 : } else {
807 : // when known, blockPos.nPos points at the offset of the block data in the blk file. that already accounts for
808 : // the serialization header present in the file (the 4 magic message start bytes + the 4 length bytes = 8 bytes = BLOCK_SERIALIZATION_HEADER_SIZE).
809 : // we add BLOCK_SERIALIZATION_HEADER_SIZE only for new blocks since they will have the serialization header added when written to disk.
810 24702 : nBlockSize += static_cast<unsigned int>(BLOCK_SERIALIZATION_HEADER_SIZE);
811 : }
812 24703 : if (!FindBlockPos(blockPos, nBlockSize, nHeight, active_chain, block.GetBlockTime(), position_known)) {
813 0 : error("%s: FindBlockPos failed", __func__);
814 0 : return FlatFilePos();
815 : }
816 24703 : if (!position_known) {
817 24702 : if (!WriteBlockToDisk(block, blockPos, GetParams().MessageStart())) {
818 0 : AbortNode("Failed to write block");
819 0 : return FlatFilePos();
820 : }
821 24702 : }
822 24703 : return blockPos;
823 24703 : }
824 :
825 : struct CImportingNow {
826 0 : CImportingNow()
827 0 : {
828 0 : assert(fImporting == false);
829 0 : fImporting = true;
830 0 : }
831 :
832 0 : ~CImportingNow()
833 0 : {
834 0 : assert(fImporting == true);
835 0 : fImporting = false;
836 0 : }
837 : };
838 :
839 0 : void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args)
840 : {
841 0 : ScheduleBatchPriority();
842 :
843 : {
844 0 : CImportingNow imp;
845 :
846 : // -reindex
847 0 : if (fReindex) {
848 0 : int nFile = 0;
849 : // Map of disk positions for blocks with unknown parent (only used for reindex);
850 : // parent hash -> child disk position, multiple children can have the same parent.
851 0 : std::multimap<uint256, FlatFilePos> blocks_with_unknown_parent;
852 0 : while (true) {
853 0 : FlatFilePos pos(nFile, 0);
854 0 : if (!fs::exists(GetBlockPosFilename(pos))) {
855 0 : break; // No block files left to reindex
856 : }
857 0 : FILE* file = OpenBlockFile(pos, true);
858 0 : if (!file) {
859 0 : break; // This error is logged in OpenBlockFile
860 : }
861 0 : LogPrintf("Reindexing block file blk%05u.dat...\n", (unsigned int)nFile);
862 0 : chainman.ActiveChainstate().LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent);
863 0 : if (ShutdownRequested()) {
864 0 : LogPrintf("Shutdown requested. Exit %s\n", __func__);
865 0 : return;
866 : }
867 0 : nFile++;
868 : }
869 0 : WITH_LOCK(::cs_main, chainman.m_blockman.m_block_tree_db->WriteReindexing(false));
870 0 : fReindex = false;
871 0 : LogPrintf("Reindexing finished\n");
872 : // To avoid ending up in a situation without genesis block, re-try initializing (no-op if reindexing worked):
873 0 : chainman.ActiveChainstate().LoadGenesisBlock();
874 0 : }
875 :
876 : // -loadblock=
877 0 : for (const fs::path& path : vImportFiles) {
878 0 : FILE *file = fsbridge::fopen(path, "rb");
879 0 : if (file) {
880 0 : LogPrintf("Importing blocks file %s...\n", fs::PathToString(path));
881 0 : chainman.ActiveChainstate().LoadExternalBlockFile(file);
882 0 : if (ShutdownRequested()) {
883 0 : LogPrintf("Shutdown requested. Exit %s\n", __func__);
884 0 : return;
885 : }
886 0 : } else {
887 0 : LogPrintf("Warning: Could not open blocks file %s\n", fs::PathToString(path));
888 : }
889 : }
890 :
891 : // scan for better chains in the block chain database, that are not yet connected in the active best chain
892 :
893 : // We can't hold cs_main during ActivateBestChain even though we're accessing
894 : // the chainman unique_ptrs since ABC requires us not to be holding cs_main, so retrieve
895 : // the relevant pointers before the ABC call.
896 0 : for (CChainState* chainstate : WITH_LOCK(::cs_main, return chainman.GetAll())) {
897 0 : BlockValidationState state;
898 0 : if (!chainstate->ActivateBestChain(state, nullptr)) {
899 0 : LogPrintf("Failed to connect best block (%s)\n", state.ToString());
900 0 : StartShutdown();
901 0 : return;
902 : }
903 0 : }
904 :
905 0 : if (args.GetBoolArg("-stopafterblockimport", DEFAULT_STOPAFTERBLOCKIMPORT)) {
906 0 : LogPrintf("Stopping after block import\n");
907 0 : StartShutdown();
908 0 : return;
909 : }
910 0 : } // End scope of CImportingNow
911 :
912 0 : chainman.ActiveChainstate().LoadMempool(args);
913 0 : }
914 : } // namespace node
|
