Line data Source code
1 : // Copyright (c) 2011-2021 The Bitcoin Core developers
2 : // Distributed under the MIT software license, see the accompanying
3 : // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 :
5 : #include <node/blockstorage.h>
6 :
7 : #include <chain.h>
8 : #include <chainparams.h>
9 : #include <clientversion.h>
10 : #include <consensus/validation.h>
11 : #include <flatfile.h>
12 : #include <fs.h>
13 : #include <hash.h>
14 : #include <pow.h>
15 : #include <shutdown.h>
16 : #include <streams.h>
17 : #include <undo.h>
18 : #include <util/system.h>
19 : #include <validation.h>
20 : #include <walletinitinterface.h>
21 :
22 : #include <map>
23 : #include <ranges>
24 : #include <unordered_map>
25 :
26 : namespace node {
27 : std::atomic_bool fImporting(false);
28 : std::atomic_bool fReindex(false);
29 : bool fPruneMode = false;
30 : uint64_t nPruneTarget = 0;
31 :
32 764669065 : bool CBlockIndexWorkComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const
33 : {
34 : // First sort by most total work, ...
35 764669065 : if (pa->nChainWork > pb->nChainWork) return false;
36 628957415 : if (pa->nChainWork < pb->nChainWork) return true;
37 :
38 : // ... then by earliest time received, ...
39 7792486 : if (pa->nSequenceId < pb->nSequenceId) return false;
40 7736116 : if (pa->nSequenceId > pb->nSequenceId) return true;
41 :
42 : // Use pointer address as tie breaker (should only happen with blocks
43 : // loaded from disk, as those all have id 0).
44 7718435 : if (pa < pb) return false;
45 7717800 : if (pa > pb) return true;
46 :
47 : // Identical blocks.
48 7716784 : return false;
49 764669065 : }
50 :
51 5781602 : bool CBlockIndexHeightOnlyComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const
52 : {
53 5781602 : return pa->nHeight < pb->nHeight;
54 : }
55 :
56 : static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false);
57 : static FlatFileSeq BlockFileSeq();
58 : static FlatFileSeq UndoFileSeq();
59 :
60 5984 : std::vector<CBlockIndex*> BlockManager::GetAllBlockIndices()
61 : {
62 5984 : AssertLockHeld(cs_main);
63 5984 : std::vector<CBlockIndex*> rv;
64 5984 : rv.reserve(m_block_index.size());
65 653908 : for (auto& [_, block_index] : m_block_index) {
66 647924 : rv.push_back(&block_index);
67 : }
68 5984 : return rv;
69 5984 : }
70 :
71 2534241 : CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash)
72 : {
73 2534241 : AssertLockHeld(cs_main);
74 2534241 : BlockMap::iterator it = m_block_index.find(hash);
75 2534241 : return it == m_block_index.end() ? nullptr : &it->second;
76 : }
77 :
78 259504 : const CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) const
79 : {
80 259504 : AssertLockHeld(cs_main);
81 259504 : BlockMap::const_iterator it = m_block_index.find(hash);
82 259504 : return it == m_block_index.end() ? nullptr : &it->second;
83 : }
84 :
85 270019 : CBlockIndex* BlockManager::AddToBlockIndex(const CBlockHeader& block, const uint256& hash, CBlockIndex*& best_header,
86 : enum BlockStatus nStatus)
87 : {
88 270019 : assert(!(nStatus & BLOCK_FAILED_MASK)); // no failed blocks allowed
89 270019 : AssertLockHeld(cs_main);
90 :
91 270019 : auto [mi, inserted] = m_block_index.try_emplace(hash, block);
92 270019 : if (!inserted) {
93 5 : return &mi->second;
94 : }
95 270014 : CBlockIndex* pindexNew = &(*mi).second;
96 :
97 : // We assign the sequence id to blocks only when the full data is available,
98 : // to avoid miners withholding blocks but broadcasting headers, to get a
99 : // competitive advantage.
100 270014 : pindexNew->nSequenceId = 0;
101 :
102 270014 : pindexNew->phashBlock = &((*mi).first);
103 270014 : BlockMap::iterator miPrev = m_block_index.find(block.hashPrevBlock);
104 270014 : if (miPrev != m_block_index.end()) {
105 268946 : pindexNew->pprev = &(*miPrev).second;
106 268946 : pindexNew->nHeight = pindexNew->pprev->nHeight + 1;
107 268946 : pindexNew->BuildSkip();
108 268946 : }
109 270014 : pindexNew->nTimeMax = (pindexNew->pprev ? std::max(pindexNew->pprev->nTimeMax, pindexNew->nTime) : pindexNew->nTime);
110 270014 : pindexNew->nChainWork = (pindexNew->pprev ? pindexNew->pprev->nChainWork : 0) + GetBlockProof(*pindexNew);
111 270014 : if (nStatus & BLOCK_VALID_MASK) {
112 270008 : pindexNew->RaiseValidity(nStatus);
113 270008 : if (best_header == nullptr || best_header->nChainWork < pindexNew->nChainWork) {
114 264898 : best_header = pindexNew;
115 264898 : }
116 270008 : } else {
117 6 : pindexNew->RaiseValidity(BLOCK_VALID_TREE); // required validity level
118 6 : pindexNew->nStatus |= nStatus;
119 : }
120 :
121 270014 : m_dirty_blockindex.insert(pindexNew);
122 :
123 : // track prevBlockHash -> pindex (multimap)
124 270014 : if (pindexNew->pprev) {
125 268946 : m_prev_block_index.emplace(pindexNew->pprev->GetBlockHash(), pindexNew);
126 268946 : }
127 :
128 270014 : return pindexNew;
129 270019 : }
130 :
131 3 : void BlockManager::PruneOneBlockFile(const int fileNumber)
132 : {
133 3 : AssertLockHeld(cs_main);
134 3 : LOCK(cs_LastBlockFile);
135 :
136 309 : for (auto& entry : m_block_index) {
137 306 : CBlockIndex* pindex = &entry.second;
138 306 : if (pindex->nFile == fileNumber) {
139 203 : pindex->nStatus &= ~BLOCK_HAVE_DATA;
140 203 : pindex->nStatus &= ~BLOCK_HAVE_UNDO;
141 203 : pindex->nFile = 0;
142 203 : pindex->nDataPos = 0;
143 203 : pindex->nUndoPos = 0;
144 203 : m_dirty_blockindex.insert(pindex);
145 :
146 : // Prune from m_blocks_unlinked -- any block we prune would have
147 : // to be downloaded again in order to consider its chain, at which
148 : // point it would be considered as a candidate for
149 : // m_blocks_unlinked or setBlockIndexCandidates.
150 203 : auto range = m_blocks_unlinked.equal_range(pindex->pprev);
151 203 : while (range.first != range.second) {
152 0 : std::multimap<CBlockIndex*, CBlockIndex*>::iterator _it = range.first;
153 0 : range.first++;
154 0 : if (_it->second == pindex) {
155 0 : m_blocks_unlinked.erase(_it);
156 0 : }
157 : }
158 203 : }
159 : }
160 :
161 3 : m_blockfile_info.at(fileNumber) = CBlockFileInfo{};
162 3 : m_dirty_fileinfo.insert(fileNumber);
163 3 : }
164 :
165 0 : void BlockManager::FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight, int chain_tip_height)
166 : {
167 0 : assert(fPruneMode && nManualPruneHeight > 0);
168 :
169 0 : LOCK2(cs_main, cs_LastBlockFile);
170 0 : if (chain_tip_height < 0) {
171 0 : return;
172 : }
173 :
174 : // last block to prune is the lesser of (user-specified height, MIN_BLOCKS_TO_KEEP from the tip)
175 0 : unsigned int nLastBlockWeCanPrune = std::min((unsigned)nManualPruneHeight, chain_tip_height - MIN_BLOCKS_TO_KEEP);
176 0 : int count = 0;
177 0 : for (int fileNumber = 0; fileNumber < m_last_blockfile; fileNumber++) {
178 0 : if (m_blockfile_info[fileNumber].nSize == 0 || m_blockfile_info[fileNumber].nHeightLast > nLastBlockWeCanPrune) {
179 0 : continue;
180 : }
181 0 : PruneOneBlockFile(fileNumber);
182 0 : setFilesToPrune.insert(fileNumber);
183 0 : count++;
184 0 : }
185 0 : LogPrintf("Prune (Manual): prune_height=%d removed %d blk/rev pairs\n", nLastBlockWeCanPrune, count);
186 0 : }
187 :
188 30 : void BlockManager::FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight, int chain_tip_height, int prune_height, bool is_ibd)
189 : {
190 30 : LOCK2(cs_main, cs_LastBlockFile);
191 30 : if (chain_tip_height < 0 || nPruneTarget == 0) {
192 10 : return;
193 : }
194 20 : if ((uint64_t)chain_tip_height <= nPruneAfterHeight) {
195 20 : return;
196 : }
197 :
198 0 : unsigned int nLastBlockWeCanPrune{(unsigned)std::min(prune_height, chain_tip_height - static_cast<int>(MIN_BLOCKS_TO_KEEP))};
199 0 : uint64_t nCurrentUsage = CalculateCurrentUsage();
200 : // We don't check to prune until after we've allocated new space for files
201 : // So we should leave a buffer under our target to account for another allocation
202 : // before the next pruning.
203 0 : uint64_t nBuffer = BLOCKFILE_CHUNK_SIZE + UNDOFILE_CHUNK_SIZE;
204 : uint64_t nBytesToPrune;
205 0 : int count = 0;
206 :
207 0 : if (nCurrentUsage + nBuffer >= nPruneTarget) {
208 : // On a prune event, the chainstate DB is flushed.
209 : // To avoid excessive prune events negating the benefit of high dbcache
210 : // values, we should not prune too rapidly.
211 : // So when pruning in IBD, increase the buffer a bit to avoid a re-prune too soon.
212 0 : if (is_ibd) {
213 : // Since this is only relevant during IBD, we use a fixed 10%
214 0 : nBuffer += nPruneTarget / 10;
215 0 : }
216 :
217 0 : for (int fileNumber = 0; fileNumber < m_last_blockfile; fileNumber++) {
218 0 : nBytesToPrune = m_blockfile_info[fileNumber].nSize + m_blockfile_info[fileNumber].nUndoSize;
219 :
220 0 : if (m_blockfile_info[fileNumber].nSize == 0) {
221 0 : continue;
222 : }
223 :
224 0 : if (nCurrentUsage + nBuffer < nPruneTarget) { // are we below our target?
225 0 : break;
226 : }
227 :
228 : // don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the main chain's tip but keep scanning
229 0 : if (m_blockfile_info[fileNumber].nHeightLast > nLastBlockWeCanPrune) {
230 0 : continue;
231 : }
232 :
233 0 : PruneOneBlockFile(fileNumber);
234 : // Queue up the files for removal
235 0 : setFilesToPrune.insert(fileNumber);
236 0 : nCurrentUsage -= nBytesToPrune;
237 0 : count++;
238 0 : }
239 0 : }
240 :
241 0 : LogPrint(BCLog::PRUNE, "target=%dMiB actual=%dMiB diff=%dMiB max_prune_height=%d removed %d blk/rev pairs\n",
242 : nPruneTarget/1024/1024, nCurrentUsage/1024/1024,
243 : ((int64_t)nPruneTarget - (int64_t)nCurrentUsage)/1024/1024,
244 : nLastBlockWeCanPrune, count);
245 30 : }
246 :
247 99399 : void BlockManager::UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) {
248 99399 : AssertLockHeld(::cs_main);
249 99399 : m_prune_locks[name] = lock_info;
250 99399 : }
251 :
252 648226 : CBlockIndex* BlockManager::InsertBlockIndex(const uint256& hash)
253 : {
254 648226 : AssertLockHeld(cs_main);
255 :
256 648226 : if (hash.IsNull()) {
257 1992 : return nullptr;
258 : }
259 :
260 1292468 : const auto [mi, inserted]{m_block_index.try_emplace(hash)};
261 646234 : CBlockIndex* pindex = &(*mi).second;
262 646234 : if (inserted) {
263 324161 : pindex->phashBlock = &((*mi).first);
264 324161 : }
265 646234 : return pindex;
266 648226 : }
267 :
268 2994 : bool BlockManager::LoadBlockIndex()
269 : {
270 651220 : if (!m_block_tree_db->LoadBlockIndexGuts(GetConsensus(), [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); })) {
271 0 : return false;
272 : }
273 :
274 327357 : for (auto& [_, block_index] : m_block_index) {
275 : // build m_blockman.m_prev_block_index
276 324363 : if (block_index.pprev) {
277 966963 : m_prev_block_index.emplace(block_index.pprev->GetBlockHash(), &block_index);
278 322321 : }
279 : }
280 :
281 : // Calculate nChainWork
282 2994 : std::vector<CBlockIndex*> vSortedByHeight{GetAllBlockIndices()};
283 2994 : std::sort(vSortedByHeight.begin(), vSortedByHeight.end(),
284 : CBlockIndexHeightOnlyComparator());
285 :
286 2994 : CBlockIndex* previous_index{nullptr};
287 327035 : for (CBlockIndex* pindex : vSortedByHeight) {
288 324043 : if (ShutdownRequested()) return false;
289 324043 : if (previous_index && pindex->nHeight > previous_index->nHeight + 1) {
290 2 : return error("%s: block index is non-contiguous, index of height %d missing", __func__, previous_index->nHeight + 1);
291 : }
292 324041 : previous_index = pindex;
293 324041 : pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex);
294 324041 : pindex->nTimeMax = (pindex->pprev ? std::max(pindex->pprev->nTimeMax, pindex->nTime) : pindex->nTime);
295 :
296 : // We can link the chain of blocks for which we've received transactions at some point, or
297 : // blocks that are assumed-valid on the basis of snapshot load (see
298 : // PopulateAndValidateSnapshot()).
299 : // Pruned nodes may have deleted the block.
300 324041 : if (pindex->nTx > 0) {
301 323461 : if (pindex->pprev) {
302 321467 : if (pindex->pprev->nChainTx > 0) {
303 321467 : pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx;
304 321467 : } else {
305 0 : pindex->nChainTx = 0;
306 0 : m_blocks_unlinked.insert(std::make_pair(pindex->pprev, pindex));
307 : }
308 321467 : } else {
309 1994 : pindex->nChainTx = pindex->nTx;
310 : }
311 323461 : }
312 324041 : if (!(pindex->nStatus & BLOCK_FAILED_MASK) && pindex->pprev && (pindex->pprev->nStatus & BLOCK_FAILED_MASK)) {
313 0 : pindex->nStatus |= BLOCK_FAILED_CHILD;
314 0 : m_dirty_blockindex.insert(pindex);
315 0 : }
316 324041 : if (pindex->pprev) {
317 321999 : pindex->BuildSkip();
318 321999 : }
319 : }
320 :
321 2992 : return true;
322 2994 : }
323 :
324 7520 : bool BlockManager::WriteBlockIndexDB()
325 : {
326 7520 : AssertLockHeld(::cs_main);
327 7520 : std::vector<std::pair<int, const CBlockFileInfo*>> vFiles;
328 7520 : vFiles.reserve(m_dirty_fileinfo.size());
329 11098 : for (std::set<int>::iterator it = m_dirty_fileinfo.begin(); it != m_dirty_fileinfo.end();) {
330 3578 : vFiles.emplace_back(*it, &m_blockfile_info[*it]);
331 3578 : m_dirty_fileinfo.erase(it++);
332 : }
333 7520 : std::vector<const CBlockIndex*> vBlocks;
334 7520 : vBlocks.reserve(m_dirty_blockindex.size());
335 255323 : for (std::set<CBlockIndex*>::iterator it = m_dirty_blockindex.begin(); it != m_dirty_blockindex.end();) {
336 247803 : vBlocks.push_back(*it);
337 247803 : m_dirty_blockindex.erase(it++);
338 : }
339 7520 : if (!m_block_tree_db->WriteBatchSync(vFiles, m_last_blockfile, vBlocks)) {
340 0 : return false;
341 : }
342 7520 : return true;
343 7520 : }
344 :
345 2994 : bool BlockManager::LoadBlockIndexDB()
346 : {
347 2994 : if (!LoadBlockIndex()) {
348 2 : return false;
349 : }
350 :
351 : // Load block file info
352 2992 : m_block_tree_db->ReadLastBlockFile(m_last_blockfile);
353 2992 : m_blockfile_info.resize(m_last_blockfile + 1);
354 2992 : LogPrintf("%s: last block file = %i\n", __func__, m_last_blockfile);
355 5984 : for (int nFile = 0; nFile <= m_last_blockfile; nFile++) {
356 2992 : m_block_tree_db->ReadBlockFileInfo(nFile, m_blockfile_info[nFile]);
357 2992 : }
358 2992 : LogPrintf("%s: last block file info: %s\n", __func__, m_blockfile_info[m_last_blockfile].ToString());
359 2992 : for (int nFile = m_last_blockfile + 1; true; nFile++) {
360 2992 : CBlockFileInfo info;
361 2992 : if (m_block_tree_db->ReadBlockFileInfo(nFile, info)) {
362 0 : m_blockfile_info.push_back(info);
363 0 : } else {
364 2992 : break;
365 : }
366 0 : }
367 :
368 : // Check presence of blk files
369 2992 : LogPrintf("Checking all blk files are present...\n");
370 2992 : std::set<int> setBlkDataFiles;
371 326955 : for (const auto& [_, block_index] : m_block_index) {
372 323963 : if (block_index.nStatus & BLOCK_HAVE_DATA) {
373 323411 : setBlkDataFiles.insert(block_index.nFile);
374 323411 : }
375 : }
376 4982 : for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) {
377 1992 : FlatFilePos pos(*it, 0);
378 1992 : if (CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION).IsNull()) {
379 2 : return false;
380 : }
381 1990 : }
382 :
383 : // Check whether we have ever pruned block & undo files
384 2990 : m_block_tree_db->ReadFlag("prunedblockfiles", m_have_pruned);
385 2990 : if (m_have_pruned) {
386 0 : LogPrintf("LoadBlockIndexDB(): Block files have previously been pruned\n");
387 0 : }
388 :
389 : // Check whether we need to continue reindexing
390 2990 : bool fReindexing = false;
391 2990 : m_block_tree_db->ReadReindexing(fReindexing);
392 2990 : if (fReindexing) fReindex = true;
393 :
394 : // Migrate old synchronous index data from block index database
395 : // to async indexes in separate databases
396 2990 : if (!m_block_tree_db->MigrateOldIndexData()) {
397 0 : return false;
398 : }
399 :
400 2990 : return true;
401 2994 : }
402 :
403 356608 : const CBlockIndex* BlockManager::GetLastCheckpoint(const CCheckpointData& data)
404 : {
405 356608 : const MapCheckpoints& checkpoints = data.mapCheckpoints;
406 :
407 371658 : for (const MapCheckpoints::value_type& i : checkpoints | std::views::reverse) {
408 371006 : const uint256& hash = i.second;
409 371006 : const CBlockIndex* pindex = LookupBlockIndex(hash);
410 371006 : if (pindex) {
411 355956 : return pindex;
412 : }
413 : }
414 652 : return nullptr;
415 356608 : }
416 :
417 5622 : bool BlockManager::IsBlockPruned(const CBlockIndex* pblockindex)
418 : {
419 5622 : AssertLockHeld(::cs_main);
420 5622 : return (m_have_pruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0);
421 : }
422 :
423 747 : const CBlockIndex* BlockManager::GetFirstStoredBlock(const CBlockIndex& start_block)
424 : {
425 747 : AssertLockHeld(::cs_main);
426 747 : const CBlockIndex* last_block = &start_block;
427 104045 : while (last_block->pprev && (last_block->pprev->nStatus & BLOCK_HAVE_DATA)) {
428 103298 : last_block = last_block->pprev;
429 : }
430 747 : return last_block;
431 : }
432 :
433 : // If we're using -prune with -reindex, then delete block files that will be ignored by the
434 : // reindex. Since reindexing works by starting at block file 0 and looping until a blockfile
435 : // is missing, do the same here to delete any later block files after a gap. Also delete all
436 : // rev files since they'll be rewritten by the reindex anyway. This ensures that m_blockfile_info
437 : // is in sync with what's actually on disk by the time we start downloading, so that pruning
438 : // works correctly.
439 0 : void CleanupBlockRevFiles()
440 : {
441 0 : std::map<std::string, fs::path> mapBlockFiles;
442 :
443 : // Glob all blk?????.dat and rev?????.dat files from the blocks directory.
444 : // Remove the rev files immediately and insert the blk file paths into an
445 : // ordered map keyed by block file index.
446 0 : LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n");
447 0 : const fs::path& blocksdir = gArgs.GetBlocksDirPath();
448 0 : for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) {
449 0 : const std::string path = fs::PathToString(it->path().filename());
450 0 : if (fs::is_regular_file(*it) &&
451 0 : path.length() == 12 &&
452 0 : path.substr(8,4) == ".dat")
453 : {
454 0 : if (path.substr(0, 3) == "blk") {
455 0 : mapBlockFiles[path.substr(3, 5)] = it->path();
456 0 : } else if (path.substr(0, 3) == "rev") {
457 0 : remove(it->path());
458 0 : }
459 0 : }
460 0 : }
461 :
462 : // Remove all block files that aren't part of a contiguous set starting at
463 : // zero by walking the ordered map (keys are block file indices) by
464 : // keeping a separate counter. Once we hit a gap (or if 0 doesn't exist)
465 : // start removing block files.
466 0 : int nContigCounter = 0;
467 0 : for (const std::pair<const std::string, fs::path>& item : mapBlockFiles) {
468 0 : if (LocaleIndependentAtoi<int>(item.first) == nContigCounter) {
469 0 : nContigCounter++;
470 0 : continue;
471 : }
472 0 : remove(item.second);
473 : }
474 0 : }
475 :
476 2 : CBlockFileInfo* BlockManager::GetBlockFileInfo(size_t n)
477 : {
478 2 : LOCK(cs_LastBlockFile);
479 :
480 2 : return &m_blockfile_info.at(n);
481 2 : }
482 :
483 246789 : static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart)
484 : {
485 : // Open history file to append
486 246789 : CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION);
487 246789 : if (fileout.IsNull()) {
488 0 : return error("%s: OpenUndoFile failed", __func__);
489 : }
490 :
491 : // Write index header
492 246789 : unsigned int nSize = GetSerializeSize(blockundo, fileout.GetVersion());
493 246789 : fileout << messageStart << nSize;
494 :
495 : // Write undo data
496 246789 : long fileOutPos = ftell(fileout.Get());
497 246789 : if (fileOutPos < 0) {
498 0 : return error("%s: ftell failed", __func__);
499 : }
500 246789 : pos.nPos = (unsigned int)fileOutPos;
501 246789 : fileout << blockundo;
502 :
503 : // calculate & write checksum
504 246789 : HashWriter hasher{};
505 246789 : hasher << hashBlock;
506 246789 : hasher << blockundo;
507 246789 : fileout << hasher.GetHash();
508 :
509 246789 : return true;
510 246789 : }
511 :
512 189390 : bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex)
513 : {
514 378780 : const FlatFilePos pos{WITH_LOCK(::cs_main, return pindex->GetUndoPos())};
515 :
516 189390 : if (pos.IsNull()) {
517 0 : return error("%s: no undo data available", __func__);
518 : }
519 :
520 : // Open history file to read
521 189390 : CAutoFile filein(OpenUndoFile(pos, true), SER_DISK, CLIENT_VERSION);
522 189390 : if (filein.IsNull()) {
523 18 : return error("%s: OpenUndoFile failed", __func__);
524 : }
525 :
526 : // Read block
527 189372 : uint256 hashChecksum;
528 189372 : CHashVerifier<CAutoFile> verifier(&filein); // We need a CHashVerifier as reserializing may lose data
529 : try {
530 189372 : verifier << pindex->pprev->GetBlockHash();
531 189371 : verifier >> blockundo;
532 189370 : filein >> hashChecksum;
533 189372 : } catch (const std::exception& e) {
534 2 : return error("%s: Deserialize or I/O error - %s", __func__, e.what());
535 2 : }
536 :
537 : // Verify checksum
538 189370 : if (hashChecksum != verifier.GetHash()) {
539 0 : return error("%s: Checksum mismatch", __func__);
540 : }
541 :
542 189370 : return true;
543 189392 : }
544 :
545 7548 : void BlockManager::FlushUndoFile(int block_file, bool finalize)
546 : {
547 7548 : FlatFilePos undo_pos_old(block_file, m_blockfile_info[block_file].nUndoSize);
548 7548 : if (!UndoFileSeq().Flush(undo_pos_old, finalize)) {
549 0 : AbortNode("Flushing undo file to disk failed. This is likely the result of an I/O error.");
550 0 : }
551 7548 : }
552 :
553 7548 : void BlockManager::FlushBlockFile(bool fFinalize, bool finalize_undo)
554 : {
555 7548 : LOCK(cs_LastBlockFile);
556 7548 : FlatFilePos block_pos_old(m_last_blockfile, m_blockfile_info[m_last_blockfile].nSize);
557 7548 : if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) {
558 0 : AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error.");
559 0 : }
560 : // we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks,
561 : // e.g. during IBD or a sync after a node going offline
562 7548 : if (!fFinalize || finalize_undo) FlushUndoFile(m_last_blockfile, finalize_undo);
563 7548 : }
564 :
565 15136 : uint64_t BlockManager::CalculateCurrentUsage()
566 : {
567 15136 : LOCK(cs_LastBlockFile);
568 :
569 15136 : uint64_t retval = 0;
570 30272 : for (const CBlockFileInfo& file : m_blockfile_info) {
571 15136 : retval += file.nSize + file.nUndoSize;
572 : }
573 15136 : return retval;
574 15136 : }
575 :
576 3 : void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune)
577 : {
578 6 : for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) {
579 3 : FlatFilePos pos(*it, 0);
580 3 : fs::remove(BlockFileSeq().FileName(pos));
581 3 : fs::remove(UndoFileSeq().FileName(pos));
582 3 : LogPrint(BCLog::BLOCKSTORE, "Prune: %s deleted blk/rev (%05u)\n", __func__, *it);
583 3 : }
584 3 : }
585 :
586 1226792 : static FlatFileSeq BlockFileSeq()
587 : {
588 2453584 : return FlatFileSeq(gArgs.GetBlocksDirPath(), "blk",
589 2453148 : gArgs.GetBoolArg("-fastprune", false) ? 0x4000 /* 16kb */ :
590 1226356 : (gArgs.GetBoolArg("-tinyblk", false) ? 0x10000 /* 64kb */ : BLOCKFILE_CHUNK_SIZE));
591 0 : }
592 :
593 690519 : static FlatFileSeq UndoFileSeq()
594 : {
595 690519 : return FlatFileSeq(gArgs.GetBlocksDirPath(), "rev", gArgs.GetBoolArg("-tinyblk", false) ? 0x10000 /* 64kb */ : UNDOFILE_CHUNK_SIZE);
596 0 : }
597 :
598 980630 : FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly)
599 : {
600 980630 : return BlockFileSeq().Open(pos, fReadOnly);
601 0 : }
602 :
603 : /** Open an undo file (rev?????.dat) */
604 436179 : static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly)
605 : {
606 436179 : return UndoFileSeq().Open(pos, fReadOnly);
607 0 : }
608 :
609 130 : fs::path GetBlockPosFilename(const FlatFilePos& pos)
610 : {
611 130 : return BlockFileSeq().FileName(pos);
612 0 : }
613 :
614 248577 : bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown)
615 : {
616 248577 : LOCK(cs_LastBlockFile);
617 :
618 248577 : unsigned int nFile = fKnown ? pos.nFile : m_last_blockfile;
619 248577 : if (m_blockfile_info.size() <= nFile) {
620 69 : m_blockfile_info.resize(nFile + 1);
621 69 : }
622 :
623 248577 : bool finalize_undo = false;
624 248577 : if (!fKnown) {
625 238481 : unsigned int max_blockfile_size{MAX_BLOCKFILE_SIZE};
626 : // Use smaller blockfiles in test-only -fastprune mode - but avoid
627 : // the possibility of having a block not fit into the block file.
628 238481 : if (gArgs.GetBoolArg("-fastprune", false)) {
629 206 : max_blockfile_size = 0x10000; // 64kiB
630 206 : if (nAddSize >= max_blockfile_size) {
631 : // dynamically adjust the blockfile size to be larger than the added size
632 2 : max_blockfile_size = nAddSize + 1;
633 2 : }
634 206 : }
635 238481 : assert(nAddSize < max_blockfile_size);
636 238509 : while (m_blockfile_info[nFile].nSize + nAddSize >= max_blockfile_size) {
637 : // when the undo file is keeping up with the block file, we want to flush it explicitly
638 : // when it is lagging behind (more blocks arrive than are being connected), we let the
639 : // undo block write case handle it
640 28 : finalize_undo = (m_blockfile_info[nFile].nHeightLast == (unsigned int)active_chain.Tip()->nHeight);
641 28 : nFile++;
642 28 : if (m_blockfile_info.size() <= nFile) {
643 28 : m_blockfile_info.resize(nFile + 1);
644 28 : }
645 : }
646 238481 : pos.nFile = nFile;
647 238481 : pos.nPos = m_blockfile_info[nFile].nSize;
648 238481 : }
649 :
650 248577 : if ((int)nFile != m_last_blockfile) {
651 28 : if (!fKnown) {
652 28 : LogPrint(BCLog::BLOCKSTORE, "Leaving block file %i: %s\n", m_last_blockfile, m_blockfile_info[m_last_blockfile].ToString());
653 28 : }
654 28 : FlushBlockFile(!fKnown, finalize_undo);
655 28 : m_last_blockfile = nFile;
656 28 : }
657 :
658 248577 : m_blockfile_info[nFile].AddBlock(nHeight, nTime);
659 248577 : if (fKnown) {
660 10096 : m_blockfile_info[nFile].nSize = std::max(pos.nPos + nAddSize, m_blockfile_info[nFile].nSize);
661 10096 : } else {
662 238481 : m_blockfile_info[nFile].nSize += nAddSize;
663 : }
664 :
665 248577 : if (!fKnown) {
666 : bool out_of_space;
667 238481 : size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space);
668 238481 : if (out_of_space) {
669 0 : return AbortNode("Disk space is too low!", _("Disk space is too low!"));
670 : }
671 238481 : if (bytes_allocated != 0 && fPruneMode) {
672 10 : m_check_for_pruning = true;
673 10 : }
674 238481 : }
675 :
676 248577 : m_dirty_fileinfo.insert(nFile);
677 248577 : return true;
678 248577 : }
679 :
680 246789 : bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize)
681 : {
682 246789 : pos.nFile = nFile;
683 :
684 246789 : LOCK(cs_LastBlockFile);
685 :
686 246789 : pos.nPos = m_blockfile_info[nFile].nUndoSize;
687 246789 : m_blockfile_info[nFile].nUndoSize += nAddSize;
688 246789 : m_dirty_fileinfo.insert(nFile);
689 :
690 : bool out_of_space;
691 246789 : size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space);
692 246789 : if (out_of_space) {
693 0 : return AbortNode(state, "Disk space is too low!", _("Disk space is too low!"));
694 : }
695 246789 : if (bytes_allocated != 0 && fPruneMode) {
696 4 : m_check_for_pruning = true;
697 4 : }
698 :
699 246789 : return true;
700 246789 : }
701 :
702 238481 : static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart)
703 : {
704 : // Open history file to append
705 238481 : CAutoFile fileout(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION);
706 238481 : if (fileout.IsNull()) {
707 0 : return error("WriteBlockToDisk: OpenBlockFile failed");
708 : }
709 :
710 : // Write index header
711 238481 : unsigned int nSize = GetSerializeSize(block, fileout.GetVersion());
712 238481 : fileout << messageStart << nSize;
713 :
714 : // Write block
715 238481 : long fileOutPos = ftell(fileout.Get());
716 238481 : if (fileOutPos < 0) {
717 0 : return error("WriteBlockToDisk: ftell failed");
718 : }
719 238481 : pos.nPos = (unsigned int)fileOutPos;
720 238481 : fileout << block;
721 :
722 238481 : return true;
723 238481 : }
724 :
725 252982 : bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block)
726 : {
727 252982 : AssertLockHeld(::cs_main);
728 : // Write undo information to disk
729 252982 : if (block.GetUndoPos().IsNull()) {
730 246789 : FlatFilePos _pos;
731 246789 : if (!FindUndoPos(state, block.nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) {
732 0 : return error("ConnectBlock(): FindUndoPos failed");
733 : }
734 246789 : if (!UndoWriteToDisk(blockundo, _pos, block.pprev->GetBlockHash(), GetParams().MessageStart())) {
735 0 : return AbortNode(state, "Failed to write undo data");
736 : }
737 : // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order)
738 : // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height
739 : // in the block file info as below; note that this does not catch the case where the undo writes are keeping up
740 : // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in
741 : // the FindBlockPos function
742 246789 : if (_pos.nFile < m_last_blockfile && static_cast<uint32_t>(block.nHeight) == m_blockfile_info[_pos.nFile].nHeightLast) {
743 0 : FlushUndoFile(_pos.nFile, true);
744 0 : }
745 :
746 : // update nUndoPos in block index
747 246789 : block.nUndoPos = _pos.nPos;
748 246789 : block.nStatus |= BLOCK_HAVE_UNDO;
749 246789 : m_dirty_blockindex.insert(&block);
750 246789 : }
751 :
752 252982 : return true;
753 252982 : }
754 :
755 712321 : std::optional<uint256> ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams)
756 : {
757 712321 : block.SetNull();
758 :
759 : // Open history file to read
760 712321 : CAutoFile filein(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION);
761 712321 : if (filein.IsNull()) {
762 135 : error("ReadBlockFromDisk: OpenBlockFile failed for %s", pos.ToString());
763 135 : return std::nullopt;
764 : }
765 :
766 : // Read block
767 : try {
768 712177 : filein >> block;
769 712177 : } catch (const std::exception& e) {
770 2 : error("%s: Deserialize or I/O error - %s at %s", __func__, e.what(), pos.ToString());
771 2 : return std::nullopt;
772 2 : }
773 :
774 : // Check the header
775 712175 : const uint256 hash{block.GetHash()};
776 712176 : if (!CheckProofOfWork(hash, block.nBits, consensusParams)) {
777 0 : error("ReadBlockFromDisk: Errors in block header at %s", pos.ToString());
778 0 : return std::nullopt;
779 : }
780 :
781 712174 : return hash;
782 712325 : }
783 :
784 712153 : bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams)
785 : {
786 1424306 : const FlatFilePos block_pos{WITH_LOCK(cs_main, return pindex->GetBlockPos())};
787 :
788 712153 : const auto hash{ReadBlockFromDisk(block, block_pos, consensusParams)};
789 712153 : if (!hash) {
790 137 : return false;
791 : }
792 712016 : if (*hash != pindex->GetBlockHash()) {
793 0 : return error("ReadBlockFromDisk(CBlock&, CBlockIndex*): GetHash() doesn't match index for %s at %s",
794 0 : pindex->ToString(), block_pos.ToString());
795 : }
796 712016 : return true;
797 712153 : }
798 :
799 248577 : FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const FlatFilePos* dbp)
800 : {
801 248577 : unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION);
802 248577 : FlatFilePos blockPos;
803 248577 : const auto position_known {dbp != nullptr};
804 248577 : if (position_known) {
805 10096 : blockPos = *dbp;
806 10096 : } else {
807 : // when known, blockPos.nPos points at the offset of the block data in the blk file. that already accounts for
808 : // the serialization header present in the file (the 4 magic message start bytes + the 4 length bytes = 8 bytes = BLOCK_SERIALIZATION_HEADER_SIZE).
809 : // we add BLOCK_SERIALIZATION_HEADER_SIZE only for new blocks since they will have the serialization header added when written to disk.
810 238481 : nBlockSize += static_cast<unsigned int>(BLOCK_SERIALIZATION_HEADER_SIZE);
811 : }
812 248577 : if (!FindBlockPos(blockPos, nBlockSize, nHeight, active_chain, block.GetBlockTime(), position_known)) {
813 0 : error("%s: FindBlockPos failed", __func__);
814 0 : return FlatFilePos();
815 : }
816 248577 : if (!position_known) {
817 238481 : if (!WriteBlockToDisk(block, blockPos, GetParams().MessageStart())) {
818 0 : AbortNode("Failed to write block");
819 0 : return FlatFilePos();
820 : }
821 238481 : }
822 248577 : return blockPos;
823 248577 : }
824 :
825 : struct CImportingNow {
826 5662 : CImportingNow()
827 2831 : {
828 2831 : assert(fImporting == false);
829 2831 : fImporting = true;
830 5662 : }
831 :
832 5662 : ~CImportingNow()
833 2831 : {
834 2831 : assert(fImporting == true);
835 2831 : fImporting = false;
836 5662 : }
837 : };
838 :
839 2831 : void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args)
840 : {
841 2831 : ScheduleBatchPriority();
842 :
843 : {
844 2831 : CImportingNow imp;
845 :
846 : // -reindex
847 2831 : if (fReindex) {
848 65 : int nFile = 0;
849 : // Map of disk positions for blocks with unknown parent (only used for reindex);
850 : // parent hash -> child disk position, multiple children can have the same parent.
851 65 : std::multimap<uint256, FlatFilePos> blocks_with_unknown_parent;
852 130 : while (true) {
853 130 : FlatFilePos pos(nFile, 0);
854 130 : if (!fs::exists(GetBlockPosFilename(pos))) {
855 65 : break; // No block files left to reindex
856 : }
857 65 : FILE* file = OpenBlockFile(pos, true);
858 65 : if (!file) {
859 0 : break; // This error is logged in OpenBlockFile
860 : }
861 65 : LogPrintf("Reindexing block file blk%05u.dat...\n", (unsigned int)nFile);
862 65 : chainman.ActiveChainstate().LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent);
863 65 : if (ShutdownRequested()) {
864 0 : LogPrintf("Shutdown requested. Exit %s\n", __func__);
865 0 : return;
866 : }
867 65 : nFile++;
868 : }
869 130 : WITH_LOCK(::cs_main, chainman.m_blockman.m_block_tree_db->WriteReindexing(false));
870 65 : fReindex = false;
871 65 : LogPrintf("Reindexing finished\n");
872 : // To avoid ending up in a situation without genesis block, re-try initializing (no-op if reindexing worked):
873 65 : chainman.ActiveChainstate().LoadGenesisBlock();
874 65 : }
875 :
876 : // -loadblock=
877 2833 : for (const fs::path& path : vImportFiles) {
878 2 : FILE *file = fsbridge::fopen(path, "rb");
879 2 : if (file) {
880 2 : LogPrintf("Importing blocks file %s...\n", fs::PathToString(path));
881 2 : chainman.ActiveChainstate().LoadExternalBlockFile(file);
882 2 : if (ShutdownRequested()) {
883 0 : LogPrintf("Shutdown requested. Exit %s\n", __func__);
884 0 : return;
885 : }
886 2 : } else {
887 0 : LogPrintf("Warning: Could not open blocks file %s\n", fs::PathToString(path));
888 : }
889 : }
890 :
891 : // scan for better chains in the block chain database, that are not yet connected in the active best chain
892 :
893 : // We can't hold cs_main during ActivateBestChain even though we're accessing
894 : // the chainman unique_ptrs since ABC requires us not to be holding cs_main, so retrieve
895 : // the relevant pointers before the ABC call.
896 8493 : for (CChainState* chainstate : WITH_LOCK(::cs_main, return chainman.GetAll())) {
897 2831 : BlockValidationState state;
898 2831 : if (!chainstate->ActivateBestChain(state, nullptr)) {
899 0 : LogPrintf("Failed to connect best block (%s)\n", state.ToString());
900 0 : StartShutdown();
901 0 : return;
902 : }
903 2831 : }
904 :
905 2831 : if (args.GetBoolArg("-stopafterblockimport", DEFAULT_STOPAFTERBLOCKIMPORT)) {
906 0 : LogPrintf("Stopping after block import\n");
907 0 : StartShutdown();
908 0 : return;
909 : }
910 2831 : } // End scope of CImportingNow
911 :
912 2831 : chainman.ActiveChainstate().LoadMempool(args);
913 2831 : }
914 : } // namespace node
|
