Line data Source code
1 : // Copyright (c) 2009-2010 Satoshi Nakamoto
2 : // Copyright (c) 2009-2021 The Bitcoin Core developers
3 : // Distributed under the MIT software license, see the accompanying
4 : // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 :
6 : #include <policy/fees.h>
7 :
8 : #include <clientversion.h>
9 : #include <consensus/amount.h>
10 : #include <fs.h>
11 : #include <logging.h>
12 : #include <policy/feerate.h>
13 : #include <primitives/transaction.h>
14 : #include <random.h>
15 : #include <serialize.h>
16 : #include <streams.h>
17 : #include <sync.h>
18 : #include <tinyformat.h>
19 : #include <txmempool.h>
20 : #include <uint256.h>
21 : #include <util/serfloat.h>
22 : #include <util/system.h>
23 : #include <util/time.h>
24 :
25 : #include <algorithm>
26 : #include <cassert>
27 : #include <cmath>
28 : #include <cstddef>
29 : #include <cstdint>
30 : #include <exception>
31 : #include <stdexcept>
32 : #include <utility>
33 :
34 : static const char* FEE_ESTIMATES_FILENAME = "fee_estimates.dat";
35 :
36 : static constexpr double INF_FEERATE = 1e99;
37 :
38 762 : std::string StringForFeeEstimateHorizon(FeeEstimateHorizon horizon)
39 : {
40 762 : switch (horizon) {
41 150 : case FeeEstimateHorizon::SHORT_HALFLIFE: return "short";
42 306 : case FeeEstimateHorizon::MED_HALFLIFE: return "medium";
43 306 : case FeeEstimateHorizon::LONG_HALFLIFE: return "long";
44 : } // no default case, so the compiler can warn about missing cases
45 0 : assert(false);
46 762 : }
47 :
48 : namespace {
49 :
50 : struct EncodedDoubleFormatter
51 : {
52 90565052 : template<typename Stream> void Ser(Stream &s, double v)
53 : {
54 90565052 : s << EncodeDouble(v);
55 90565052 : }
56 :
57 43052470 : template<typename Stream> void Unser(Stream& s, double& v)
58 : {
59 : uint64_t encoded;
60 43052470 : s >> encoded;
61 43052470 : v = DecodeDouble(encoded);
62 43052470 : }
63 : };
64 :
65 : } // namespace
66 :
67 : /**
68 : * We will instantiate an instance of this class to track transactions that were
69 : * included in a block. We will lump transactions into a bucket according to their
70 : * approximate feerate and then track how long it took for those txs to be included in a block
71 : *
72 : * The tracking of unconfirmed (mempool) transactions is completely independent of the
73 : * historical tracking of transactions that have been confirmed in a block.
74 : */
75 : class TxConfirmStats
76 : {
77 : private:
78 : //Define the buckets we will group transactions into
79 : const std::vector<double>& buckets; // The upper-bound of the range for the bucket (inclusive)
80 : const std::map<double, unsigned int>& bucketMap; // Map of bucket upper-bound to index into all vectors by bucket
81 :
82 : // For each bucket X:
83 : // Count the total # of txs in each bucket
84 : // Track the historical moving average of this total over blocks
85 : std::vector<double> txCtAvg;
86 :
87 : // Count the total # of txs confirmed within Y blocks in each bucket
88 : // Track the historical moving average of these totals over blocks
89 : std::vector<std::vector<double>> confAvg; // confAvg[Y][X]
90 :
91 : // Track moving avg of txs which have been evicted from the mempool
92 : // after failing to be confirmed within Y blocks
93 : std::vector<std::vector<double>> failAvg; // failAvg[Y][X]
94 :
95 : // Sum the total feerate of all tx's in each bucket
96 : // Track the historical moving average of this total over blocks
97 : std::vector<double> m_feerate_avg;
98 :
99 : // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X
100 : // Combine the total value with the tx counts to calculate the avg feerate per bucket
101 :
102 : double decay;
103 :
104 : // Resolution (# of blocks) with which confirmations are tracked
105 : unsigned int scale;
106 :
107 : // Mempool counts of outstanding transactions
108 : // For each bucket X, track the number of transactions in the mempool
109 : // that are unconfirmed for each possible confirmation value Y
110 : std::vector<std::vector<int> > unconfTxs; //unconfTxs[Y][X]
111 : // transactions still unconfirmed after GetMaxConfirms for each bucket
112 : std::vector<int> oldUnconfTxs;
113 :
114 : void resizeInMemoryCounters(size_t newbuckets);
115 :
116 : public:
117 : /**
118 : * Create new TxConfirmStats. This is called by BlockPolicyEstimator's
119 : * constructor with default values.
120 : * @param defaultBuckets contains the upper limits for the bucket boundaries
121 : * @param maxPeriods max number of periods to track
122 : * @param decay how much to decay the historical moving average per block
123 : */
124 : TxConfirmStats(const std::vector<double>& defaultBuckets, const std::map<double, unsigned int>& defaultBucketMap,
125 : unsigned int maxPeriods, double decay, unsigned int scale);
126 :
127 : /** Roll the circular buffer for unconfirmed txs*/
128 : void ClearCurrent(unsigned int nBlockHeight);
129 :
130 : /**
131 : * Record a new transaction data point in the current block stats
132 : * @param blocksToConfirm the number of blocks it took this transaction to confirm
133 : * @param val the feerate of the transaction
134 : * @warning blocksToConfirm is 1-based and has to be >= 1
135 : */
136 : void Record(int blocksToConfirm, double val);
137 :
138 : /** Record a new transaction entering the mempool*/
139 : unsigned int NewTx(unsigned int nBlockHeight, double val);
140 :
141 : /** Remove a transaction from mempool tracking stats*/
142 : void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight,
143 : unsigned int bucketIndex, bool inBlock);
144 :
145 : /** Update our estimates by decaying our historical moving average and updating
146 : with the data gathered from the current block */
147 : void UpdateMovingAverages();
148 :
149 : /**
150 : * Calculate a feerate estimate. Find the lowest value bucket (or range of buckets
151 : * to make sure we have enough data points) whose transactions still have sufficient likelihood
152 : * of being confirmed within the target number of confirmations
153 : * @param confTarget target number of confirmations
154 : * @param sufficientTxVal required average number of transactions per block in a bucket range
155 : * @param minSuccess the success probability we require
156 : * @param nBlockHeight the current block height
157 : */
158 : double EstimateMedianVal(int confTarget, double sufficientTxVal,
159 : double minSuccess, unsigned int nBlockHeight,
160 : EstimationResult *result = nullptr) const;
161 :
162 : /** Return the max number of confirms we're tracking */
163 2754912152 : unsigned int GetMaxConfirms() const { return scale * confAvg.size(); }
164 :
165 : /** Write state of estimation data to a file*/
166 : void Write(AutoFile& fileout) const;
167 :
168 : /**
169 : * Read saved state of estimation data from a file and replace all internal data structures and
170 : * variables with this state.
171 : */
172 : void Read(AutoFile& filein, int nFileVersion, size_t numBuckets);
173 : };
174 :
175 :
176 53976 : TxConfirmStats::TxConfirmStats(const std::vector<double>& defaultBuckets,
177 : const std::map<double, unsigned int>& defaultBucketMap,
178 : unsigned int maxPeriods, double _decay, unsigned int _scale)
179 26988 : : buckets(defaultBuckets), bucketMap(defaultBucketMap), decay(_decay), scale(_scale)
180 13494 : {
181 26988 : assert(_scale != 0 && "_scale must be non-zero");
182 13494 : confAvg.resize(maxPeriods);
183 13494 : failAvg.resize(maxPeriods);
184 364338 : for (unsigned int i = 0; i < maxPeriods; i++) {
185 350844 : confAvg[i].resize(buckets.size());
186 350844 : failAvg[i].resize(buckets.size());
187 350844 : }
188 :
189 13494 : txCtAvg.resize(buckets.size());
190 13494 : m_feerate_avg.resize(buckets.size());
191 :
192 13494 : resizeInMemoryCounters(buckets.size());
193 26988 : }
194 :
195 17664 : void TxConfirmStats::resizeInMemoryCounters(size_t newbuckets) {
196 : // newbuckets must be passed in because the buckets referred to during Read have not been updated yet.
197 17664 : unconfTxs.resize(GetMaxConfirms());
198 6306048 : for (unsigned int i = 0; i < unconfTxs.size(); i++) {
199 6288384 : unconfTxs[i].resize(newbuckets);
200 6288384 : }
201 17664 : oldUnconfTxs.resize(newbuckets);
202 17664 : }
203 :
204 : // Roll the unconfirmed txs circular buffer
205 680874 : void TxConfirmStats::ClearCurrent(unsigned int nBlockHeight)
206 : {
207 130046934 : for (unsigned int j = 0; j < buckets.size(); j++) {
208 129366060 : oldUnconfTxs[j] += unconfTxs[nBlockHeight % unconfTxs.size()][j];
209 129366060 : unconfTxs[nBlockHeight%unconfTxs.size()][j] = 0;
210 129366060 : }
211 680874 : }
212 :
213 :
214 166524 : void TxConfirmStats::Record(int blocksToConfirm, double feerate)
215 : {
216 : // blocksToConfirm is 1-based
217 166524 : if (blocksToConfirm < 1)
218 0 : return;
219 166524 : int periodsToConfirm = (blocksToConfirm + scale - 1) / scale;
220 166524 : unsigned int bucketindex = bucketMap.lower_bound(feerate)->second;
221 4378751 : for (size_t i = periodsToConfirm; i <= confAvg.size(); i++) {
222 4212227 : confAvg[i - 1][bucketindex]++;
223 4212227 : }
224 166524 : txCtAvg[bucketindex]++;
225 166524 : m_feerate_avg[bucketindex] += feerate;
226 166524 : }
227 :
228 680874 : void TxConfirmStats::UpdateMovingAverages()
229 : {
230 680874 : assert(confAvg.size() == failAvg.size());
231 130046934 : for (unsigned int j = 0; j < buckets.size(); j++) {
232 3492883620 : for (unsigned int i = 0; i < confAvg.size(); i++) {
233 3363517560 : confAvg[i][j] *= decay;
234 3363517560 : failAvg[i][j] *= decay;
235 3363517560 : }
236 129366060 : m_feerate_avg[j] *= decay;
237 129366060 : txCtAvg[j] *= decay;
238 129366060 : }
239 680874 : }
240 :
241 : // returns -1 on error conditions
242 80112 : double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal,
243 : double successBreakPoint, unsigned int nBlockHeight,
244 : EstimationResult *result) const
245 : {
246 : // Counters for a bucket (or range of buckets)
247 80112 : double nConf = 0; // Number of tx's confirmed within the confTarget
248 80112 : double totalNum = 0; // Total number of tx's that were ever confirmed
249 80112 : int extraNum = 0; // Number of tx's still in mempool for confTarget or longer
250 80112 : double failNum = 0; // Number of tx's that were never confirmed but removed from the mempool after confTarget
251 80112 : const int periodTarget = (confTarget + scale - 1) / scale;
252 80112 : const int maxbucketindex = buckets.size() - 1;
253 :
254 : // We'll combine buckets until we have enough samples.
255 : // The near and far variables will define the range we've combined
256 : // The best variables are the last range we saw which still had a high
257 : // enough confirmation rate to count as success.
258 : // The cur variables are the current range we're counting.
259 80112 : unsigned int curNearBucket = maxbucketindex;
260 80112 : unsigned int bestNearBucket = maxbucketindex;
261 80112 : unsigned int curFarBucket = maxbucketindex;
262 80112 : unsigned int bestFarBucket = maxbucketindex;
263 :
264 : // We'll always group buckets into sets that meet sufficientTxVal --
265 : // this ensures that we're using consistent groups between different
266 : // confirmation targets.
267 80112 : double partialNum = 0;
268 :
269 80112 : bool foundAnswer = false;
270 80112 : unsigned int bins = unconfTxs.size();
271 80112 : bool newBucketRange = true;
272 80112 : bool passing = true;
273 80112 : EstimatorBucket passBucket;
274 80112 : EstimatorBucket failBucket;
275 :
276 : // Start counting from highest feerate transactions
277 15301392 : for (int bucket = maxbucketindex; bucket >= 0; --bucket) {
278 15221280 : if (newBucketRange) {
279 132406 : curNearBucket = bucket;
280 132406 : newBucketRange = false;
281 132406 : }
282 15221280 : curFarBucket = bucket;
283 15221280 : nConf += confAvg[periodTarget - 1][bucket];
284 15221280 : partialNum += txCtAvg[bucket];
285 15221280 : totalNum += txCtAvg[bucket];
286 15221280 : failNum += failAvg[periodTarget - 1][bucket];
287 2754377180 : for (unsigned int confct = confTarget; confct < GetMaxConfirms(); confct++)
288 2739155900 : extraNum += unconfTxs[(nBlockHeight - confct) % bins][bucket];
289 15221280 : extraNum += oldUnconfTxs[bucket];
290 : // If we have enough transaction data points in this range of buckets,
291 : // we can test for success
292 : // (Only count the confirmed data points, so that each confirmation count
293 : // will be looking at the same amount of data and same bucket breaks)
294 :
295 15221280 : if (partialNum < sufficientTxVal / (1 - decay)) {
296 : // the buckets we've added in this round aren't sufficient
297 : // so keep adding
298 15119749 : continue;
299 : } else {
300 101531 : partialNum = 0; // reset for the next range we'll add
301 :
302 101531 : double curPct = nConf / (totalNum + failNum + extraNum);
303 :
304 : // Check to see if we are no longer getting confirmed at the success rate
305 101531 : if (curPct < successBreakPoint) {
306 15318 : if (passing == true) {
307 : // First time we hit a failure record the failed bucket
308 1983 : unsigned int failMinBucket = std::min(curNearBucket, curFarBucket);
309 1983 : unsigned int failMaxBucket = std::max(curNearBucket, curFarBucket);
310 1983 : failBucket.start = failMinBucket ? buckets[failMinBucket - 1] : 0;
311 1983 : failBucket.end = buckets[failMaxBucket];
312 1983 : failBucket.withinTarget = nConf;
313 1983 : failBucket.totalConfirmed = totalNum;
314 1983 : failBucket.inMempool = extraNum;
315 1983 : failBucket.leftMempool = failNum;
316 1983 : passing = false;
317 1983 : }
318 15318 : continue;
319 : }
320 : // Otherwise update the cumulative stats, and the bucket variables
321 : // and reset the counters
322 : else {
323 86213 : failBucket = EstimatorBucket(); // Reset any failed bucket, currently passing
324 86213 : foundAnswer = true;
325 86213 : passing = true;
326 86213 : passBucket.withinTarget = nConf;
327 86213 : nConf = 0;
328 86213 : passBucket.totalConfirmed = totalNum;
329 86213 : totalNum = 0;
330 86213 : passBucket.inMempool = extraNum;
331 86213 : passBucket.leftMempool = failNum;
332 86213 : failNum = 0;
333 86213 : extraNum = 0;
334 86213 : bestNearBucket = curNearBucket;
335 86213 : bestFarBucket = curFarBucket;
336 86213 : newBucketRange = true;
337 : }
338 : }
339 86213 : }
340 :
341 80112 : double median = -1;
342 80112 : double txSum = 0;
343 :
344 : // Calculate the "average" feerate of the best bucket range that met success conditions
345 : // Find the bucket with the median transaction and then report the average feerate from that bucket
346 : // This is a compromise between finding the median which we can't since we don't save all tx's
347 : // and reporting the average which is less accurate
348 80112 : unsigned int minBucket = std::min(bestNearBucket, bestFarBucket);
349 80112 : unsigned int maxBucket = std::max(bestNearBucket, bestFarBucket);
350 6566806 : for (unsigned int j = minBucket; j <= maxBucket; j++) {
351 6486694 : txSum += txCtAvg[j];
352 6486694 : }
353 80112 : if (foundAnswer && txSum != 0) {
354 36159 : txSum = txSum / 2;
355 38190 : for (unsigned int j = minBucket; j <= maxBucket; j++) {
356 38190 : if (txCtAvg[j] < txSum)
357 2031 : txSum -= txCtAvg[j];
358 : else { // we're in the right bucket
359 36159 : median = m_feerate_avg[j] / txCtAvg[j];
360 36159 : break;
361 : }
362 2031 : }
363 :
364 36159 : passBucket.start = minBucket ? buckets[minBucket-1] : 0;
365 36159 : passBucket.end = buckets[maxBucket];
366 36159 : }
367 :
368 : // If we were passing until we reached last few buckets with insufficient data, then report those as failed
369 80112 : if (passing && !newBucketRange) {
370 44476 : unsigned int failMinBucket = std::min(curNearBucket, curFarBucket);
371 44476 : unsigned int failMaxBucket = std::max(curNearBucket, curFarBucket);
372 44476 : failBucket.start = failMinBucket ? buckets[failMinBucket - 1] : 0;
373 44476 : failBucket.end = buckets[failMaxBucket];
374 44476 : failBucket.withinTarget = nConf;
375 44476 : failBucket.totalConfirmed = totalNum;
376 44476 : failBucket.inMempool = extraNum;
377 44476 : failBucket.leftMempool = failNum;
378 44476 : }
379 :
380 80112 : float passed_within_target_perc = 0.0;
381 80112 : float failed_within_target_perc = 0.0;
382 80112 : if ((passBucket.totalConfirmed + passBucket.inMempool + passBucket.leftMempool)) {
383 36159 : passed_within_target_perc = 100 * passBucket.withinTarget / (passBucket.totalConfirmed + passBucket.inMempool + passBucket.leftMempool);
384 36159 : }
385 80112 : if ((failBucket.totalConfirmed + failBucket.inMempool + failBucket.leftMempool)) {
386 46047 : failed_within_target_perc = 100 * failBucket.withinTarget / (failBucket.totalConfirmed + failBucket.inMempool + failBucket.leftMempool);
387 46047 : }
388 :
389 80112 : LogPrint(BCLog::ESTIMATEFEE, "FeeEst: %d > %.0f%% decay %.5f: feerate: %g from (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out) Fail: (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out)\n",
390 : confTarget, 100.0 * successBreakPoint, decay,
391 : median, passBucket.start, passBucket.end,
392 : passed_within_target_perc,
393 : passBucket.withinTarget, passBucket.totalConfirmed, passBucket.inMempool, passBucket.leftMempool,
394 : failBucket.start, failBucket.end,
395 : failed_within_target_perc,
396 : failBucket.withinTarget, failBucket.totalConfirmed, failBucket.inMempool, failBucket.leftMempool);
397 :
398 :
399 80112 : if (result) {
400 80024 : result->pass = passBucket;
401 80024 : result->fail = failBucket;
402 80024 : result->decay = decay;
403 80024 : result->scale = scale;
404 80024 : }
405 80112 : return median;
406 0 : }
407 :
408 8772 : void TxConfirmStats::Write(AutoFile& fileout) const
409 : {
410 8772 : fileout << Using<EncodedDoubleFormatter>(decay);
411 8772 : fileout << scale;
412 8772 : fileout << Using<VectorFormatter<EncodedDoubleFormatter>>(m_feerate_avg);
413 8772 : fileout << Using<VectorFormatter<EncodedDoubleFormatter>>(txCtAvg);
414 8772 : fileout << Using<VectorFormatter<VectorFormatter<EncodedDoubleFormatter>>>(confAvg);
415 8772 : fileout << Using<VectorFormatter<VectorFormatter<EncodedDoubleFormatter>>>(failAvg);
416 8772 : }
417 :
418 4170 : void TxConfirmStats::Read(AutoFile& filein, int nFileVersion, size_t numBuckets)
419 : {
420 : // Read data file and do some very basic sanity checking
421 : // buckets and bucketMap are not updated yet, so don't access them
422 : // If there is a read failure, we'll just discard this entire object anyway
423 : size_t maxConfirms, maxPeriods;
424 :
425 : // The current version will store the decay with each individual TxConfirmStats and also keep a scale factor
426 4170 : filein >> Using<EncodedDoubleFormatter>(decay);
427 4170 : if (decay <= 0 || decay >= 1) {
428 0 : throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)");
429 : }
430 4170 : filein >> scale;
431 4170 : if (scale == 0) {
432 0 : throw std::runtime_error("Corrupt estimates file. Scale must be non-zero");
433 : }
434 :
435 4170 : filein >> Using<VectorFormatter<EncodedDoubleFormatter>>(m_feerate_avg);
436 4170 : if (m_feerate_avg.size() != numBuckets) {
437 0 : throw std::runtime_error("Corrupt estimates file. Mismatch in feerate average bucket count");
438 : }
439 4170 : filein >> Using<VectorFormatter<EncodedDoubleFormatter>>(txCtAvg);
440 4170 : if (txCtAvg.size() != numBuckets) {
441 0 : throw std::runtime_error("Corrupt estimates file. Mismatch in tx count bucket count");
442 : }
443 4170 : filein >> Using<VectorFormatter<VectorFormatter<EncodedDoubleFormatter>>>(confAvg);
444 4170 : maxPeriods = confAvg.size();
445 4170 : maxConfirms = scale * maxPeriods;
446 :
447 4170 : if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) { // one week
448 0 : throw std::runtime_error("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms");
449 : }
450 112590 : for (unsigned int i = 0; i < maxPeriods; i++) {
451 108420 : if (confAvg[i].size() != numBuckets) {
452 0 : throw std::runtime_error("Corrupt estimates file. Mismatch in feerate conf average bucket count");
453 : }
454 108420 : }
455 :
456 4170 : filein >> Using<VectorFormatter<VectorFormatter<EncodedDoubleFormatter>>>(failAvg);
457 4170 : if (maxPeriods != failAvg.size()) {
458 0 : throw std::runtime_error("Corrupt estimates file. Mismatch in confirms tracked for failures");
459 : }
460 112590 : for (unsigned int i = 0; i < maxPeriods; i++) {
461 108420 : if (failAvg[i].size() != numBuckets) {
462 0 : throw std::runtime_error("Corrupt estimates file. Mismatch in one of failure average bucket counts");
463 : }
464 108420 : }
465 :
466 : // Resize the current block variables which aren't stored in the data file
467 : // to match the number of confirms and buckets
468 4170 : resizeInMemoryCounters(numBuckets);
469 :
470 4170 : LogPrint(BCLog::ESTIMATEFEE, "Reading estimates: %u buckets counting confirms up to %u blocks\n",
471 : numBuckets, maxConfirms);
472 4170 : }
473 :
474 170442 : unsigned int TxConfirmStats::NewTx(unsigned int nBlockHeight, double val)
475 : {
476 170442 : unsigned int bucketindex = bucketMap.lower_bound(val)->second;
477 170442 : unsigned int blockIndex = nBlockHeight % unconfTxs.size();
478 170442 : unconfTxs[blockIndex][bucketindex]++;
479 170442 : return bucketindex;
480 : }
481 :
482 170427 : void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketindex, bool inBlock)
483 : {
484 : //nBestSeenHeight is not updated yet for the new block
485 170427 : int blocksAgo = nBestSeenHeight - entryHeight;
486 170427 : if (nBestSeenHeight == 0) // the BlockPolicyEstimator hasn't seen any blocks yet
487 0 : blocksAgo = 0;
488 170427 : if (blocksAgo < 0) {
489 0 : LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error, blocks ago is negative for mempool tx\n");
490 0 : return; //This can't happen because we call this with our best seen height, no entries can have higher
491 : }
492 :
493 170427 : if (blocksAgo >= (int)unconfTxs.size()) {
494 2944 : if (oldUnconfTxs[bucketindex] > 0) {
495 2944 : oldUnconfTxs[bucketindex]--;
496 2944 : } else {
497 0 : LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error, mempool tx removed from >25 blocks,bucketIndex=%u already\n",
498 : bucketindex);
499 : }
500 2944 : }
501 : else {
502 167483 : unsigned int blockIndex = entryHeight % unconfTxs.size();
503 167483 : if (unconfTxs[blockIndex][bucketindex] > 0) {
504 167483 : unconfTxs[blockIndex][bucketindex]--;
505 167483 : } else {
506 0 : LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error, mempool tx removed from blockIndex=%u,bucketIndex=%u already\n",
507 : blockIndex, bucketindex);
508 : }
509 : }
510 170427 : if (!inBlock && (unsigned int)blocksAgo >= scale) { // Only counts as a failure if not confirmed for entire period
511 421 : assert(scale != 0);
512 421 : unsigned int periodsAgo = blocksAgo / scale;
513 2701 : for (size_t i = 0; i < periodsAgo && i < failAvg.size(); i++) {
514 2280 : failAvg[i][bucketindex]++;
515 2280 : }
516 421 : }
517 170427 : }
518 :
519 : // This function is called from CTxMemPool::removeUnchecked to ensure
520 : // txs removed from the mempool for any reason are no longer
521 : // tracked. Txs that were part of a block have already been removed in
522 : // processBlockTx to ensure they are never double tracked, but it is
523 : // of no harm to try to remove them again.
524 59424 : bool CBlockPolicyEstimator::removeTx(uint256 hash, bool inBlock)
525 : {
526 59424 : LOCK(m_cs_fee_estimator);
527 59424 : return _removeTx(hash, inBlock);
528 59424 : }
529 :
530 119210 : bool CBlockPolicyEstimator::_removeTx(const uint256& hash, bool inBlock)
531 : {
532 119210 : AssertLockHeld(m_cs_fee_estimator);
533 119210 : std::map<uint256, TxStatsInfo>::iterator pos = mapMemPoolTxs.find(hash);
534 119210 : if (pos != mapMemPoolTxs.end()) {
535 56809 : feeStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock);
536 56809 : shortStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock);
537 56809 : longStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock);
538 56809 : mapMemPoolTxs.erase(hash);
539 56809 : return true;
540 : } else {
541 62401 : return false;
542 : }
543 119210 : }
544 :
545 6216 : CBlockPolicyEstimator::CBlockPolicyEstimator()
546 3108 : {
547 : static_assert(MIN_BUCKET_FEERATE > 0, "Min feerate must be nonzero");
548 : size_t bucketIndex = 0;
549 :
550 : for (double bucketBoundary = MIN_BUCKET_FEERATE; bucketBoundary <= MAX_BUCKET_FEERATE; bucketBoundary *= FEE_SPACING, bucketIndex++) {
551 : buckets.push_back(bucketBoundary);
552 : bucketMap[bucketBoundary] = bucketIndex;
553 : }
554 : buckets.push_back(INF_FEERATE);
555 : bucketMap[INF_FEERATE] = bucketIndex;
556 : assert(bucketMap.size() == buckets.size());
557 :
558 : feeStats = std::make_unique<TxConfirmStats>(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE);
559 : shortStats = std::make_unique<TxConfirmStats>(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE);
560 : longStats = std::make_unique<TxConfirmStats>(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE);
561 :
562 : // If the fee estimation file is present, read recorded estimations
563 : fs::path est_filepath = gArgs.GetDataDirNet() / FEE_ESTIMATES_FILENAME;
564 : AutoFile est_file{fsbridge::fopen(est_filepath, "rb")};
565 : if (est_file.IsNull() || !Read(est_file)) {
566 : LogPrintf("Failed to read fee estimates from %s. Continue anyway.\n", fs::PathToString(est_filepath));
567 : }
568 3108 : }
569 :
570 6216 : CBlockPolicyEstimator::~CBlockPolicyEstimator() = default;
571 :
572 63706 : void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate)
573 : {
574 63706 : LOCK(m_cs_fee_estimator);
575 63706 : unsigned int txHeight = entry.GetHeight();
576 63706 : uint256 hash = entry.GetTx().GetHash();
577 63706 : if (mapMemPoolTxs.count(hash)) {
578 2 : LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error mempool tx %s already being tracked\n", hash.ToString());
579 2 : return;
580 : }
581 :
582 63704 : if (txHeight != nBestSeenHeight) {
583 : // Ignore side chains and re-orgs; assuming they are random they don't
584 : // affect the estimate. We'll potentially double count transactions in 1-block reorgs.
585 : // Ignore txs if BlockPolicyEstimator is not in sync with ActiveChain().Tip().
586 : // It will be synced next time a block is processed.
587 3131 : return;
588 : }
589 :
590 : // Only want to be updating estimates when our blockchain is synced,
591 : // otherwise we'll miscalculate how many blocks its taking to get included.
592 60573 : if (!validFeeEstimate) {
593 3759 : untrackedTxs++;
594 3759 : return;
595 : }
596 56814 : trackedTxs++;
597 :
598 : // Feerates are stored and reported as BTC-per-kb:
599 56814 : CFeeRate feeRate(entry.GetFee(), entry.GetTxSize());
600 :
601 56814 : mapMemPoolTxs[hash].blockHeight = txHeight;
602 56814 : unsigned int bucketIndex = feeStats->NewTx(txHeight, (double)feeRate.GetFeePerK());
603 56814 : mapMemPoolTxs[hash].bucketIndex = bucketIndex;
604 56814 : unsigned int bucketIndex2 = shortStats->NewTx(txHeight, (double)feeRate.GetFeePerK());
605 56814 : assert(bucketIndex == bucketIndex2);
606 56814 : unsigned int bucketIndex3 = longStats->NewTx(txHeight, (double)feeRate.GetFeePerK());
607 56814 : assert(bucketIndex == bucketIndex3);
608 63706 : }
609 :
610 58596 : bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry)
611 : {
612 58596 : AssertLockHeld(m_cs_fee_estimator);
613 58596 : if (!_removeTx(entry->GetTx().GetHash(), true)) {
614 : // This transaction wasn't being tracked for fee estimation
615 3088 : return false;
616 : }
617 :
618 : // How many blocks did it take for miners to include this transaction?
619 : // blocksToConfirm is 1-based, so a transaction included in the earliest
620 : // possible block has confirmation count of 1
621 55508 : int blocksToConfirm = nBlockHeight - entry->GetHeight();
622 55508 : if (blocksToConfirm <= 0) {
623 : // This can't happen because we don't process transactions from a block with a height
624 : // lower than our greatest seen height
625 0 : LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error Transaction had negative blocksToConfirm\n");
626 0 : return false;
627 : }
628 :
629 : // Feerates are stored and reported as BTC-per-kb:
630 55508 : CFeeRate feeRate(entry->GetFee(), entry->GetTxSize());
631 :
632 55508 : feeStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK());
633 55508 : shortStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK());
634 55508 : longStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK());
635 55508 : return true;
636 58596 : }
637 :
638 253840 : void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight,
639 : std::vector<const CTxMemPoolEntry*>& entries)
640 : {
641 253840 : LOCK(m_cs_fee_estimator);
642 253840 : if (nBlockHeight <= nBestSeenHeight) {
643 : // Ignore side chains and re-orgs; assuming they are random
644 : // they don't affect the estimate.
645 : // And if an attacker can re-org the chain at will, then
646 : // you've got much bigger problems than "attacker can influence
647 : // transaction fees."
648 26882 : return;
649 : }
650 :
651 : // Must update nBestSeenHeight in sync with ClearCurrent so that
652 : // calls to removeTx (via processBlockTx) correctly calculate age
653 : // of unconfirmed txs to remove from tracking.
654 226958 : nBestSeenHeight = nBlockHeight;
655 :
656 : // Update unconfirmed circular buffer
657 226958 : feeStats->ClearCurrent(nBlockHeight);
658 226958 : shortStats->ClearCurrent(nBlockHeight);
659 226958 : longStats->ClearCurrent(nBlockHeight);
660 :
661 : // Decay all exponential averages
662 226958 : feeStats->UpdateMovingAverages();
663 226958 : shortStats->UpdateMovingAverages();
664 226958 : longStats->UpdateMovingAverages();
665 :
666 226958 : unsigned int countedTxs = 0;
667 : // Update averages with data points from current block
668 285554 : for (const auto& entry : entries) {
669 58596 : if (processBlockTx(nBlockHeight, entry))
670 55508 : countedTxs++;
671 : }
672 :
673 226958 : if (firstRecordedHeight == 0 && countedTxs > 0) {
674 815 : firstRecordedHeight = nBestSeenHeight;
675 815 : LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy first recorded height %u\n", firstRecordedHeight);
676 815 : }
677 :
678 :
679 226958 : LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy estimates updated by %u of %u block txs, since last block %u of %u tracked, mempool map size %u, max target %u from %s\n",
680 : countedTxs, entries.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size(),
681 : MaxUsableEstimate(), HistoricalBlockSpan() > BlockSpan() ? "historical" : "current");
682 :
683 226958 : trackedTxs = 0;
684 226958 : untrackedTxs = 0;
685 253840 : }
686 :
687 94 : CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) const
688 : {
689 : // It's not possible to get reasonable estimates for confTarget of 1
690 94 : if (confTarget <= 1)
691 6 : return CFeeRate(0);
692 :
693 88 : return estimateRawFee(confTarget, DOUBLE_SUCCESS_PCT, FeeEstimateHorizon::MED_HALFLIFE);
694 94 : }
695 :
696 850 : CFeeRate CBlockPolicyEstimator::estimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult* result) const
697 : {
698 850 : TxConfirmStats* stats = nullptr;
699 850 : double sufficientTxs = SUFFICIENT_FEETXS;
700 850 : switch (horizon) {
701 : case FeeEstimateHorizon::SHORT_HALFLIFE: {
702 150 : stats = shortStats.get();
703 150 : sufficientTxs = SUFFICIENT_TXS_SHORT;
704 150 : break;
705 : }
706 : case FeeEstimateHorizon::MED_HALFLIFE: {
707 394 : stats = feeStats.get();
708 394 : break;
709 : }
710 : case FeeEstimateHorizon::LONG_HALFLIFE: {
711 306 : stats = longStats.get();
712 306 : break;
713 : }
714 : } // no default case, so the compiler can warn about missing cases
715 850 : assert(stats);
716 :
717 850 : LOCK(m_cs_fee_estimator);
718 : // Return failure if trying to analyze a target we're not tracking
719 850 : if (confTarget <= 0 || (unsigned int)confTarget > stats->GetMaxConfirms())
720 0 : return CFeeRate(0);
721 850 : if (successThreshold > 1)
722 0 : return CFeeRate(0);
723 :
724 850 : double median = stats->EstimateMedianVal(confTarget, sufficientTxs, successThreshold, nBestSeenHeight, result);
725 :
726 850 : if (median < 0)
727 34 : return CFeeRate(0);
728 :
729 816 : return CFeeRate(llround(median));
730 850 : }
731 :
732 16034 : unsigned int CBlockPolicyEstimator::HighestTargetTracked(FeeEstimateHorizon horizon) const
733 : {
734 16034 : LOCK(m_cs_fee_estimator);
735 16034 : switch (horizon) {
736 : case FeeEstimateHorizon::SHORT_HALFLIFE: {
737 306 : return shortStats->GetMaxConfirms();
738 : }
739 : case FeeEstimateHorizon::MED_HALFLIFE: {
740 306 : return feeStats->GetMaxConfirms();
741 : }
742 : case FeeEstimateHorizon::LONG_HALFLIFE: {
743 15422 : return longStats->GetMaxConfirms();
744 : }
745 : } // no default case, so the compiler can warn about missing cases
746 0 : assert(false);
747 16034 : }
748 :
749 483191 : unsigned int CBlockPolicyEstimator::BlockSpan() const
750 : {
751 483191 : if (firstRecordedHeight == 0) return 0;
752 120147 : assert(nBestSeenHeight >= firstRecordedHeight);
753 :
754 120147 : return nBestSeenHeight - firstRecordedHeight;
755 483191 : }
756 :
757 483191 : unsigned int CBlockPolicyEstimator::HistoricalBlockSpan() const
758 : {
759 483191 : if (historicalFirst == 0) return 0;
760 27857 : assert(historicalBest >= historicalFirst);
761 :
762 27857 : if (nBestSeenHeight - historicalBest > OLDEST_ESTIMATE_HISTORY) return 0;
763 :
764 27857 : return historicalBest - historicalFirst;
765 483191 : }
766 :
767 253309 : unsigned int CBlockPolicyEstimator::MaxUsableEstimate() const
768 : {
769 : // Block spans are divided by 2 to make sure there are enough potential failing data points for the estimate
770 253309 : return std::min(longStats->GetMaxConfirms(), std::max(BlockSpan(), HistoricalBlockSpan()) / 2);
771 : }
772 :
773 : /** Return a fee estimate at the required successThreshold from the shortest
774 : * time horizon which tracks confirmations up to the desired target. If
775 : * checkShorterHorizon is requested, also allow short time horizon estimates
776 : * for a lower target to reduce the given answer */
777 49422 : double CBlockPolicyEstimator::estimateCombinedFee(unsigned int confTarget, double successThreshold, bool checkShorterHorizon, EstimationResult *result) const
778 : {
779 49422 : double estimate = -1;
780 49422 : if (confTarget >= 1 && confTarget <= longStats->GetMaxConfirms()) {
781 : // Find estimate from shortest time horizon possible
782 49422 : if (confTarget <= shortStats->GetMaxConfirms()) { // short horizon
783 42974 : estimate = shortStats->EstimateMedianVal(confTarget, SUFFICIENT_TXS_SHORT, successThreshold, nBestSeenHeight, result);
784 42974 : }
785 6448 : else if (confTarget <= feeStats->GetMaxConfirms()) { // medium horizon
786 3968 : estimate = feeStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, successThreshold, nBestSeenHeight, result);
787 3968 : }
788 : else { // long horizon
789 2480 : estimate = longStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, successThreshold, nBestSeenHeight, result);
790 : }
791 49422 : if (checkShorterHorizon) {
792 41610 : EstimationResult tempResult;
793 : // If a lower confTarget from a more recent horizon returns a lower answer use it.
794 41610 : if (confTarget > feeStats->GetMaxConfirms()) {
795 2480 : double medMax = feeStats->EstimateMedianVal(feeStats->GetMaxConfirms(), SUFFICIENT_FEETXS, successThreshold, nBestSeenHeight, &tempResult);
796 2480 : if (medMax > 0 && (estimate == -1 || medMax < estimate)) {
797 1066 : estimate = medMax;
798 1066 : if (result) *result = tempResult;
799 1066 : }
800 2480 : }
801 41610 : if (confTarget > shortStats->GetMaxConfirms()) {
802 6258 : double shortMax = shortStats->EstimateMedianVal(shortStats->GetMaxConfirms(), SUFFICIENT_TXS_SHORT, successThreshold, nBestSeenHeight, &tempResult);
803 6258 : if (shortMax > 0 && (estimate == -1 || shortMax < estimate)) {
804 878 : estimate = shortMax;
805 878 : if (result) *result = tempResult;
806 878 : }
807 6258 : }
808 41610 : }
809 49422 : }
810 49422 : return estimate;
811 : }
812 :
813 : /** Ensure that for a conservative estimate, the DOUBLE_SUCCESS_PCT is also met
814 : * at 2 * target for any longer time horizons.
815 : */
816 11757 : double CBlockPolicyEstimator::estimateConservativeFee(unsigned int doubleTarget, EstimationResult *result) const
817 : {
818 11757 : double estimate = -1;
819 11757 : EstimationResult tempResult;
820 11757 : if (doubleTarget <= shortStats->GetMaxConfirms()) {
821 10119 : estimate = feeStats->EstimateMedianVal(doubleTarget, SUFFICIENT_FEETXS, DOUBLE_SUCCESS_PCT, nBestSeenHeight, result);
822 10119 : }
823 11757 : if (doubleTarget <= feeStats->GetMaxConfirms()) {
824 10983 : double longEstimate = longStats->EstimateMedianVal(doubleTarget, SUFFICIENT_FEETXS, DOUBLE_SUCCESS_PCT, nBestSeenHeight, &tempResult);
825 10983 : if (longEstimate > estimate) {
826 652 : estimate = longEstimate;
827 652 : if (result) *result = tempResult;
828 652 : }
829 10983 : }
830 11757 : return estimate;
831 : }
832 :
833 : /** estimateSmartFee returns the max of the feerates calculated with a 60%
834 : * threshold required at target / 2, an 85% threshold required at target and a
835 : * 95% threshold required at 2 * target. Each calculation is performed at the
836 : * shortest time horizon which tracks the required target. Conservative
837 : * estimates, however, required the 95% threshold at 2 * target be met for any
838 : * longer time horizons also.
839 : */
840 26351 : CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, FeeCalculation *feeCalc, bool conservative) const
841 : {
842 26351 : LOCK(m_cs_fee_estimator);
843 :
844 26351 : if (feeCalc) {
845 12058 : feeCalc->desiredTarget = confTarget;
846 12058 : feeCalc->returnedTarget = confTarget;
847 12058 : }
848 :
849 26351 : double median = -1;
850 26351 : EstimationResult tempResult;
851 :
852 : // Return failure if trying to analyze a target we're not tracking
853 26351 : if (confTarget <= 0 || (unsigned int)confTarget > longStats->GetMaxConfirms()) {
854 0 : return CFeeRate(0); // error condition
855 : }
856 :
857 : // It's not possible to get reasonable estimates for confTarget of 1
858 26351 : if (confTarget == 1) confTarget = 2;
859 :
860 26351 : unsigned int maxUsableEstimate = MaxUsableEstimate();
861 26351 : if ((unsigned int)confTarget > maxUsableEstimate) {
862 23344 : confTarget = maxUsableEstimate;
863 23344 : }
864 26351 : if (feeCalc) feeCalc->returnedTarget = confTarget;
865 :
866 26351 : if (confTarget <= 1) return CFeeRate(0); // error condition
867 :
868 16474 : assert(confTarget > 0); //estimateCombinedFee and estimateConservativeFee take unsigned ints
869 : /** true is passed to estimateCombined fee for target/2 and target so
870 : * that we check the max confirms for shorter time horizons as well.
871 : * This is necessary to preserve monotonically increasing estimates.
872 : * For non-conservative estimates we do the same thing for 2*target, but
873 : * for conservative estimates we want to skip these shorter horizons
874 : * checks for 2*target because we are taking the max over all time
875 : * horizons so we already have monotonically increasing estimates and
876 : * the purpose of conservative estimates is not to let short term
877 : * fluctuations lower our estimates by too much.
878 : */
879 16474 : double halfEst = estimateCombinedFee(confTarget/2, HALF_SUCCESS_PCT, true, &tempResult);
880 16474 : if (feeCalc) {
881 7812 : feeCalc->est = tempResult;
882 7812 : feeCalc->reason = FeeReason::HALF_ESTIMATE;
883 7812 : }
884 16474 : median = halfEst;
885 16474 : double actualEst = estimateCombinedFee(confTarget, SUCCESS_PCT, true, &tempResult);
886 16474 : if (actualEst > median) {
887 109 : median = actualEst;
888 109 : if (feeCalc) {
889 109 : feeCalc->est = tempResult;
890 109 : feeCalc->reason = FeeReason::FULL_ESTIMATE;
891 109 : }
892 109 : }
893 16474 : double doubleEst = estimateCombinedFee(2 * confTarget, DOUBLE_SUCCESS_PCT, !conservative, &tempResult);
894 16474 : if (doubleEst > median) {
895 0 : median = doubleEst;
896 0 : if (feeCalc) {
897 0 : feeCalc->est = tempResult;
898 0 : feeCalc->reason = FeeReason::DOUBLE_ESTIMATE;
899 0 : }
900 0 : }
901 :
902 16474 : if (conservative || median == -1) {
903 11757 : double consEst = estimateConservativeFee(2 * confTarget, &tempResult);
904 11757 : if (consEst > median) {
905 1099 : median = consEst;
906 1099 : if (feeCalc) {
907 1099 : feeCalc->est = tempResult;
908 1099 : feeCalc->reason = FeeReason::CONSERVATIVE;
909 1099 : }
910 1099 : }
911 11757 : }
912 :
913 16474 : if (median < 0) return CFeeRate(0); // error condition
914 :
915 9406 : return CFeeRate(llround(median));
916 26351 : }
917 :
918 2924 : void CBlockPolicyEstimator::Flush() {
919 2924 : FlushUnconfirmed();
920 :
921 2924 : fs::path est_filepath = gArgs.GetDataDirNet() / FEE_ESTIMATES_FILENAME;
922 2924 : AutoFile est_file{fsbridge::fopen(est_filepath, "wb")};
923 2924 : if (est_file.IsNull() || !Write(est_file)) {
924 0 : LogPrintf("Failed to write fee estimates to %s. Continue anyway.\n", fs::PathToString(est_filepath));
925 0 : }
926 2924 : }
927 :
928 2924 : bool CBlockPolicyEstimator::Write(AutoFile& fileout) const
929 : {
930 : try {
931 2924 : LOCK(m_cs_fee_estimator);
932 2924 : fileout << 140100; // version required to read: 0.14.1 or later
933 2924 : fileout << CLIENT_VERSION; // version that wrote the file
934 2924 : fileout << nBestSeenHeight;
935 2924 : if (BlockSpan() > HistoricalBlockSpan()/2) {
936 660 : fileout << firstRecordedHeight << nBestSeenHeight;
937 660 : }
938 : else {
939 2264 : fileout << historicalFirst << historicalBest;
940 : }
941 2924 : fileout << Using<VectorFormatter<EncodedDoubleFormatter>>(buckets);
942 2924 : feeStats->Write(fileout);
943 2924 : shortStats->Write(fileout);
944 2924 : longStats->Write(fileout);
945 2924 : }
946 : catch (const std::exception&) {
947 0 : LogPrintf("CBlockPolicyEstimator::Write(): unable to write policy estimator data (non-fatal)\n");
948 0 : return false;
949 0 : }
950 2924 : return true;
951 2924 : }
952 :
953 1390 : bool CBlockPolicyEstimator::Read(AutoFile& filein)
954 : {
955 : try {
956 1390 : LOCK(m_cs_fee_estimator);
957 : int nVersionRequired, nVersionThatWrote;
958 : unsigned int nFileBestSeenHeight;
959 1390 : filein >> nVersionRequired >> nVersionThatWrote;
960 1390 : if (nVersionRequired > CLIENT_VERSION) {
961 0 : throw std::runtime_error(strprintf("up-version (%d) fee estimate file", nVersionRequired));
962 : }
963 :
964 : // Read fee estimates file into temporary variables so existing data
965 : // structures aren't corrupted if there is an exception.
966 1390 : filein >> nFileBestSeenHeight;
967 :
968 1390 : if (nVersionRequired < 140100) {
969 0 : LogPrintf("%s: incompatible old fee estimation data (non-fatal). Version: %d\n", __func__, nVersionRequired);
970 0 : } else { // New format introduced in 140100
971 : unsigned int nFileHistoricalFirst, nFileHistoricalBest;
972 1390 : filein >> nFileHistoricalFirst >> nFileHistoricalBest;
973 1390 : if (nFileHistoricalFirst > nFileHistoricalBest || nFileHistoricalBest > nFileBestSeenHeight) {
974 0 : throw std::runtime_error("Corrupt estimates file. Historical block range for estimates is invalid");
975 : }
976 1390 : std::vector<double> fileBuckets;
977 1390 : filein >> Using<VectorFormatter<EncodedDoubleFormatter>>(fileBuckets);
978 1390 : size_t numBuckets = fileBuckets.size();
979 1390 : if (numBuckets <= 1 || numBuckets > 1000) {
980 0 : throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets");
981 : }
982 :
983 1390 : auto fileFeeStats{std::make_unique<TxConfirmStats>(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE)};
984 1390 : auto fileShortStats{std::make_unique<TxConfirmStats>(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE)};
985 1390 : auto fileLongStats{std::make_unique<TxConfirmStats>(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE)};
986 1390 : fileFeeStats->Read(filein, nVersionThatWrote, numBuckets);
987 1390 : fileShortStats->Read(filein, nVersionThatWrote, numBuckets);
988 1390 : fileLongStats->Read(filein, nVersionThatWrote, numBuckets);
989 :
990 : // Fee estimates file parsed correctly
991 : // Copy buckets from file and refresh our bucketmap
992 1390 : buckets = fileBuckets;
993 1390 : bucketMap.clear();
994 265490 : for (unsigned int i = 0; i < buckets.size(); i++) {
995 264100 : bucketMap[buckets[i]] = i;
996 264100 : }
997 :
998 : // Destroy old TxConfirmStats and point to new ones that already reference buckets and bucketMap
999 1390 : feeStats = std::move(fileFeeStats);
1000 1390 : shortStats = std::move(fileShortStats);
1001 1390 : longStats = std::move(fileLongStats);
1002 :
1003 1390 : nBestSeenHeight = nFileBestSeenHeight;
1004 1390 : historicalFirst = nFileHistoricalFirst;
1005 1390 : historicalBest = nFileHistoricalBest;
1006 1390 : }
1007 1390 : }
1008 : catch (const std::exception& e) {
1009 0 : LogPrintf("CBlockPolicyEstimator::Read(): unable to read policy estimator data (non-fatal): %s\n",e.what());
1010 0 : return false;
1011 0 : }
1012 1390 : return true;
1013 1390 : }
1014 :
1015 2924 : void CBlockPolicyEstimator::FlushUnconfirmed() {
1016 2924 : int64_t startclear = GetTimeMicros();
1017 2924 : LOCK(m_cs_fee_estimator);
1018 2924 : size_t num_entries = mapMemPoolTxs.size();
1019 : // Remove every entry in mapMemPoolTxs
1020 4114 : while (!mapMemPoolTxs.empty()) {
1021 1190 : auto mi = mapMemPoolTxs.begin();
1022 1190 : _removeTx(mi->first, false); // this calls erase() on mapMemPoolTxs
1023 : }
1024 2924 : int64_t endclear = GetTimeMicros();
1025 2924 : LogPrint(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %ld micros\n", num_entries, endclear - startclear);
1026 2924 : }
|
