LCOV - code coverage report
Current view: top level - src/policy - fees.cpp (source / functions) Hit Total Coverage
Test: fuzz_coverage.info Lines: 98 600 16.3 %
Date: 2023-11-12 01:39:15 Functions: 11 43 25.6 %
Branches: 58 688 8.4 %

           Branch data     Line data    Source code
       1                 :            : // Copyright (c) 2009-2010 Satoshi Nakamoto
       2                 :            : // Copyright (c) 2009-2022 The Bitcoin Core developers
       3                 :            : // Distributed under the MIT software license, see the accompanying
       4                 :            : // file COPYING or http://www.opensource.org/licenses/mit-license.php.
       5                 :            : 
       6                 :            : #include <policy/fees.h>
       7                 :            : 
       8                 :            : #include <clientversion.h>
       9                 :            : #include <common/system.h>
      10                 :            : #include <consensus/amount.h>
      11                 :            : #include <kernel/mempool_entry.h>
      12                 :            : #include <logging.h>
      13                 :            : #include <policy/feerate.h>
      14                 :            : #include <primitives/transaction.h>
      15                 :            : #include <random.h>
      16                 :            : #include <serialize.h>
      17         [ +  - ]:          2 : #include <streams.h>
      18         [ +  - ]:          2 : #include <sync.h>
      19                 :            : #include <tinyformat.h>
      20                 :            : #include <uint256.h>
      21                 :            : #include <util/fs.h>
      22                 :            : #include <util/serfloat.h>
      23                 :            : #include <util/time.h>
      24                 :            : 
      25                 :            : #include <algorithm>
      26                 :            : #include <cassert>
      27                 :            : #include <chrono>
      28                 :            : #include <cmath>
      29                 :            : #include <cstddef>
      30                 :            : #include <cstdint>
      31                 :            : #include <exception>
      32                 :            : #include <stdexcept>
      33                 :            : #include <utility>
      34                 :            : 
      35                 :            : static constexpr double INF_FEERATE = 1e99;
      36                 :            : 
      37                 :          0 : std::string StringForFeeEstimateHorizon(FeeEstimateHorizon horizon)
      38                 :            : {
      39   [ #  #  #  # ]:          0 :     switch (horizon) {
      40         [ #  # ]:          0 :     case FeeEstimateHorizon::SHORT_HALFLIFE: return "short";
      41         [ #  # ]:          0 :     case FeeEstimateHorizon::MED_HALFLIFE: return "medium";
      42         [ #  # ]:          0 :     case FeeEstimateHorizon::LONG_HALFLIFE: return "long";
      43                 :            :     } // no default case, so the compiler can warn about missing cases
      44                 :          0 :     assert(false);
      45                 :          0 : }
      46                 :            : 
      47                 :            : namespace {
      48                 :            : 
      49                 :            : struct EncodedDoubleFormatter
      50                 :            : {
      51                 :          0 :     template<typename Stream> void Ser(Stream &s, double v)
      52                 :            :     {
      53                 :          0 :         s << EncodeDouble(v);
      54                 :          0 :     }
      55                 :            : 
      56                 :          0 :     template<typename Stream> void Unser(Stream& s, double& v)
      57                 :            :     {
      58                 :            :         uint64_t encoded;
      59                 :          0 :         s >> encoded;
      60                 :          0 :         v = DecodeDouble(encoded);
      61                 :          0 :     }
      62                 :            : };
      63                 :            : 
      64                 :            : } // namespace
      65                 :            : 
      66                 :            : /**
      67                 :            :  * We will instantiate an instance of this class to track transactions that were
      68                 :            :  * included in a block. We will lump transactions into a bucket according to their
      69                 :            :  * approximate feerate and then track how long it took for those txs to be included in a block
      70                 :            :  *
      71                 :            :  * The tracking of unconfirmed (mempool) transactions is completely independent of the
      72                 :            :  * historical tracking of transactions that have been confirmed in a block.
      73                 :            :  */
      74                 :          0 : class TxConfirmStats
      75                 :            : {
      76                 :            : private:
      77                 :            :     //Define the buckets we will group transactions into
      78                 :            :     const std::vector<double>& buckets;              // The upper-bound of the range for the bucket (inclusive)
      79                 :            :     const std::map<double, unsigned int>& bucketMap; // Map of bucket upper-bound to index into all vectors by bucket
      80                 :            : 
      81                 :            :     // For each bucket X:
      82                 :            :     // Count the total # of txs in each bucket
      83                 :            :     // Track the historical moving average of this total over blocks
      84                 :            :     std::vector<double> txCtAvg;
      85                 :            : 
      86                 :            :     // Count the total # of txs confirmed within Y blocks in each bucket
      87                 :            :     // Track the historical moving average of these totals over blocks
      88                 :            :     std::vector<std::vector<double>> confAvg; // confAvg[Y][X]
      89                 :            : 
      90                 :            :     // Track moving avg of txs which have been evicted from the mempool
      91                 :            :     // after failing to be confirmed within Y blocks
      92                 :            :     std::vector<std::vector<double>> failAvg; // failAvg[Y][X]
      93                 :            : 
      94                 :            :     // Sum the total feerate of all tx's in each bucket
      95                 :            :     // Track the historical moving average of this total over blocks
      96                 :            :     std::vector<double> m_feerate_avg;
      97                 :            : 
      98                 :            :     // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X
      99                 :            :     // Combine the total value with the tx counts to calculate the avg feerate per bucket
     100                 :            : 
     101                 :            :     double decay;
     102                 :            : 
     103                 :            :     // Resolution (# of blocks) with which confirmations are tracked
     104                 :            :     unsigned int scale;
     105                 :            : 
     106                 :            :     // Mempool counts of outstanding transactions
     107                 :            :     // For each bucket X, track the number of transactions in the mempool
     108                 :            :     // that are unconfirmed for each possible confirmation value Y
     109                 :            :     std::vector<std::vector<int> > unconfTxs;  //unconfTxs[Y][X]
     110                 :            :     // transactions still unconfirmed after GetMaxConfirms for each bucket
     111                 :            :     std::vector<int> oldUnconfTxs;
     112                 :            : 
     113                 :            :     void resizeInMemoryCounters(size_t newbuckets);
     114                 :            : 
     115                 :            : public:
     116                 :            :     /**
     117                 :            :      * Create new TxConfirmStats. This is called by BlockPolicyEstimator's
     118                 :            :      * constructor with default values.
     119                 :            :      * @param defaultBuckets contains the upper limits for the bucket boundaries
     120                 :            :      * @param maxPeriods max number of periods to track
     121                 :            :      * @param decay how much to decay the historical moving average per block
     122                 :            :      */
     123                 :            :     TxConfirmStats(const std::vector<double>& defaultBuckets, const std::map<double, unsigned int>& defaultBucketMap,
     124                 :            :                    unsigned int maxPeriods, double decay, unsigned int scale);
     125                 :            : 
     126                 :            :     /** Roll the circular buffer for unconfirmed txs*/
     127                 :            :     void ClearCurrent(unsigned int nBlockHeight);
     128                 :            : 
     129                 :            :     /**
     130                 :            :      * Record a new transaction data point in the current block stats
     131                 :            :      * @param blocksToConfirm the number of blocks it took this transaction to confirm
     132                 :            :      * @param val the feerate of the transaction
     133                 :            :      * @warning blocksToConfirm is 1-based and has to be >= 1
     134                 :            :      */
     135                 :            :     void Record(int blocksToConfirm, double val);
     136                 :            : 
     137                 :            :     /** Record a new transaction entering the mempool*/
     138                 :            :     unsigned int NewTx(unsigned int nBlockHeight, double val);
     139                 :            : 
     140                 :            :     /** Remove a transaction from mempool tracking stats*/
     141                 :            :     void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight,
     142                 :            :                   unsigned int bucketIndex, bool inBlock);
     143                 :            : 
     144                 :            :     /** Update our estimates by decaying our historical moving average and updating
     145                 :            :         with the data gathered from the current block */
     146                 :            :     void UpdateMovingAverages();
     147                 :            : 
     148                 :            :     /**
     149                 :            :      * Calculate a feerate estimate.  Find the lowest value bucket (or range of buckets
     150                 :            :      * to make sure we have enough data points) whose transactions still have sufficient likelihood
     151                 :            :      * of being confirmed within the target number of confirmations
     152                 :            :      * @param confTarget target number of confirmations
     153                 :            :      * @param sufficientTxVal required average number of transactions per block in a bucket range
     154                 :            :      * @param minSuccess the success probability we require
     155                 :            :      * @param nBlockHeight the current block height
     156                 :            :      */
     157                 :            :     double EstimateMedianVal(int confTarget, double sufficientTxVal,
     158                 :            :                              double minSuccess, unsigned int nBlockHeight,
     159                 :            :                              EstimationResult *result = nullptr) const;
     160                 :            : 
     161                 :            :     /** Return the max number of confirms we're tracking */
     162                 :          3 :     unsigned int GetMaxConfirms() const { return scale * confAvg.size(); }
     163                 :            : 
     164                 :            :     /** Write state of estimation data to a file*/
     165                 :            :     void Write(AutoFile& fileout) const;
     166                 :            : 
     167                 :            :     /**
     168                 :            :      * Read saved state of estimation data from a file and replace all internal data structures and
     169                 :            :      * variables with this state.
     170                 :            :      */
     171                 :            :     void Read(AutoFile& filein, int nFileVersion, size_t numBuckets);
     172                 :            : };
     173                 :            : 
     174                 :            : 
     175                 :          9 : TxConfirmStats::TxConfirmStats(const std::vector<double>& defaultBuckets,
     176                 :            :                                 const std::map<double, unsigned int>& defaultBucketMap,
     177                 :            :                                unsigned int maxPeriods, double _decay, unsigned int _scale)
     178                 :          6 :     : buckets(defaultBuckets), bucketMap(defaultBucketMap), decay(_decay), scale(_scale)
     179                 :            : {
     180 [ -  + ][ +  - ]:          6 :     assert(_scale != 0 && "_scale must be non-zero");
     181         [ +  - ]:          3 :     confAvg.resize(maxPeriods);
     182         [ +  - ]:          3 :     failAvg.resize(maxPeriods);
     183         [ +  + ]:         81 :     for (unsigned int i = 0; i < maxPeriods; i++) {
     184         [ +  - ]:         78 :         confAvg[i].resize(buckets.size());
     185         [ +  - ]:         78 :         failAvg[i].resize(buckets.size());
     186                 :         78 :     }
     187                 :            : 
     188         [ +  - ]:          3 :     txCtAvg.resize(buckets.size());
     189         [ +  - ]:          3 :     m_feerate_avg.resize(buckets.size());
     190                 :            : 
     191         [ +  - ]:          3 :     resizeInMemoryCounters(buckets.size());
     192                 :          3 : }
     193                 :            : 
     194                 :          3 : void TxConfirmStats::resizeInMemoryCounters(size_t newbuckets) {
     195                 :            :     // newbuckets must be passed in because the buckets referred to during Read have not been updated yet.
     196                 :          3 :     unconfTxs.resize(GetMaxConfirms());
     197         [ +  + ]:       1071 :     for (unsigned int i = 0; i < unconfTxs.size(); i++) {
     198                 :       1068 :         unconfTxs[i].resize(newbuckets);
     199                 :       1068 :     }
     200                 :          3 :     oldUnconfTxs.resize(newbuckets);
     201                 :          3 : }
     202                 :            : 
     203                 :            : // Roll the unconfirmed txs circular buffer
     204                 :        600 : void TxConfirmStats::ClearCurrent(unsigned int nBlockHeight)
     205                 :            : {
     206         [ +  + ]:     114600 :     for (unsigned int j = 0; j < buckets.size(); j++) {
     207                 :     114000 :         oldUnconfTxs[j] += unconfTxs[nBlockHeight % unconfTxs.size()][j];
     208                 :     114000 :         unconfTxs[nBlockHeight%unconfTxs.size()][j] = 0;
     209                 :     114000 :     }
     210                 :        600 : }
     211                 :            : 
     212                 :            : 
     213                 :          0 : void TxConfirmStats::Record(int blocksToConfirm, double feerate)
     214                 :            : {
     215                 :            :     // blocksToConfirm is 1-based
     216         [ #  # ]:          0 :     if (blocksToConfirm < 1)
     217                 :          0 :         return;
     218                 :          0 :     int periodsToConfirm = (blocksToConfirm + scale - 1) / scale;
     219                 :          0 :     unsigned int bucketindex = bucketMap.lower_bound(feerate)->second;
     220         [ #  # ]:          0 :     for (size_t i = periodsToConfirm; i <= confAvg.size(); i++) {
     221                 :          0 :         confAvg[i - 1][bucketindex]++;
     222                 :          0 :     }
     223                 :          0 :     txCtAvg[bucketindex]++;
     224                 :          0 :     m_feerate_avg[bucketindex] += feerate;
     225                 :          0 : }
     226                 :            : 
     227                 :        600 : void TxConfirmStats::UpdateMovingAverages()
     228                 :            : {
     229         [ +  - ]:        600 :     assert(confAvg.size() == failAvg.size());
     230         [ +  + ]:     114600 :     for (unsigned int j = 0; j < buckets.size(); j++) {
     231         [ +  + ]:    3078000 :         for (unsigned int i = 0; i < confAvg.size(); i++) {
     232                 :    2964000 :             confAvg[i][j] *= decay;
     233                 :    2964000 :             failAvg[i][j] *= decay;
     234                 :    2964000 :         }
     235                 :     114000 :         m_feerate_avg[j] *= decay;
     236                 :     114000 :         txCtAvg[j] *= decay;
     237                 :     114000 :     }
     238                 :        600 : }
     239                 :            : 
     240                 :            : // returns -1 on error conditions
     241                 :          0 : double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal,
     242                 :            :                                          double successBreakPoint, unsigned int nBlockHeight,
     243                 :            :                                          EstimationResult *result) const
     244                 :            : {
     245                 :            :     // Counters for a bucket (or range of buckets)
     246                 :          0 :     double nConf = 0; // Number of tx's confirmed within the confTarget
     247                 :          0 :     double totalNum = 0; // Total number of tx's that were ever confirmed
     248                 :          0 :     int extraNum = 0;  // Number of tx's still in mempool for confTarget or longer
     249                 :          0 :     double failNum = 0; // Number of tx's that were never confirmed but removed from the mempool after confTarget
     250                 :          0 :     const int periodTarget = (confTarget + scale - 1) / scale;
     251                 :          0 :     const int maxbucketindex = buckets.size() - 1;
     252                 :            : 
     253                 :            :     // We'll combine buckets until we have enough samples.
     254                 :            :     // The near and far variables will define the range we've combined
     255                 :            :     // The best variables are the last range we saw which still had a high
     256                 :            :     // enough confirmation rate to count as success.
     257                 :            :     // The cur variables are the current range we're counting.
     258                 :          0 :     unsigned int curNearBucket = maxbucketindex;
     259                 :          0 :     unsigned int bestNearBucket = maxbucketindex;
     260                 :          0 :     unsigned int curFarBucket = maxbucketindex;
     261                 :          0 :     unsigned int bestFarBucket = maxbucketindex;
     262                 :            : 
     263                 :            :     // We'll always group buckets into sets that meet sufficientTxVal --
     264                 :            :     // this ensures that we're using consistent groups between different
     265                 :            :     // confirmation targets.
     266                 :          1 :     double partialNum = 0;
     267                 :          1 : 
     268                 :          1 :     bool foundAnswer = false;
     269                 :          1 :     unsigned int bins = unconfTxs.size();
     270                 :          0 :     bool newBucketRange = true;
     271                 :          0 :     bool passing = true;
     272                 :          0 :     EstimatorBucket passBucket;
     273                 :          0 :     EstimatorBucket failBucket;
     274                 :            : 
     275                 :            :     // Start counting from highest feerate transactions
     276         [ #  # ]:          0 :     for (int bucket = maxbucketindex; bucket >= 0; --bucket) {
     277         [ #  # ]:          0 :         if (newBucketRange) {
     278                 :          0 :             curNearBucket = bucket;
     279                 :          0 :             newBucketRange = false;
     280                 :          0 :         }
     281                 :          0 :         curFarBucket = bucket;
     282                 :          0 :         nConf += confAvg[periodTarget - 1][bucket];
     283                 :          0 :         partialNum += txCtAvg[bucket];
     284                 :          0 :         totalNum += txCtAvg[bucket];
     285                 :          0 :         failNum += failAvg[periodTarget - 1][bucket];
     286         [ #  # ]:          1 :         for (unsigned int confct = confTarget; confct < GetMaxConfirms(); confct++)
     287                 :          1 :             extraNum += unconfTxs[(nBlockHeight - confct) % bins][bucket];
     288                 :          0 :         extraNum += oldUnconfTxs[bucket];
     289                 :            :         // If we have enough transaction data points in this range of buckets,
     290                 :            :         // we can test for success
     291                 :            :         // (Only count the confirmed data points, so that each confirmation count
     292                 :            :         // will be looking at the same amount of data and same bucket breaks)
     293                 :            : 
     294         [ #  # ]:          0 :         if (partialNum < sufficientTxVal / (1 - decay)) {
     295                 :            :             // the buckets we've added in this round aren't sufficient
     296                 :            :             // so keep adding
     297                 :          0 :             continue;
     298                 :            :         } else {
     299                 :          0 :             partialNum = 0; // reset for the next range we'll add
     300                 :            : 
     301                 :          0 :             double curPct = nConf / (totalNum + failNum + extraNum);
     302                 :            : 
     303                 :            :             // Check to see if we are no longer getting confirmed at the success rate
     304         [ #  # ]:          0 :             if (curPct < successBreakPoint) {
     305         [ #  # ]:          0 :                 if (passing == true) {
     306                 :            :                     // First time we hit a failure record the failed bucket
     307                 :          0 :                     unsigned int failMinBucket = std::min(curNearBucket, curFarBucket);
     308                 :          0 :                     unsigned int failMaxBucket = std::max(curNearBucket, curFarBucket);
     309         [ #  # ]:          0 :                     failBucket.start = failMinBucket ? buckets[failMinBucket - 1] : 0;
     310                 :          0 :                     failBucket.end = buckets[failMaxBucket];
     311                 :          0 :                     failBucket.withinTarget = nConf;
     312                 :          0 :                     failBucket.totalConfirmed = totalNum;
     313                 :          0 :                     failBucket.inMempool = extraNum;
     314                 :          0 :                     failBucket.leftMempool = failNum;
     315                 :          0 :                     passing = false;
     316                 :          0 :                 }
     317                 :          0 :                 continue;
     318                 :            :             }
     319                 :            :             // Otherwise update the cumulative stats, and the bucket variables
     320                 :            :             // and reset the counters
     321                 :            :             else {
     322                 :          0 :                 failBucket = EstimatorBucket(); // Reset any failed bucket, currently passing
     323                 :          0 :                 foundAnswer = true;
     324                 :          0 :                 passing = true;
     325                 :          0 :                 passBucket.withinTarget = nConf;
     326                 :          0 :                 nConf = 0;
     327                 :          0 :                 passBucket.totalConfirmed = totalNum;
     328                 :          0 :                 totalNum = 0;
     329                 :          0 :                 passBucket.inMempool = extraNum;
     330                 :          0 :                 passBucket.leftMempool = failNum;
     331                 :          0 :                 failNum = 0;
     332                 :          0 :                 extraNum = 0;
     333                 :          0 :                 bestNearBucket = curNearBucket;
     334                 :          0 :                 bestFarBucket = curFarBucket;
     335                 :          0 :                 newBucketRange = true;
     336                 :            :             }
     337                 :            :         }
     338                 :          0 :     }
     339                 :            : 
     340                 :          0 :     double median = -1;
     341                 :          0 :     double txSum = 0;
     342                 :            : 
     343                 :            :     // Calculate the "average" feerate of the best bucket range that met success conditions
     344                 :            :     // Find the bucket with the median transaction and then report the average feerate from that bucket
     345                 :            :     // This is a compromise between finding the median which we can't since we don't save all tx's
     346                 :            :     // and reporting the average which is less accurate
     347                 :          0 :     unsigned int minBucket = std::min(bestNearBucket, bestFarBucket);
     348                 :          0 :     unsigned int maxBucket = std::max(bestNearBucket, bestFarBucket);
     349         [ #  # ]:          0 :     for (unsigned int j = minBucket; j <= maxBucket; j++) {
     350                 :          0 :         txSum += txCtAvg[j];
     351                 :          0 :     }
     352 [ #  # ][ #  # ]:          0 :     if (foundAnswer && txSum != 0) {
     353                 :          0 :         txSum = txSum / 2;
     354         [ #  # ]:          0 :         for (unsigned int j = minBucket; j <= maxBucket; j++) {
     355         [ #  # ]:          0 :             if (txCtAvg[j] < txSum)
     356                 :          0 :                 txSum -= txCtAvg[j];
     357                 :            :             else { // we're in the right bucket
     358                 :          0 :                 median = m_feerate_avg[j] / txCtAvg[j];
     359                 :          0 :                 break;
     360                 :            :             }
     361                 :          0 :         }
     362                 :            : 
     363         [ #  # ]:          0 :         passBucket.start = minBucket ? buckets[minBucket-1] : 0;
     364                 :          0 :         passBucket.end = buckets[maxBucket];
     365                 :          0 :     }
     366                 :            : 
     367                 :            :     // If we were passing until we reached last few buckets with insufficient data, then report those as failed
     368 [ #  # ][ #  # ]:          0 :     if (passing && !newBucketRange) {
     369                 :          0 :         unsigned int failMinBucket = std::min(curNearBucket, curFarBucket);
     370                 :          0 :         unsigned int failMaxBucket = std::max(curNearBucket, curFarBucket);
     371         [ #  # ]:          0 :         failBucket.start = failMinBucket ? buckets[failMinBucket - 1] : 0;
     372                 :          0 :         failBucket.end = buckets[failMaxBucket];
     373                 :          0 :         failBucket.withinTarget = nConf;
     374                 :          0 :         failBucket.totalConfirmed = totalNum;
     375                 :          0 :         failBucket.inMempool = extraNum;
     376                 :          0 :         failBucket.leftMempool = failNum;
     377                 :          0 :     }
     378                 :            : 
     379                 :          0 :     float passed_within_target_perc = 0.0;
     380                 :          0 :     float failed_within_target_perc = 0.0;
     381         [ #  # ]:          0 :     if ((passBucket.totalConfirmed + passBucket.inMempool + passBucket.leftMempool)) {
     382                 :          0 :         passed_within_target_perc = 100 * passBucket.withinTarget / (passBucket.totalConfirmed + passBucket.inMempool + passBucket.leftMempool);
     383                 :          0 :     }
     384         [ #  # ]:          0 :     if ((failBucket.totalConfirmed + failBucket.inMempool + failBucket.leftMempool)) {
     385                 :          0 :         failed_within_target_perc = 100 * failBucket.withinTarget / (failBucket.totalConfirmed + failBucket.inMempool + failBucket.leftMempool);
     386                 :          0 :     }
     387                 :            : 
     388 [ #  # ][ #  # ]:          0 :     LogPrint(BCLog::ESTIMATEFEE, "FeeEst: %d > %.0f%% decay %.5f: feerate: %g from (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out) Fail: (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out)\n",
         [ #  # ][ #  # ]
     389                 :            :              confTarget, 100.0 * successBreakPoint, decay,
     390                 :            :              median, passBucket.start, passBucket.end,
     391                 :            :              passed_within_target_perc,
     392                 :            :              passBucket.withinTarget, passBucket.totalConfirmed, passBucket.inMempool, passBucket.leftMempool,
     393                 :            :              failBucket.start, failBucket.end,
     394                 :            :              failed_within_target_perc,
     395                 :            :              failBucket.withinTarget, failBucket.totalConfirmed, failBucket.inMempool, failBucket.leftMempool);
     396                 :            : 
     397                 :            : 
     398         [ #  # ]:          0 :     if (result) {
     399                 :          0 :         result->pass = passBucket;
     400                 :          0 :         result->fail = failBucket;
     401                 :          0 :         result->decay = decay;
     402                 :          0 :         result->scale = scale;
     403                 :          0 :     }
     404                 :          0 :     return median;
     405                 :          0 : }
     406                 :            : 
     407                 :          0 : void TxConfirmStats::Write(AutoFile& fileout) const
     408                 :            : {
     409                 :          0 :     fileout << Using<EncodedDoubleFormatter>(decay);
     410                 :          0 :     fileout << scale;
     411                 :          0 :     fileout << Using<VectorFormatter<EncodedDoubleFormatter>>(m_feerate_avg);
     412                 :          0 :     fileout << Using<VectorFormatter<EncodedDoubleFormatter>>(txCtAvg);
     413                 :          0 :     fileout << Using<VectorFormatter<VectorFormatter<EncodedDoubleFormatter>>>(confAvg);
     414                 :          0 :     fileout << Using<VectorFormatter<VectorFormatter<EncodedDoubleFormatter>>>(failAvg);
     415                 :          0 : }
     416                 :            : 
     417                 :          0 : void TxConfirmStats::Read(AutoFile& filein, int nFileVersion, size_t numBuckets)
     418                 :            : {
     419                 :            :     // Read data file and do some very basic sanity checking
     420                 :            :     // buckets and bucketMap are not updated yet, so don't access them
     421                 :            :     // If there is a read failure, we'll just discard this entire object anyway
     422                 :            :     size_t maxConfirms, maxPeriods;
     423                 :            : 
     424                 :            :     // The current version will store the decay with each individual TxConfirmStats and also keep a scale factor
     425                 :          0 :     filein >> Using<EncodedDoubleFormatter>(decay);
     426         [ #  # ]:          0 :     if (decay <= 0 || decay >= 1) {
     427         [ #  # ]:          0 :         throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)");
     428                 :            :     }
     429                 :          0 :     filein >> scale;
     430         [ #  # ]:          0 :     if (scale == 0) {
     431         [ #  # ]:          0 :         throw std::runtime_error("Corrupt estimates file. Scale must be non-zero");
     432                 :            :     }
     433                 :            : 
     434                 :          0 :     filein >> Using<VectorFormatter<EncodedDoubleFormatter>>(m_feerate_avg);
     435         [ #  # ]:          0 :     if (m_feerate_avg.size() != numBuckets) {
     436         [ #  # ]:          0 :         throw std::runtime_error("Corrupt estimates file. Mismatch in feerate average bucket count");
     437                 :            :     }
     438                 :          0 :     filein >> Using<VectorFormatter<EncodedDoubleFormatter>>(txCtAvg);
     439         [ #  # ]:          0 :     if (txCtAvg.size() != numBuckets) {
     440         [ #  # ]:          0 :         throw std::runtime_error("Corrupt estimates file. Mismatch in tx count bucket count");
     441                 :            :     }
     442                 :          0 :     filein >> Using<VectorFormatter<VectorFormatter<EncodedDoubleFormatter>>>(confAvg);
     443                 :          0 :     maxPeriods = confAvg.size();
     444                 :          0 :     maxConfirms = scale * maxPeriods;
     445                 :            : 
     446         [ #  # ]:          0 :     if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) { // one week
     447         [ #  # ]:          0 :         throw std::runtime_error("Corrupt estimates file.  Must maintain estimates for between 1 and 1008 (one week) confirms");
     448                 :            :     }
     449         [ #  # ]:          0 :     for (unsigned int i = 0; i < maxPeriods; i++) {
     450         [ #  # ]:          0 :         if (confAvg[i].size() != numBuckets) {
     451         [ #  # ]:          0 :             throw std::runtime_error("Corrupt estimates file. Mismatch in feerate conf average bucket count");
     452                 :            :         }
     453                 :          0 :     }
     454                 :            : 
     455                 :          0 :     filein >> Using<VectorFormatter<VectorFormatter<EncodedDoubleFormatter>>>(failAvg);
     456         [ #  # ]:          0 :     if (maxPeriods != failAvg.size()) {
     457         [ #  # ]:          0 :         throw std::runtime_error("Corrupt estimates file. Mismatch in confirms tracked for failures");
     458                 :            :     }
     459         [ #  # ]:          0 :     for (unsigned int i = 0; i < maxPeriods; i++) {
     460         [ #  # ]:          0 :         if (failAvg[i].size() != numBuckets) {
     461         [ #  # ]:          0 :             throw std::runtime_error("Corrupt estimates file. Mismatch in one of failure average bucket counts");
     462                 :            :         }
     463                 :          0 :     }
     464                 :            : 
     465                 :            :     // Resize the current block variables which aren't stored in the data file
     466                 :            :     // to match the number of confirms and buckets
     467                 :          0 :     resizeInMemoryCounters(numBuckets);
     468                 :            : 
     469 [ #  # ][ #  # ]:          0 :     LogPrint(BCLog::ESTIMATEFEE, "Reading estimates: %u buckets counting confirms up to %u blocks\n",
         [ #  # ][ #  # ]
     470                 :            :              numBuckets, maxConfirms);
     471                 :          0 : }
     472                 :            : 
     473                 :          0 : unsigned int TxConfirmStats::NewTx(unsigned int nBlockHeight, double val)
     474                 :            : {
     475                 :          0 :     unsigned int bucketindex = bucketMap.lower_bound(val)->second;
     476                 :          0 :     unsigned int blockIndex = nBlockHeight % unconfTxs.size();
     477                 :          0 :     unconfTxs[blockIndex][bucketindex]++;
     478                 :          0 :     return bucketindex;
     479                 :            : }
     480                 :            : 
     481                 :          0 : void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketindex, bool inBlock)
     482                 :            : {
     483                 :            :     //nBestSeenHeight is not updated yet for the new block
     484                 :          0 :     int blocksAgo = nBestSeenHeight - entryHeight;
     485         [ #  # ]:          0 :     if (nBestSeenHeight == 0)  // the BlockPolicyEstimator hasn't seen any blocks yet
     486                 :          0 :         blocksAgo = 0;
     487         [ #  # ]:          0 :     if (blocksAgo < 0) {
     488 [ #  # ][ #  # ]:          0 :         LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error, blocks ago is negative for mempool tx\n");
         [ #  # ][ #  # ]
     489                 :          0 :         return;  //This can't happen because we call this with our best seen height, no entries can have higher
     490                 :            :     }
     491                 :            : 
     492         [ #  # ]:          0 :     if (blocksAgo >= (int)unconfTxs.size()) {
     493         [ #  # ]:          0 :         if (oldUnconfTxs[bucketindex] > 0) {
     494                 :          0 :             oldUnconfTxs[bucketindex]--;
     495                 :          0 :         } else {
     496 [ #  # ][ #  # ]:          0 :             LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error, mempool tx removed from >25 blocks,bucketIndex=%u already\n",
         [ #  # ][ #  # ]
     497                 :            :                      bucketindex);
     498                 :            :         }
     499                 :          0 :     }
     500                 :            :     else {
     501                 :          0 :         unsigned int blockIndex = entryHeight % unconfTxs.size();
     502         [ #  # ]:          0 :         if (unconfTxs[blockIndex][bucketindex] > 0) {
     503                 :          0 :             unconfTxs[blockIndex][bucketindex]--;
     504                 :          0 :         } else {
     505 [ #  # ][ #  # ]:          0 :             LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error, mempool tx removed from blockIndex=%u,bucketIndex=%u already\n",
         [ #  # ][ #  # ]
     506                 :            :                      blockIndex, bucketindex);
     507                 :            :         }
     508                 :            :     }
     509 [ #  # ][ #  # ]:          0 :     if (!inBlock && (unsigned int)blocksAgo >= scale) { // Only counts as a failure if not confirmed for entire period
     510         [ #  # ]:          0 :         assert(scale != 0);
     511                 :          0 :         unsigned int periodsAgo = blocksAgo / scale;
     512 [ #  # ][ #  # ]:          0 :         for (size_t i = 0; i < periodsAgo && i < failAvg.size(); i++) {
     513                 :          0 :             failAvg[i][bucketindex]++;
     514                 :          0 :         }
     515                 :          0 :     }
     516                 :          0 : }
     517                 :            : 
     518                 :            : // This function is called from CTxMemPool::removeUnchecked to ensure
     519                 :            : // txs removed from the mempool for any reason are no longer
     520                 :            : // tracked. Txs that were part of a block have already been removed in
     521                 :            : // processBlockTx to ensure they are never double tracked, but it is
     522                 :            : // of no harm to try to remove them again.
     523                 :          0 : bool CBlockPolicyEstimator::removeTx(uint256 hash, bool inBlock)
     524                 :            : {
     525                 :          0 :     LOCK(m_cs_fee_estimator);
     526         [ #  # ]:          0 :     return _removeTx(hash, inBlock);
     527                 :          0 : }
     528                 :            : 
     529                 :          0 : bool CBlockPolicyEstimator::_removeTx(const uint256& hash, bool inBlock)
     530                 :            : {
     531                 :          0 :     AssertLockHeld(m_cs_fee_estimator);
     532                 :          0 :     std::map<uint256, TxStatsInfo>::iterator pos = mapMemPoolTxs.find(hash);
     533         [ #  # ]:          0 :     if (pos != mapMemPoolTxs.end()) {
     534                 :          0 :         feeStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock);
     535                 :          0 :         shortStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock);
     536                 :          0 :         longStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock);
     537                 :          0 :         mapMemPoolTxs.erase(hash);
     538                 :          0 :         return true;
     539                 :            :     } else {
     540                 :          0 :         return false;
     541                 :            :     }
     542                 :          0 : }
     543                 :            : 
     544                 :          4 : CBlockPolicyEstimator::CBlockPolicyEstimator(const fs::path& estimation_filepath, const bool read_stale_estimates)
     545                 :          1 :     : m_estimation_filepath{estimation_filepath}
     546                 :            : {
     547                 :            :     static_assert(MIN_BUCKET_FEERATE > 0, "Min feerate must be nonzero");
     548                 :          1 :     size_t bucketIndex = 0;
     549                 :            : 
     550         [ +  + ]:        190 :     for (double bucketBoundary = MIN_BUCKET_FEERATE; bucketBoundary <= MAX_BUCKET_FEERATE; bucketBoundary *= FEE_SPACING, bucketIndex++) {
     551         [ +  - ]:        189 :         buckets.push_back(bucketBoundary);
     552         [ +  - ]:        189 :         bucketMap[bucketBoundary] = bucketIndex;
     553                 :        189 :     }
     554         [ +  - ]:          1 :     buckets.push_back(INF_FEERATE);
     555         [ +  - ]:          1 :     bucketMap[INF_FEERATE] = bucketIndex;
     556         [ +  - ]:          1 :     assert(bucketMap.size() == buckets.size());
     557                 :            : 
     558 [ +  - ][ +  - ]:          1 :     feeStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE));
     559 [ +  - ][ +  - ]:          1 :     shortStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE));
     560 [ +  - ][ +  - ]:          1 :     longStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE));
     561                 :            : 
     562 [ +  - ][ +  - ]:          1 :     AutoFile est_file{fsbridge::fopen(m_estimation_filepath, "rb")};
     563                 :            : 
     564 [ +  - ][ -  + ]:          1 :     if (est_file.IsNull()) {
     565 [ +  - ][ +  - ]:          1 :         LogPrintf("%s is not found. Continue anyway.\n", fs::PathToString(m_estimation_filepath));
         [ +  - ][ +  - ]
     566                 :          1 :         return;
     567                 :            :     }
     568                 :            : 
     569         [ #  # ]:          0 :     std::chrono::hours file_age = GetFeeEstimatorFileAge();
     570 [ #  # ][ #  # ]:          0 :     if (file_age > MAX_FILE_AGE && !read_stale_estimates) {
                 [ #  # ]
     571 [ #  # ][ #  # ]:          0 :         LogPrintf("Fee estimation file %s too old (age=%lld > %lld hours) and will not be used to avoid serving stale estimates.\n", fs::PathToString(m_estimation_filepath), Ticks<std::chrono::hours>(file_age), Ticks<std::chrono::hours>(MAX_FILE_AGE));
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
     572                 :          0 :         return;
     573                 :            :     }
     574                 :            : 
     575 [ #  # ][ #  # ]:          0 :     if (!Read(est_file)) {
     576 [ #  # ][ #  # ]:          0 :         LogPrintf("Failed to read fee estimates from %s. Continue anyway.\n", fs::PathToString(m_estimation_filepath));
         [ #  # ][ #  # ]
     577                 :          0 :     }
     578         [ -  + ]:          1 : }
     579                 :            : 
     580                 :          1 : CBlockPolicyEstimator::~CBlockPolicyEstimator() = default;
     581                 :            : 
     582                 :          0 : void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate)
     583                 :            : {
     584                 :          0 :     LOCK(m_cs_fee_estimator);
     585         [ #  # ]:          0 :     unsigned int txHeight = entry.GetHeight();
     586 [ #  # ][ #  # ]:          0 :     uint256 hash = entry.GetTx().GetHash();
                 [ #  # ]
     587 [ #  # ][ #  # ]:          0 :     if (mapMemPoolTxs.count(hash)) {
     588 [ #  # ][ #  # ]:          0 :         LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error mempool tx %s already being tracked\n",
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
     589                 :            :                  hash.ToString());
     590                 :          0 :         return;
     591                 :            :     }
     592                 :            : 
     593         [ #  # ]:          0 :     if (txHeight != nBestSeenHeight) {
     594                 :            :         // Ignore side chains and re-orgs; assuming they are random they don't
     595                 :            :         // affect the estimate.  We'll potentially double count transactions in 1-block reorgs.
     596                 :            :         // Ignore txs if BlockPolicyEstimator is not in sync with ActiveChain().Tip().
     597                 :            :         // It will be synced next time a block is processed.
     598                 :          0 :         return;
     599                 :            :     }
     600                 :            : 
     601                 :            :     // Only want to be updating estimates when our blockchain is synced,
     602                 :            :     // otherwise we'll miscalculate how many blocks its taking to get included.
     603         [ #  # ]:          0 :     if (!validFeeEstimate) {
     604                 :          0 :         untrackedTxs++;
     605                 :          0 :         return;
     606                 :            :     }
     607                 :          0 :     trackedTxs++;
     608                 :            : 
     609                 :            :     // Feerates are stored and reported as BTC-per-kb:
     610 [ #  # ][ #  # ]:          0 :     CFeeRate feeRate(entry.GetFee(), entry.GetTxSize());
                 [ #  # ]
     611                 :            : 
     612         [ #  # ]:          0 :     mapMemPoolTxs[hash].blockHeight = txHeight;
     613 [ #  # ][ #  # ]:          0 :     unsigned int bucketIndex = feeStats->NewTx(txHeight, (double)feeRate.GetFeePerK());
     614         [ #  # ]:          0 :     mapMemPoolTxs[hash].bucketIndex = bucketIndex;
     615 [ #  # ][ #  # ]:          0 :     unsigned int bucketIndex2 = shortStats->NewTx(txHeight, (double)feeRate.GetFeePerK());
     616         [ #  # ]:          0 :     assert(bucketIndex == bucketIndex2);
     617 [ #  # ][ #  # ]:          0 :     unsigned int bucketIndex3 = longStats->NewTx(txHeight, (double)feeRate.GetFeePerK());
     618         [ #  # ]:          0 :     assert(bucketIndex == bucketIndex3);
     619         [ #  # ]:          0 : }
     620                 :            : 
     621                 :          0 : bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry)
     622                 :            : {
     623                 :          0 :     AssertLockHeld(m_cs_fee_estimator);
     624         [ #  # ]:          0 :     if (!_removeTx(entry->GetTx().GetHash(), true)) {
     625                 :            :         // This transaction wasn't being tracked for fee estimation
     626                 :          0 :         return false;
     627                 :            :     }
     628                 :            : 
     629                 :            :     // How many blocks did it take for miners to include this transaction?
     630                 :            :     // blocksToConfirm is 1-based, so a transaction included in the earliest
     631                 :            :     // possible block has confirmation count of 1
     632                 :          0 :     int blocksToConfirm = nBlockHeight - entry->GetHeight();
     633         [ #  # ]:          0 :     if (blocksToConfirm <= 0) {
     634                 :            :         // This can't happen because we don't process transactions from a block with a height
     635                 :            :         // lower than our greatest seen height
     636 [ #  # ][ #  # ]:          0 :         LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error Transaction had negative blocksToConfirm\n");
         [ #  # ][ #  # ]
     637                 :          0 :         return false;
     638                 :            :     }
     639                 :            : 
     640                 :            :     // Feerates are stored and reported as BTC-per-kb:
     641                 :          0 :     CFeeRate feeRate(entry->GetFee(), entry->GetTxSize());
     642                 :            : 
     643                 :          0 :     feeStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK());
     644                 :          0 :     shortStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK());
     645                 :          0 :     longStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK());
     646                 :          0 :     return true;
     647                 :          0 : }
     648                 :            : 
     649                 :        201 : void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight,
     650                 :            :                                          std::vector<const CTxMemPoolEntry*>& entries)
     651                 :            : {
     652                 :        201 :     LOCK(m_cs_fee_estimator);
     653         [ +  + ]:        201 :     if (nBlockHeight <= nBestSeenHeight) {
     654                 :            :         // Ignore side chains and re-orgs; assuming they are random
     655                 :            :         // they don't affect the estimate.
     656                 :            :         // And if an attacker can re-org the chain at will, then
     657                 :            :         // you've got much bigger problems than "attacker can influence
     658                 :            :         // transaction fees."
     659                 :          1 :         return;
     660                 :            :     }
     661                 :            : 
     662                 :            :     // Must update nBestSeenHeight in sync with ClearCurrent so that
     663                 :            :     // calls to removeTx (via processBlockTx) correctly calculate age
     664                 :            :     // of unconfirmed txs to remove from tracking.
     665                 :        200 :     nBestSeenHeight = nBlockHeight;
     666                 :            : 
     667                 :            :     // Update unconfirmed circular buffer
     668                 :        200 :     feeStats->ClearCurrent(nBlockHeight);
     669                 :        200 :     shortStats->ClearCurrent(nBlockHeight);
     670                 :        200 :     longStats->ClearCurrent(nBlockHeight);
     671                 :            : 
     672                 :            :     // Decay all exponential averages
     673                 :        200 :     feeStats->UpdateMovingAverages();
     674                 :        200 :     shortStats->UpdateMovingAverages();
     675                 :        200 :     longStats->UpdateMovingAverages();
     676                 :            : 
     677                 :        200 :     unsigned int countedTxs = 0;
     678                 :            :     // Update averages with data points from current block
     679         [ -  + ]:        200 :     for (const auto& entry : entries) {
     680 [ #  # ][ #  # ]:          0 :         if (processBlockTx(nBlockHeight, entry))
     681                 :          0 :             countedTxs++;
     682                 :            :     }
     683                 :            : 
     684 [ +  - ][ +  - ]:        200 :     if (firstRecordedHeight == 0 && countedTxs > 0) {
     685                 :          0 :         firstRecordedHeight = nBestSeenHeight;
     686 [ #  # ][ #  # ]:          0 :         LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy first recorded height %u\n", firstRecordedHeight);
         [ #  # ][ #  # ]
                 [ #  # ]
     687                 :          0 :     }
     688                 :            : 
     689                 :            : 
     690 [ +  - ][ +  - ]:        200 :     LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy estimates updated by %u of %u block txs, since last block %u of %u tracked, mempool map size %u, max target %u from %s\n",
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
     691                 :            :              countedTxs, entries.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size(),
     692                 :            :              MaxUsableEstimate(), HistoricalBlockSpan() > BlockSpan() ? "historical" : "current");
     693                 :            : 
     694                 :        200 :     trackedTxs = 0;
     695                 :        200 :     untrackedTxs = 0;
     696         [ -  + ]:        201 : }
     697                 :            : 
     698                 :          0 : CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) const
     699                 :            : {
     700                 :            :     // It's not possible to get reasonable estimates for confTarget of 1
     701         [ #  # ]:          0 :     if (confTarget <= 1)
     702                 :          0 :         return CFeeRate(0);
     703                 :            : 
     704                 :          0 :     return estimateRawFee(confTarget, DOUBLE_SUCCESS_PCT, FeeEstimateHorizon::MED_HALFLIFE);
     705                 :          0 : }
     706                 :            : 
     707                 :          0 : CFeeRate CBlockPolicyEstimator::estimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult* result) const
     708                 :            : {
     709                 :          0 :     TxConfirmStats* stats = nullptr;
     710                 :          0 :     double sufficientTxs = SUFFICIENT_FEETXS;
     711   [ #  #  #  # ]:          0 :     switch (horizon) {
     712                 :            :     case FeeEstimateHorizon::SHORT_HALFLIFE: {
     713                 :          0 :         stats = shortStats.get();
     714                 :          0 :         sufficientTxs = SUFFICIENT_TXS_SHORT;
     715                 :          0 :         break;
     716                 :            :     }
     717                 :            :     case FeeEstimateHorizon::MED_HALFLIFE: {
     718                 :          0 :         stats = feeStats.get();
     719                 :          0 :         break;
     720                 :            :     }
     721                 :            :     case FeeEstimateHorizon::LONG_HALFLIFE: {
     722                 :          0 :         stats = longStats.get();
     723                 :          0 :         break;
     724                 :            :     }
     725                 :            :     } // no default case, so the compiler can warn about missing cases
     726         [ #  # ]:          0 :     assert(stats);
     727                 :            : 
     728                 :          0 :     LOCK(m_cs_fee_estimator);
     729                 :            :     // Return failure if trying to analyze a target we're not tracking
     730 [ #  # ][ #  # ]:          0 :     if (confTarget <= 0 || (unsigned int)confTarget > stats->GetMaxConfirms())
                 [ #  # ]
     731         [ #  # ]:          0 :         return CFeeRate(0);
     732         [ #  # ]:          0 :     if (successThreshold > 1)
     733         [ #  # ]:          0 :         return CFeeRate(0);
     734                 :            : 
     735         [ #  # ]:          0 :     double median = stats->EstimateMedianVal(confTarget, sufficientTxs, successThreshold, nBestSeenHeight, result);
     736                 :            : 
     737         [ #  # ]:          0 :     if (median < 0)
     738         [ #  # ]:          0 :         return CFeeRate(0);
     739                 :            : 
     740         [ #  # ]:          0 :     return CFeeRate(llround(median));
     741                 :          0 : }
     742                 :            : 
     743                 :          0 : unsigned int CBlockPolicyEstimator::HighestTargetTracked(FeeEstimateHorizon horizon) const
     744                 :            : {
     745                 :          0 :     LOCK(m_cs_fee_estimator);
     746   [ #  #  #  # ]:          0 :     switch (horizon) {
     747                 :            :     case FeeEstimateHorizon::SHORT_HALFLIFE: {
     748         [ #  # ]:          0 :         return shortStats->GetMaxConfirms();
     749                 :            :     }
     750                 :            :     case FeeEstimateHorizon::MED_HALFLIFE: {
     751         [ #  # ]:          0 :         return feeStats->GetMaxConfirms();
     752                 :            :     }
     753                 :            :     case FeeEstimateHorizon::LONG_HALFLIFE: {
     754         [ #  # ]:          0 :         return longStats->GetMaxConfirms();
     755                 :            :     }
     756                 :            :     } // no default case, so the compiler can warn about missing cases
     757                 :          0 :     assert(false);
     758                 :          0 : }
     759                 :            : 
     760                 :          0 : unsigned int CBlockPolicyEstimator::BlockSpan() const
     761                 :            : {
     762         [ #  # ]:          0 :     if (firstRecordedHeight == 0) return 0;
     763         [ #  # ]:          0 :     assert(nBestSeenHeight >= firstRecordedHeight);
     764                 :            : 
     765                 :          0 :     return nBestSeenHeight - firstRecordedHeight;
     766                 :          0 : }
     767                 :            : 
     768                 :          0 : unsigned int CBlockPolicyEstimator::HistoricalBlockSpan() const
     769                 :            : {
     770         [ #  # ]:          0 :     if (historicalFirst == 0) return 0;
     771         [ #  # ]:          0 :     assert(historicalBest >= historicalFirst);
     772                 :            : 
     773         [ #  # ]:          0 :     if (nBestSeenHeight - historicalBest > OLDEST_ESTIMATE_HISTORY) return 0;
     774                 :            : 
     775                 :          0 :     return historicalBest - historicalFirst;
     776                 :          0 : }
     777                 :            : 
     778                 :          0 : unsigned int CBlockPolicyEstimator::MaxUsableEstimate() const
     779                 :            : {
     780                 :            :     // Block spans are divided by 2 to make sure there are enough potential failing data points for the estimate
     781                 :          0 :     return std::min(longStats->GetMaxConfirms(), std::max(BlockSpan(), HistoricalBlockSpan()) / 2);
     782                 :            : }
     783                 :            : 
     784                 :            : /** Return a fee estimate at the required successThreshold from the shortest
     785                 :            :  * time horizon which tracks confirmations up to the desired target.  If
     786                 :            :  * checkShorterHorizon is requested, also allow short time horizon estimates
     787                 :            :  * for a lower target to reduce the given answer */
     788                 :          0 : double CBlockPolicyEstimator::estimateCombinedFee(unsigned int confTarget, double successThreshold, bool checkShorterHorizon, EstimationResult *result) const
     789                 :            : {
     790                 :          0 :     double estimate = -1;
     791 [ #  # ][ #  # ]:          0 :     if (confTarget >= 1 && confTarget <= longStats->GetMaxConfirms()) {
     792                 :            :         // Find estimate from shortest time horizon possible
     793         [ #  # ]:          0 :         if (confTarget <= shortStats->GetMaxConfirms()) { // short horizon
     794                 :          0 :             estimate = shortStats->EstimateMedianVal(confTarget, SUFFICIENT_TXS_SHORT, successThreshold, nBestSeenHeight, result);
     795                 :          0 :         }
     796         [ #  # ]:          0 :         else if (confTarget <= feeStats->GetMaxConfirms()) { // medium horizon
     797                 :          0 :             estimate = feeStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, successThreshold, nBestSeenHeight, result);
     798                 :          0 :         }
     799                 :            :         else { // long horizon
     800                 :          0 :             estimate = longStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, successThreshold, nBestSeenHeight, result);
     801                 :            :         }
     802         [ #  # ]:          0 :         if (checkShorterHorizon) {
     803                 :          0 :             EstimationResult tempResult;
     804                 :            :             // If a lower confTarget from a more recent horizon returns a lower answer use it.
     805         [ #  # ]:          0 :             if (confTarget > feeStats->GetMaxConfirms()) {
     806                 :          0 :                 double medMax = feeStats->EstimateMedianVal(feeStats->GetMaxConfirms(), SUFFICIENT_FEETXS, successThreshold, nBestSeenHeight, &tempResult);
     807 [ #  # ][ #  # ]:          0 :                 if (medMax > 0 && (estimate == -1 || medMax < estimate)) {
                 [ #  # ]
     808                 :          0 :                     estimate = medMax;
     809         [ #  # ]:          0 :                     if (result) *result = tempResult;
     810                 :          0 :                 }
     811                 :          0 :             }
     812         [ #  # ]:          0 :             if (confTarget > shortStats->GetMaxConfirms()) {
     813                 :          0 :                 double shortMax = shortStats->EstimateMedianVal(shortStats->GetMaxConfirms(), SUFFICIENT_TXS_SHORT, successThreshold, nBestSeenHeight, &tempResult);
     814 [ #  # ][ #  # ]:          0 :                 if (shortMax > 0 && (estimate == -1 || shortMax < estimate)) {
                 [ #  # ]
     815                 :          0 :                     estimate = shortMax;
     816         [ #  # ]:          0 :                     if (result) *result = tempResult;
     817                 :          0 :                 }
     818                 :          0 :             }
     819                 :          0 :         }
     820                 :          0 :     }
     821                 :          0 :     return estimate;
     822                 :            : }
     823                 :            : 
     824                 :            : /** Ensure that for a conservative estimate, the DOUBLE_SUCCESS_PCT is also met
     825                 :            :  * at 2 * target for any longer time horizons.
     826                 :            :  */
     827                 :          0 : double CBlockPolicyEstimator::estimateConservativeFee(unsigned int doubleTarget, EstimationResult *result) const
     828                 :            : {
     829                 :          0 :     double estimate = -1;
     830                 :          0 :     EstimationResult tempResult;
     831         [ #  # ]:          0 :     if (doubleTarget <= shortStats->GetMaxConfirms()) {
     832                 :          0 :         estimate = feeStats->EstimateMedianVal(doubleTarget, SUFFICIENT_FEETXS, DOUBLE_SUCCESS_PCT, nBestSeenHeight, result);
     833                 :          0 :     }
     834         [ #  # ]:          0 :     if (doubleTarget <= feeStats->GetMaxConfirms()) {
     835                 :          0 :         double longEstimate = longStats->EstimateMedianVal(doubleTarget, SUFFICIENT_FEETXS, DOUBLE_SUCCESS_PCT, nBestSeenHeight, &tempResult);
     836         [ #  # ]:          0 :         if (longEstimate > estimate) {
     837                 :          0 :             estimate = longEstimate;
     838         [ #  # ]:          0 :             if (result) *result = tempResult;
     839                 :          0 :         }
     840                 :          0 :     }
     841                 :          0 :     return estimate;
     842                 :            : }
     843                 :            : 
     844                 :            : /** estimateSmartFee returns the max of the feerates calculated with a 60%
     845                 :            :  * threshold required at target / 2, an 85% threshold required at target and a
     846                 :            :  * 95% threshold required at 2 * target.  Each calculation is performed at the
     847                 :            :  * shortest time horizon which tracks the required target.  Conservative
     848                 :            :  * estimates, however, required the 95% threshold at 2 * target be met for any
     849                 :            :  * longer time horizons also.
     850                 :            :  */
     851                 :          0 : CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, FeeCalculation *feeCalc, bool conservative) const
     852                 :            : {
     853                 :          0 :     LOCK(m_cs_fee_estimator);
     854                 :            : 
     855         [ #  # ]:          0 :     if (feeCalc) {
     856                 :          0 :         feeCalc->desiredTarget = confTarget;
     857                 :          0 :         feeCalc->returnedTarget = confTarget;
     858                 :          0 :     }
     859                 :            : 
     860                 :          0 :     double median = -1;
     861                 :          0 :     EstimationResult tempResult;
     862                 :            : 
     863                 :            :     // Return failure if trying to analyze a target we're not tracking
     864 [ #  # ][ #  # ]:          0 :     if (confTarget <= 0 || (unsigned int)confTarget > longStats->GetMaxConfirms()) {
                 [ #  # ]
     865         [ #  # ]:          0 :         return CFeeRate(0);  // error condition
     866                 :            :     }
     867                 :            : 
     868                 :            :     // It's not possible to get reasonable estimates for confTarget of 1
     869         [ #  # ]:          0 :     if (confTarget == 1) confTarget = 2;
     870                 :            : 
     871         [ #  # ]:          0 :     unsigned int maxUsableEstimate = MaxUsableEstimate();
     872         [ #  # ]:          0 :     if ((unsigned int)confTarget > maxUsableEstimate) {
     873                 :          0 :         confTarget = maxUsableEstimate;
     874                 :          0 :     }
     875         [ #  # ]:          0 :     if (feeCalc) feeCalc->returnedTarget = confTarget;
     876                 :            : 
     877 [ #  # ][ #  # ]:          0 :     if (confTarget <= 1) return CFeeRate(0); // error condition
     878                 :            : 
     879         [ #  # ]:          0 :     assert(confTarget > 0); //estimateCombinedFee and estimateConservativeFee take unsigned ints
     880                 :            :     /** true is passed to estimateCombined fee for target/2 and target so
     881                 :            :      * that we check the max confirms for shorter time horizons as well.
     882                 :            :      * This is necessary to preserve monotonically increasing estimates.
     883                 :            :      * For non-conservative estimates we do the same thing for 2*target, but
     884                 :            :      * for conservative estimates we want to skip these shorter horizons
     885                 :            :      * checks for 2*target because we are taking the max over all time
     886                 :            :      * horizons so we already have monotonically increasing estimates and
     887                 :            :      * the purpose of conservative estimates is not to let short term
     888                 :            :      * fluctuations lower our estimates by too much.
     889                 :            :      */
     890         [ #  # ]:          0 :     double halfEst = estimateCombinedFee(confTarget/2, HALF_SUCCESS_PCT, true, &tempResult);
     891         [ #  # ]:          0 :     if (feeCalc) {
     892                 :          0 :         feeCalc->est = tempResult;
     893                 :          0 :         feeCalc->reason = FeeReason::HALF_ESTIMATE;
     894                 :          0 :     }
     895                 :          0 :     median = halfEst;
     896         [ #  # ]:          0 :     double actualEst = estimateCombinedFee(confTarget, SUCCESS_PCT, true, &tempResult);
     897         [ #  # ]:          0 :     if (actualEst > median) {
     898                 :          0 :         median = actualEst;
     899         [ #  # ]:          0 :         if (feeCalc) {
     900                 :          0 :             feeCalc->est = tempResult;
     901                 :          0 :             feeCalc->reason = FeeReason::FULL_ESTIMATE;
     902                 :          0 :         }
     903                 :          0 :     }
     904         [ #  # ]:          0 :     double doubleEst = estimateCombinedFee(2 * confTarget, DOUBLE_SUCCESS_PCT, !conservative, &tempResult);
     905         [ #  # ]:          0 :     if (doubleEst > median) {
     906                 :          0 :         median = doubleEst;
     907         [ #  # ]:          0 :         if (feeCalc) {
     908                 :          0 :             feeCalc->est = tempResult;
     909                 :          0 :             feeCalc->reason = FeeReason::DOUBLE_ESTIMATE;
     910                 :          0 :         }
     911                 :          0 :     }
     912                 :            : 
     913 [ #  # ][ #  # ]:          0 :     if (conservative || median == -1) {
     914         [ #  # ]:          0 :         double consEst =  estimateConservativeFee(2 * confTarget, &tempResult);
     915         [ #  # ]:          0 :         if (consEst > median) {
     916                 :          0 :             median = consEst;
     917         [ #  # ]:          0 :             if (feeCalc) {
     918                 :          0 :                 feeCalc->est = tempResult;
     919                 :          0 :                 feeCalc->reason = FeeReason::CONSERVATIVE;
     920                 :          0 :             }
     921                 :          0 :         }
     922                 :          0 :     }
     923                 :            : 
     924 [ #  # ][ #  # ]:          0 :     if (median < 0) return CFeeRate(0); // error condition
     925                 :            : 
     926         [ #  # ]:          0 :     return CFeeRate(llround(median));
     927                 :          0 : }
     928                 :            : 
     929                 :          0 : void CBlockPolicyEstimator::Flush() {
     930                 :          0 :     FlushUnconfirmed();
     931                 :          0 :     FlushFeeEstimates();
     932                 :          0 : }
     933                 :            : 
     934                 :          0 : void CBlockPolicyEstimator::FlushFeeEstimates()
     935                 :            : {
     936         [ #  # ]:          0 :     AutoFile est_file{fsbridge::fopen(m_estimation_filepath, "wb")};
     937 [ #  # ][ #  # ]:          0 :     if (est_file.IsNull() || !Write(est_file)) {
         [ #  # ][ #  # ]
     938 [ #  # ][ #  # ]:          0 :         LogPrintf("Failed to write fee estimates to %s. Continue anyway.\n", fs::PathToString(m_estimation_filepath));
         [ #  # ][ #  # ]
     939                 :          0 :     } else {
     940 [ #  # ][ #  # ]:          0 :         LogPrintf("Flushed fee estimates to %s.\n", fs::PathToString(m_estimation_filepath.filename()));
         [ #  # ][ #  # ]
                 [ #  # ]
     941                 :            :     }
     942                 :          0 : }
     943                 :            : 
     944                 :          0 : bool CBlockPolicyEstimator::Write(AutoFile& fileout) const
     945                 :            : {
     946                 :            :     try {
     947 [ #  # ][ #  # ]:          0 :         LOCK(m_cs_fee_estimator);
     948         [ #  # ]:          0 :         fileout << 149900; // version required to read: 0.14.99 or later
     949         [ #  # ]:          0 :         fileout << CLIENT_VERSION; // version that wrote the file
     950         [ #  # ]:          0 :         fileout << nBestSeenHeight;
     951         [ #  # ]:          0 :         if (BlockSpan() > HistoricalBlockSpan()/2) {
     952 [ #  # ][ #  # ]:          0 :             fileout << firstRecordedHeight << nBestSeenHeight;
     953                 :          0 :         }
     954                 :            :         else {
     955 [ #  # ][ #  # ]:          0 :             fileout << historicalFirst << historicalBest;
     956                 :            :         }
     957 [ #  # ][ #  # ]:          0 :         fileout << Using<VectorFormatter<EncodedDoubleFormatter>>(buckets);
     958         [ #  # ]:          0 :         feeStats->Write(fileout);
     959         [ #  # ]:          0 :         shortStats->Write(fileout);
     960         [ #  # ]:          0 :         longStats->Write(fileout);
     961         [ #  # ]:          0 :     }
     962                 :            :     catch (const std::exception&) {
     963 [ #  # ][ #  # ]:          0 :         LogPrintf("CBlockPolicyEstimator::Write(): unable to write policy estimator data (non-fatal)\n");
                 [ #  # ]
     964                 :          0 :         return false;
     965         [ #  # ]:          0 :     }
     966                 :          0 :     return true;
     967                 :          0 : }
     968                 :            : 
     969                 :          0 : bool CBlockPolicyEstimator::Read(AutoFile& filein)
     970                 :            : {
     971                 :            :     try {
     972 [ #  # ][ #  # ]:          0 :         LOCK(m_cs_fee_estimator);
     973                 :            :         int nVersionRequired, nVersionThatWrote;
     974 [ #  # ][ #  # ]:          0 :         filein >> nVersionRequired >> nVersionThatWrote;
     975         [ #  # ]:          0 :         if (nVersionRequired > CLIENT_VERSION) {
     976 [ #  # ][ #  # ]:          0 :             throw std::runtime_error(strprintf("up-version (%d) fee estimate file", nVersionRequired));
                 [ #  # ]
     977                 :            :         }
     978                 :            : 
     979                 :            :         // Read fee estimates file into temporary variables so existing data
     980                 :            :         // structures aren't corrupted if there is an exception.
     981                 :            :         unsigned int nFileBestSeenHeight;
     982         [ #  # ]:          0 :         filein >> nFileBestSeenHeight;
     983                 :            : 
     984         [ #  # ]:          0 :         if (nVersionRequired < 149900) {
     985 [ #  # ][ #  # ]:          0 :             LogPrintf("%s: incompatible old fee estimation data (non-fatal). Version: %d\n", __func__, nVersionRequired);
                 [ #  # ]
     986                 :          0 :         } else { // New format introduced in 149900
     987                 :            :             unsigned int nFileHistoricalFirst, nFileHistoricalBest;
     988 [ #  # ][ #  # ]:          0 :             filein >> nFileHistoricalFirst >> nFileHistoricalBest;
     989         [ #  # ]:          0 :             if (nFileHistoricalFirst > nFileHistoricalBest || nFileHistoricalBest > nFileBestSeenHeight) {
     990         [ #  # ]:          0 :                 throw std::runtime_error("Corrupt estimates file. Historical block range for estimates is invalid");
     991                 :            :             }
     992                 :          0 :             std::vector<double> fileBuckets;
     993 [ #  # ][ #  # ]:          0 :             filein >> Using<VectorFormatter<EncodedDoubleFormatter>>(fileBuckets);
     994                 :          0 :             size_t numBuckets = fileBuckets.size();
     995         [ #  # ]:          0 :             if (numBuckets <= 1 || numBuckets > 1000) {
     996         [ #  # ]:          0 :                 throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets");
     997                 :            :             }
     998                 :            : 
     999 [ #  # ][ #  # ]:          0 :             std::unique_ptr<TxConfirmStats> fileFeeStats(new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE));
    1000 [ #  # ][ #  # ]:          0 :             std::unique_ptr<TxConfirmStats> fileShortStats(new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE));
    1001 [ #  # ][ #  # ]:          0 :             std::unique_ptr<TxConfirmStats> fileLongStats(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE));
    1002         [ #  # ]:          0 :             fileFeeStats->Read(filein, nVersionThatWrote, numBuckets);
    1003         [ #  # ]:          0 :             fileShortStats->Read(filein, nVersionThatWrote, numBuckets);
    1004         [ #  # ]:          0 :             fileLongStats->Read(filein, nVersionThatWrote, numBuckets);
    1005                 :            : 
    1006                 :            :             // Fee estimates file parsed correctly
    1007                 :            :             // Copy buckets from file and refresh our bucketmap
    1008         [ #  # ]:          0 :             buckets = fileBuckets;
    1009                 :          0 :             bucketMap.clear();
    1010         [ #  # ]:          0 :             for (unsigned int i = 0; i < buckets.size(); i++) {
    1011         [ #  # ]:          0 :                 bucketMap[buckets[i]] = i;
    1012                 :          0 :             }
    1013                 :            : 
    1014                 :            :             // Destroy old TxConfirmStats and point to new ones that already reference buckets and bucketMap
    1015                 :          0 :             feeStats = std::move(fileFeeStats);
    1016                 :          0 :             shortStats = std::move(fileShortStats);
    1017                 :          0 :             longStats = std::move(fileLongStats);
    1018                 :            : 
    1019                 :          0 :             nBestSeenHeight = nFileBestSeenHeight;
    1020                 :          0 :             historicalFirst = nFileHistoricalFirst;
    1021                 :          0 :             historicalBest = nFileHistoricalBest;
    1022                 :          0 :         }
    1023         [ #  # ]:          0 :     }
    1024                 :            :     catch (const std::exception& e) {
    1025 [ #  # ][ #  # ]:          0 :         LogPrintf("CBlockPolicyEstimator::Read(): unable to read policy estimator data (non-fatal): %s\n",e.what());
                 [ #  # ]
    1026                 :          0 :         return false;
    1027         [ #  # ]:          0 :     }
    1028                 :          0 :     return true;
    1029                 :          0 : }
    1030                 :            : 
    1031                 :          0 : void CBlockPolicyEstimator::FlushUnconfirmed()
    1032                 :            : {
    1033                 :          0 :     const auto startclear{SteadyClock::now()};
    1034                 :          0 :     LOCK(m_cs_fee_estimator);
    1035                 :          0 :     size_t num_entries = mapMemPoolTxs.size();
    1036                 :            :     // Remove every entry in mapMemPoolTxs
    1037         [ #  # ]:          0 :     while (!mapMemPoolTxs.empty()) {
    1038                 :          0 :         auto mi = mapMemPoolTxs.begin();
    1039         [ #  # ]:          0 :         _removeTx(mi->first, false); // this calls erase() on mapMemPoolTxs
    1040                 :            :     }
    1041                 :          0 :     const auto endclear{SteadyClock::now()};
    1042 [ #  # ][ #  # ]:          0 :     LogPrint(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %gs\n", num_entries, Ticks<SecondsDouble>(endclear - startclear));
         [ #  # ][ #  # ]
         [ #  # ][ #  # ]
                 [ #  # ]
    1043                 :          0 : }
    1044                 :            : 
    1045                 :          0 : std::chrono::hours CBlockPolicyEstimator::GetFeeEstimatorFileAge()
    1046                 :            : {
    1047                 :          0 :     auto file_time = std::filesystem::last_write_time(m_estimation_filepath);
    1048                 :          0 :     auto now = std::filesystem::file_time_type::clock::now();
    1049                 :          0 :     return std::chrono::duration_cast<std::chrono::hours>(now - file_time);
    1050                 :            : }
    1051                 :            : 
    1052                 :          1 : static std::set<double> MakeFeeSet(const CFeeRate& min_incremental_fee,
    1053                 :            :                                    double max_filter_fee_rate,
    1054                 :            :                                    double fee_filter_spacing)
    1055                 :            : {
    1056                 :          1 :     std::set<double> fee_set;
    1057                 :            : 
    1058         [ +  - ]:          1 :     const CAmount min_fee_limit{std::max(CAmount(1), min_incremental_fee.GetFeePerK() / 2)};
    1059         [ +  - ]:          1 :     fee_set.insert(0);
    1060         [ +  + ]:        105 :     for (double bucket_boundary = min_fee_limit;
    1061                 :        105 :          bucket_boundary <= max_filter_fee_rate;
    1062                 :        104 :          bucket_boundary *= fee_filter_spacing) {
    1063                 :            : 
    1064         [ +  - ]:        104 :         fee_set.insert(bucket_boundary);
    1065                 :        104 :     }
    1066                 :            : 
    1067                 :          1 :     return fee_set;
    1068         [ +  - ]:          1 : }
    1069                 :            : 
    1070                 :          2 : FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee, FastRandomContext& rng)
    1071                 :          1 :     : m_fee_set{MakeFeeSet(minIncrementalFee, MAX_FILTER_FEERATE, FEE_FILTER_SPACING)},
    1072                 :          1 :       insecure_rand{rng}
    1073                 :            : {
    1074                 :          1 : }
    1075                 :            : 
    1076                 :          0 : CAmount FeeFilterRounder::round(CAmount currentMinFee)
    1077                 :            : {
    1078                 :          0 :     AssertLockNotHeld(m_insecure_rand_mutex);
    1079                 :          0 :     std::set<double>::iterator it = m_fee_set.lower_bound(currentMinFee);
    1080 [ #  # ][ #  # ]:          0 :     if (it == m_fee_set.end() ||
    1081         [ #  # ]:          0 :         (it != m_fee_set.begin() &&
    1082                 :          0 :          WITH_LOCK(m_insecure_rand_mutex, return insecure_rand.rand32()) % 3 != 0)) {
    1083                 :          0 :         --it;
    1084                 :          0 :     }
    1085                 :          0 :     return static_cast<CAmount>(*it);
    1086                 :            : }

Generated by: LCOV version 1.14