LCOV - code coverage report
Current view: top level - src/policy - fees.cpp (source / functions) Hit Total Coverage
Test: fuzz_coverage.info Lines: 54 596 9.1 %
Date: 2023-10-05 12:38:51 Functions: 10 44 22.7 %
Branches: 39 686 5.7 %

           Branch data     Line data    Source code
       1                 :            : // Copyright (c) 2009-2010 Satoshi Nakamoto
       2                 :            : // Copyright (c) 2009-2022 The Bitcoin Core developers
       3                 :            : // Distributed under the MIT software license, see the accompanying
       4                 :            : // file COPYING or http://www.opensource.org/licenses/mit-license.php.
       5                 :            : 
       6                 :            : #include <policy/fees.h>
       7                 :            : 
       8                 :            : #include <clientversion.h>
       9                 :            : #include <common/system.h>
      10                 :            : #include <consensus/amount.h>
      11                 :            : #include <kernel/mempool_entry.h>
      12                 :            : #include <logging.h>
      13                 :            : #include <policy/feerate.h>
      14                 :            : #include <primitives/transaction.h>
      15                 :            : #include <random.h>
      16                 :            : #include <serialize.h>
      17         [ +  - ]:          2 : #include <streams.h>
      18         [ +  - ]:          2 : #include <sync.h>
      19                 :            : #include <tinyformat.h>
      20                 :            : #include <uint256.h>
      21                 :            : #include <util/fs.h>
      22                 :            : #include <util/serfloat.h>
      23                 :            : #include <util/time.h>
      24                 :            : 
      25                 :            : #include <algorithm>
      26                 :            : #include <cassert>
      27                 :            : #include <chrono>
      28                 :            : #include <cmath>
      29                 :            : #include <cstddef>
      30                 :            : #include <cstdint>
      31                 :            : #include <exception>
      32                 :            : #include <stdexcept>
      33                 :            : #include <utility>
      34                 :            : 
      35                 :            : static constexpr double INF_FEERATE = 1e99;
      36                 :            : 
      37                 :          0 : std::string StringForFeeEstimateHorizon(FeeEstimateHorizon horizon)
      38                 :            : {
      39   [ #  #  #  # ]:          0 :     switch (horizon) {
      40         [ #  # ]:          0 :     case FeeEstimateHorizon::SHORT_HALFLIFE: return "short";
      41         [ #  # ]:          0 :     case FeeEstimateHorizon::MED_HALFLIFE: return "medium";
      42         [ #  # ]:          0 :     case FeeEstimateHorizon::LONG_HALFLIFE: return "long";
      43                 :            :     } // no default case, so the compiler can warn about missing cases
      44                 :          0 :     assert(false);
      45                 :          0 : }
      46                 :            : 
      47                 :            : namespace {
      48                 :            : 
      49                 :            : struct EncodedDoubleFormatter
      50                 :            : {
      51                 :          0 :     template<typename Stream> void Ser(Stream &s, double v)
      52                 :            :     {
      53                 :          0 :         s << EncodeDouble(v);
      54                 :          0 :     }
      55                 :            : 
      56                 :          0 :     template<typename Stream> void Unser(Stream& s, double& v)
      57                 :            :     {
      58                 :            :         uint64_t encoded;
      59                 :          0 :         s >> encoded;
      60                 :          0 :         v = DecodeDouble(encoded);
      61                 :          0 :     }
      62                 :            : };
      63                 :            : 
      64                 :            : } // namespace
      65                 :            : 
      66                 :            : /**
      67                 :            :  * We will instantiate an instance of this class to track transactions that were
      68                 :            :  * included in a block. We will lump transactions into a bucket according to their
      69                 :            :  * approximate feerate and then track how long it took for those txs to be included in a block
      70                 :            :  *
      71                 :            :  * The tracking of unconfirmed (mempool) transactions is completely independent of the
      72                 :            :  * historical tracking of transactions that have been confirmed in a block.
      73                 :            :  */
      74                 :          2 : class TxConfirmStats
      75                 :            : {
      76                 :            : private:
      77                 :            :     //Define the buckets we will group transactions into
      78                 :            :     const std::vector<double>& buckets;              // The upper-bound of the range for the bucket (inclusive)
      79                 :            :     const std::map<double, unsigned int>& bucketMap; // Map of bucket upper-bound to index into all vectors by bucket
      80                 :            : 
      81                 :            :     // For each bucket X:
      82                 :            :     // Count the total # of txs in each bucket
      83                 :            :     // Track the historical moving average of this total over blocks
      84                 :            :     std::vector<double> txCtAvg;
      85                 :            : 
      86                 :            :     // Count the total # of txs confirmed within Y blocks in each bucket
      87                 :            :     // Track the historical moving average of these totals over blocks
      88                 :            :     std::vector<std::vector<double>> confAvg; // confAvg[Y][X]
      89                 :            : 
      90                 :            :     // Track moving avg of txs which have been evicted from the mempool
      91                 :            :     // after failing to be confirmed within Y blocks
      92                 :            :     std::vector<std::vector<double>> failAvg; // failAvg[Y][X]
      93                 :            : 
      94                 :            :     // Sum the total feerate of all tx's in each bucket
      95                 :            :     // Track the historical moving average of this total over blocks
      96                 :            :     std::vector<double> m_feerate_avg;
      97                 :            : 
      98                 :            :     // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X
      99                 :            :     // Combine the total value with the tx counts to calculate the avg feerate per bucket
     100                 :            : 
     101                 :            :     double decay;
     102                 :            : 
     103                 :            :     // Resolution (# of blocks) with which confirmations are tracked
     104                 :            :     unsigned int scale;
     105                 :            : 
     106                 :            :     // Mempool counts of outstanding transactions
     107                 :            :     // For each bucket X, track the number of transactions in the mempool
     108                 :            :     // that are unconfirmed for each possible confirmation value Y
     109                 :            :     std::vector<std::vector<int> > unconfTxs;  //unconfTxs[Y][X]
     110                 :            :     // transactions still unconfirmed after GetMaxConfirms for each bucket
     111                 :            :     std::vector<int> oldUnconfTxs;
     112                 :            : 
     113                 :            :     void resizeInMemoryCounters(size_t newbuckets);
     114                 :            : 
     115                 :            : public:
     116                 :            :     /**
     117                 :            :      * Create new TxConfirmStats. This is called by BlockPolicyEstimator's
     118                 :            :      * constructor with default values.
     119                 :            :      * @param defaultBuckets contains the upper limits for the bucket boundaries
     120                 :            :      * @param maxPeriods max number of periods to track
     121                 :            :      * @param decay how much to decay the historical moving average per block
     122                 :            :      */
     123                 :            :     TxConfirmStats(const std::vector<double>& defaultBuckets, const std::map<double, unsigned int>& defaultBucketMap,
     124                 :            :                    unsigned int maxPeriods, double decay, unsigned int scale);
     125                 :            : 
     126                 :            :     /** Roll the circular buffer for unconfirmed txs*/
     127                 :            :     void ClearCurrent(unsigned int nBlockHeight);
     128                 :            : 
     129                 :            :     /**
     130                 :            :      * Record a new transaction data point in the current block stats
     131                 :            :      * @param blocksToConfirm the number of blocks it took this transaction to confirm
     132                 :            :      * @param val the feerate of the transaction
     133                 :            :      * @warning blocksToConfirm is 1-based and has to be >= 1
     134                 :            :      */
     135                 :            :     void Record(int blocksToConfirm, double val);
     136                 :            : 
     137                 :            :     /** Record a new transaction entering the mempool*/
     138                 :            :     unsigned int NewTx(unsigned int nBlockHeight, double val);
     139                 :            : 
     140                 :            :     /** Remove a transaction from mempool tracking stats*/
     141                 :            :     void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight,
     142                 :            :                   unsigned int bucketIndex, bool inBlock);
     143                 :            : 
     144                 :            :     /** Update our estimates by decaying our historical moving average and updating
     145                 :            :         with the data gathered from the current block */
     146                 :            :     void UpdateMovingAverages();
     147                 :            : 
     148                 :            :     /**
     149                 :            :      * Calculate a feerate estimate.  Find the lowest value bucket (or range of buckets
     150                 :            :      * to make sure we have enough data points) whose transactions still have sufficient likelihood
     151                 :            :      * of being confirmed within the target number of confirmations
     152                 :            :      * @param confTarget target number of confirmations
     153                 :            :      * @param sufficientTxVal required average number of transactions per block in a bucket range
     154                 :            :      * @param minSuccess the success probability we require
     155                 :            :      * @param nBlockHeight the current block height
     156                 :            :      */
     157                 :            :     double EstimateMedianVal(int confTarget, double sufficientTxVal,
     158                 :            :                              double minSuccess, unsigned int nBlockHeight,
     159                 :            :                              EstimationResult *result = nullptr) const;
     160                 :            : 
     161                 :            :     /** Return the max number of confirms we're tracking */
     162                 :          3 :     unsigned int GetMaxConfirms() const { return scale * confAvg.size(); }
     163                 :            : 
     164                 :            :     /** Write state of estimation data to a file*/
     165                 :            :     void Write(AutoFile& fileout) const;
     166                 :            : 
     167                 :            :     /**
     168                 :            :      * Read saved state of estimation data from a file and replace all internal data structures and
     169                 :            :      * variables with this state.
     170                 :            :      */
     171                 :            :     void Read(AutoFile& filein, int nFileVersion, size_t numBuckets);
     172                 :            : };
     173                 :            : 
     174                 :            : 
     175                 :          9 : TxConfirmStats::TxConfirmStats(const std::vector<double>& defaultBuckets,
     176                 :            :                                 const std::map<double, unsigned int>& defaultBucketMap,
     177                 :            :                                unsigned int maxPeriods, double _decay, unsigned int _scale)
     178                 :          6 :     : buckets(defaultBuckets), bucketMap(defaultBucketMap), decay(_decay), scale(_scale)
     179                 :            : {
     180   [ -  +  +  - ]:          6 :     assert(_scale != 0 && "_scale must be non-zero");
     181         [ +  - ]:          3 :     confAvg.resize(maxPeriods);
     182         [ +  - ]:          3 :     failAvg.resize(maxPeriods);
     183         [ +  + ]:         81 :     for (unsigned int i = 0; i < maxPeriods; i++) {
     184         [ +  - ]:         78 :         confAvg[i].resize(buckets.size());
     185         [ +  - ]:         78 :         failAvg[i].resize(buckets.size());
     186                 :         78 :     }
     187                 :            : 
     188         [ +  - ]:          3 :     txCtAvg.resize(buckets.size());
     189         [ +  - ]:          3 :     m_feerate_avg.resize(buckets.size());
     190                 :            : 
     191         [ +  - ]:          3 :     resizeInMemoryCounters(buckets.size());
     192                 :          3 : }
     193                 :            : 
     194                 :          3 : void TxConfirmStats::resizeInMemoryCounters(size_t newbuckets) {
     195                 :            :     // newbuckets must be passed in because the buckets referred to during Read have not been updated yet.
     196                 :          3 :     unconfTxs.resize(GetMaxConfirms());
     197         [ +  + ]:       1071 :     for (unsigned int i = 0; i < unconfTxs.size(); i++) {
     198                 :       1068 :         unconfTxs[i].resize(newbuckets);
     199                 :       1068 :     }
     200                 :          3 :     oldUnconfTxs.resize(newbuckets);
     201                 :          3 : }
     202                 :            : 
     203                 :            : // Roll the unconfirmed txs circular buffer
     204                 :          0 : void TxConfirmStats::ClearCurrent(unsigned int nBlockHeight)
     205                 :            : {
     206         [ #  # ]:          0 :     for (unsigned int j = 0; j < buckets.size(); j++) {
     207                 :          0 :         oldUnconfTxs[j] += unconfTxs[nBlockHeight % unconfTxs.size()][j];
     208                 :          0 :         unconfTxs[nBlockHeight%unconfTxs.size()][j] = 0;
     209                 :          0 :     }
     210                 :          0 : }
     211                 :            : 
     212                 :            : 
     213                 :          0 : void TxConfirmStats::Record(int blocksToConfirm, double feerate)
     214                 :            : {
     215                 :            :     // blocksToConfirm is 1-based
     216         [ #  # ]:          0 :     if (blocksToConfirm < 1)
     217                 :          0 :         return;
     218                 :          0 :     int periodsToConfirm = (blocksToConfirm + scale - 1) / scale;
     219                 :          0 :     unsigned int bucketindex = bucketMap.lower_bound(feerate)->second;
     220         [ #  # ]:          0 :     for (size_t i = periodsToConfirm; i <= confAvg.size(); i++) {
     221                 :          0 :         confAvg[i - 1][bucketindex]++;
     222                 :          0 :     }
     223                 :          0 :     txCtAvg[bucketindex]++;
     224                 :          0 :     m_feerate_avg[bucketindex] += feerate;
     225                 :          0 : }
     226                 :            : 
     227                 :          0 : void TxConfirmStats::UpdateMovingAverages()
     228                 :            : {
     229         [ #  # ]:          0 :     assert(confAvg.size() == failAvg.size());
     230         [ #  # ]:          0 :     for (unsigned int j = 0; j < buckets.size(); j++) {
     231         [ #  # ]:          0 :         for (unsigned int i = 0; i < confAvg.size(); i++) {
     232                 :          0 :             confAvg[i][j] *= decay;
     233                 :          0 :             failAvg[i][j] *= decay;
     234                 :          0 :         }
     235                 :          0 :         m_feerate_avg[j] *= decay;
     236                 :          0 :         txCtAvg[j] *= decay;
     237                 :          0 :     }
     238                 :          0 : }
     239                 :            : 
     240                 :            : // returns -1 on error conditions
     241                 :          0 : double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal,
     242                 :            :                                          double successBreakPoint, unsigned int nBlockHeight,
     243                 :            :                                          EstimationResult *result) const
     244                 :            : {
     245                 :            :     // Counters for a bucket (or range of buckets)
     246                 :          0 :     double nConf = 0; // Number of tx's confirmed within the confTarget
     247                 :          0 :     double totalNum = 0; // Total number of tx's that were ever confirmed
     248                 :          0 :     int extraNum = 0;  // Number of tx's still in mempool for confTarget or longer
     249                 :          0 :     double failNum = 0; // Number of tx's that were never confirmed but removed from the mempool after confTarget
     250                 :          0 :     const int periodTarget = (confTarget + scale - 1) / scale;
     251                 :          0 :     const int maxbucketindex = buckets.size() - 1;
     252                 :            : 
     253                 :            :     // We'll combine buckets until we have enough samples.
     254                 :            :     // The near and far variables will define the range we've combined
     255                 :            :     // The best variables are the last range we saw which still had a high
     256                 :            :     // enough confirmation rate to count as success.
     257                 :            :     // The cur variables are the current range we're counting.
     258                 :          0 :     unsigned int curNearBucket = maxbucketindex;
     259                 :          0 :     unsigned int bestNearBucket = maxbucketindex;
     260                 :          0 :     unsigned int curFarBucket = maxbucketindex;
     261                 :          0 :     unsigned int bestFarBucket = maxbucketindex;
     262                 :            : 
     263                 :          0 :     bool foundAnswer = false;
     264                 :          0 :     unsigned int bins = unconfTxs.size();
     265                 :          0 :     bool newBucketRange = true;
     266                 :          1 :     bool passing = true;
     267                 :          1 :     EstimatorBucket passBucket;
     268                 :          1 :     EstimatorBucket failBucket;
     269                 :          1 : 
     270                 :            :     // Start counting from highest feerate transactions
     271         [ #  # ]:          0 :     for (int bucket = maxbucketindex; bucket >= 0; --bucket) {
     272         [ #  # ]:          0 :         if (newBucketRange) {
     273                 :          0 :             curNearBucket = bucket;
     274                 :          0 :             newBucketRange = false;
     275                 :          0 :         }
     276                 :          0 :         curFarBucket = bucket;
     277                 :          0 :         nConf += confAvg[periodTarget - 1][bucket];
     278                 :          0 :         totalNum += txCtAvg[bucket];
     279                 :          0 :         failNum += failAvg[periodTarget - 1][bucket];
     280         [ #  # ]:          0 :         for (unsigned int confct = confTarget; confct < GetMaxConfirms(); confct++)
     281                 :          0 :             extraNum += unconfTxs[(nBlockHeight - confct) % bins][bucket];
     282                 :          0 :         extraNum += oldUnconfTxs[bucket];
     283                 :            :         // If we have enough transaction data points in this range of buckets,
     284                 :            :         // we can test for success
     285                 :            :         // (Only count the confirmed data points, so that each confirmation count
     286                 :          1 :         // will be looking at the same amount of data and same bucket breaks)
     287         [ #  # ]:          1 :         if (totalNum >= sufficientTxVal / (1 - decay)) {
     288                 :          0 :             double curPct = nConf / (totalNum + failNum + extraNum);
     289                 :            : 
     290                 :            :             // Check to see if we are no longer getting confirmed at the success rate
     291         [ #  # ]:          0 :             if (curPct < successBreakPoint) {
     292         [ #  # ]:          0 :                 if (passing == true) {
     293                 :            :                     // First time we hit a failure record the failed bucket
     294                 :          0 :                     unsigned int failMinBucket = std::min(curNearBucket, curFarBucket);
     295                 :          0 :                     unsigned int failMaxBucket = std::max(curNearBucket, curFarBucket);
     296         [ #  # ]:          0 :                     failBucket.start = failMinBucket ? buckets[failMinBucket - 1] : 0;
     297                 :          0 :                     failBucket.end = buckets[failMaxBucket];
     298                 :          0 :                     failBucket.withinTarget = nConf;
     299                 :          0 :                     failBucket.totalConfirmed = totalNum;
     300                 :          0 :                     failBucket.inMempool = extraNum;
     301                 :          0 :                     failBucket.leftMempool = failNum;
     302                 :          0 :                     passing = false;
     303                 :          0 :                 }
     304                 :          0 :                 continue;
     305                 :            :             }
     306                 :            :             // Otherwise update the cumulative stats, and the bucket variables
     307                 :            :             // and reset the counters
     308                 :            :             else {
     309                 :          0 :                 failBucket = EstimatorBucket(); // Reset any failed bucket, currently passing
     310                 :          0 :                 foundAnswer = true;
     311                 :          0 :                 passing = true;
     312                 :          0 :                 passBucket.withinTarget = nConf;
     313                 :          0 :                 nConf = 0;
     314                 :          0 :                 passBucket.totalConfirmed = totalNum;
     315                 :          0 :                 totalNum = 0;
     316                 :          0 :                 passBucket.inMempool = extraNum;
     317                 :          0 :                 passBucket.leftMempool = failNum;
     318                 :          0 :                 failNum = 0;
     319                 :          0 :                 extraNum = 0;
     320                 :          0 :                 bestNearBucket = curNearBucket;
     321                 :          0 :                 bestFarBucket = curFarBucket;
     322                 :          0 :                 newBucketRange = true;
     323                 :            :             }
     324                 :          0 :         }
     325                 :          0 :     }
     326                 :            : 
     327                 :          0 :     double median = -1;
     328                 :          0 :     double txSum = 0;
     329                 :            : 
     330                 :            :     // Calculate the "average" feerate of the best bucket range that met success conditions
     331                 :            :     // Find the bucket with the median transaction and then report the average feerate from that bucket
     332                 :            :     // This is a compromise between finding the median which we can't since we don't save all tx's
     333                 :            :     // and reporting the average which is less accurate
     334                 :          0 :     unsigned int minBucket = std::min(bestNearBucket, bestFarBucket);
     335                 :          0 :     unsigned int maxBucket = std::max(bestNearBucket, bestFarBucket);
     336         [ #  # ]:          0 :     for (unsigned int j = minBucket; j <= maxBucket; j++) {
     337                 :          0 :         txSum += txCtAvg[j];
     338                 :          0 :     }
     339   [ #  #  #  # ]:          0 :     if (foundAnswer && txSum != 0) {
     340                 :          0 :         txSum = txSum / 2;
     341         [ #  # ]:          0 :         for (unsigned int j = minBucket; j <= maxBucket; j++) {
     342         [ #  # ]:          0 :             if (txCtAvg[j] < txSum)
     343                 :          0 :                 txSum -= txCtAvg[j];
     344                 :            :             else { // we're in the right bucket
     345                 :          0 :                 median = m_feerate_avg[j] / txCtAvg[j];
     346                 :          0 :                 break;
     347                 :            :             }
     348                 :          0 :         }
     349                 :            : 
     350         [ #  # ]:          0 :         passBucket.start = minBucket ? buckets[minBucket-1] : 0;
     351                 :          0 :         passBucket.end = buckets[maxBucket];
     352                 :          0 :     }
     353                 :            : 
     354                 :            :     // If we were passing until we reached last few buckets with insufficient data, then report those as failed
     355   [ #  #  #  # ]:          0 :     if (passing && !newBucketRange) {
     356                 :          0 :         unsigned int failMinBucket = std::min(curNearBucket, curFarBucket);
     357                 :          0 :         unsigned int failMaxBucket = std::max(curNearBucket, curFarBucket);
     358         [ #  # ]:          0 :         failBucket.start = failMinBucket ? buckets[failMinBucket - 1] : 0;
     359                 :          0 :         failBucket.end = buckets[failMaxBucket];
     360                 :          0 :         failBucket.withinTarget = nConf;
     361                 :          0 :         failBucket.totalConfirmed = totalNum;
     362                 :          0 :         failBucket.inMempool = extraNum;
     363                 :          0 :         failBucket.leftMempool = failNum;
     364                 :          0 :     }
     365                 :            : 
     366                 :          0 :     float passed_within_target_perc = 0.0;
     367                 :          0 :     float failed_within_target_perc = 0.0;
     368         [ #  # ]:          0 :     if ((passBucket.totalConfirmed + passBucket.inMempool + passBucket.leftMempool)) {
     369                 :          0 :         passed_within_target_perc = 100 * passBucket.withinTarget / (passBucket.totalConfirmed + passBucket.inMempool + passBucket.leftMempool);
     370                 :          0 :     }
     371         [ #  # ]:          0 :     if ((failBucket.totalConfirmed + failBucket.inMempool + failBucket.leftMempool)) {
     372                 :          0 :         failed_within_target_perc = 100 * failBucket.withinTarget / (failBucket.totalConfirmed + failBucket.inMempool + failBucket.leftMempool);
     373                 :          0 :     }
     374                 :            : 
     375   [ #  #  #  #  :          0 :     LogPrint(BCLog::ESTIMATEFEE, "FeeEst: %d > %.0f%% decay %.5f: feerate: %g from (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out) Fail: (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out)\n",
             #  #  #  # ]
     376                 :            :              confTarget, 100.0 * successBreakPoint, decay,
     377                 :            :              median, passBucket.start, passBucket.end,
     378                 :            :              passed_within_target_perc,
     379                 :            :              passBucket.withinTarget, passBucket.totalConfirmed, passBucket.inMempool, passBucket.leftMempool,
     380                 :            :              failBucket.start, failBucket.end,
     381                 :            :              failed_within_target_perc,
     382                 :            :              failBucket.withinTarget, failBucket.totalConfirmed, failBucket.inMempool, failBucket.leftMempool);
     383                 :            : 
     384                 :            : 
     385         [ #  # ]:          0 :     if (result) {
     386                 :          0 :         result->pass = passBucket;
     387                 :          0 :         result->fail = failBucket;
     388                 :          0 :         result->decay = decay;
     389                 :          0 :         result->scale = scale;
     390                 :          0 :     }
     391                 :          0 :     return median;
     392                 :          0 : }
     393                 :            : 
     394                 :          0 : void TxConfirmStats::Write(AutoFile& fileout) const
     395                 :            : {
     396                 :          0 :     fileout << Using<EncodedDoubleFormatter>(decay);
     397                 :          0 :     fileout << scale;
     398                 :          0 :     fileout << Using<VectorFormatter<EncodedDoubleFormatter>>(m_feerate_avg);
     399                 :          0 :     fileout << Using<VectorFormatter<EncodedDoubleFormatter>>(txCtAvg);
     400                 :          0 :     fileout << Using<VectorFormatter<VectorFormatter<EncodedDoubleFormatter>>>(confAvg);
     401                 :          0 :     fileout << Using<VectorFormatter<VectorFormatter<EncodedDoubleFormatter>>>(failAvg);
     402                 :          0 : }
     403                 :            : 
     404                 :          0 : void TxConfirmStats::Read(AutoFile& filein, int nFileVersion, size_t numBuckets)
     405                 :            : {
     406                 :            :     // Read data file and do some very basic sanity checking
     407                 :            :     // buckets and bucketMap are not updated yet, so don't access them
     408                 :            :     // If there is a read failure, we'll just discard this entire object anyway
     409                 :            :     size_t maxConfirms, maxPeriods;
     410                 :            : 
     411                 :            :     // The current version will store the decay with each individual TxConfirmStats and also keep a scale factor
     412                 :          0 :     filein >> Using<EncodedDoubleFormatter>(decay);
     413         [ #  # ]:          0 :     if (decay <= 0 || decay >= 1) {
     414         [ #  # ]:          0 :         throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)");
     415                 :            :     }
     416                 :          0 :     filein >> scale;
     417         [ #  # ]:          0 :     if (scale == 0) {
     418         [ #  # ]:          0 :         throw std::runtime_error("Corrupt estimates file. Scale must be non-zero");
     419                 :            :     }
     420                 :            : 
     421                 :          0 :     filein >> Using<VectorFormatter<EncodedDoubleFormatter>>(m_feerate_avg);
     422         [ #  # ]:          0 :     if (m_feerate_avg.size() != numBuckets) {
     423         [ #  # ]:          0 :         throw std::runtime_error("Corrupt estimates file. Mismatch in feerate average bucket count");
     424                 :            :     }
     425                 :          0 :     filein >> Using<VectorFormatter<EncodedDoubleFormatter>>(txCtAvg);
     426         [ #  # ]:          0 :     if (txCtAvg.size() != numBuckets) {
     427         [ #  # ]:          0 :         throw std::runtime_error("Corrupt estimates file. Mismatch in tx count bucket count");
     428                 :            :     }
     429                 :          0 :     filein >> Using<VectorFormatter<VectorFormatter<EncodedDoubleFormatter>>>(confAvg);
     430                 :          0 :     maxPeriods = confAvg.size();
     431                 :          0 :     maxConfirms = scale * maxPeriods;
     432                 :            : 
     433         [ #  # ]:          0 :     if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) { // one week
     434         [ #  # ]:          0 :         throw std::runtime_error("Corrupt estimates file.  Must maintain estimates for between 1 and 1008 (one week) confirms");
     435                 :            :     }
     436         [ #  # ]:          0 :     for (unsigned int i = 0; i < maxPeriods; i++) {
     437         [ #  # ]:          0 :         if (confAvg[i].size() != numBuckets) {
     438         [ #  # ]:          0 :             throw std::runtime_error("Corrupt estimates file. Mismatch in feerate conf average bucket count");
     439                 :            :         }
     440                 :          0 :     }
     441                 :            : 
     442                 :          0 :     filein >> Using<VectorFormatter<VectorFormatter<EncodedDoubleFormatter>>>(failAvg);
     443         [ #  # ]:          0 :     if (maxPeriods != failAvg.size()) {
     444         [ #  # ]:          0 :         throw std::runtime_error("Corrupt estimates file. Mismatch in confirms tracked for failures");
     445                 :            :     }
     446         [ #  # ]:          0 :     for (unsigned int i = 0; i < maxPeriods; i++) {
     447         [ #  # ]:          0 :         if (failAvg[i].size() != numBuckets) {
     448         [ #  # ]:          0 :             throw std::runtime_error("Corrupt estimates file. Mismatch in one of failure average bucket counts");
     449                 :            :         }
     450                 :          0 :     }
     451                 :            : 
     452                 :            :     // Resize the current block variables which aren't stored in the data file
     453                 :            :     // to match the number of confirms and buckets
     454                 :          0 :     resizeInMemoryCounters(numBuckets);
     455                 :            : 
     456   [ #  #  #  #  :          0 :     LogPrint(BCLog::ESTIMATEFEE, "Reading estimates: %u buckets counting confirms up to %u blocks\n",
             #  #  #  # ]
     457                 :            :              numBuckets, maxConfirms);
     458                 :          0 : }
     459                 :            : 
     460                 :          0 : unsigned int TxConfirmStats::NewTx(unsigned int nBlockHeight, double val)
     461                 :            : {
     462                 :          0 :     unsigned int bucketindex = bucketMap.lower_bound(val)->second;
     463                 :          0 :     unsigned int blockIndex = nBlockHeight % unconfTxs.size();
     464                 :          0 :     unconfTxs[blockIndex][bucketindex]++;
     465                 :          0 :     return bucketindex;
     466                 :            : }
     467                 :            : 
     468                 :          0 : void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketindex, bool inBlock)
     469                 :            : {
     470                 :            :     //nBestSeenHeight is not updated yet for the new block
     471                 :          0 :     int blocksAgo = nBestSeenHeight - entryHeight;
     472         [ #  # ]:          0 :     if (nBestSeenHeight == 0)  // the BlockPolicyEstimator hasn't seen any blocks yet
     473                 :          0 :         blocksAgo = 0;
     474         [ #  # ]:          0 :     if (blocksAgo < 0) {
     475   [ #  #  #  #  :          0 :         LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error, blocks ago is negative for mempool tx\n");
             #  #  #  # ]
     476                 :          0 :         return;  //This can't happen because we call this with our best seen height, no entries can have higher
     477                 :            :     }
     478                 :            : 
     479         [ #  # ]:          0 :     if (blocksAgo >= (int)unconfTxs.size()) {
     480         [ #  # ]:          0 :         if (oldUnconfTxs[bucketindex] > 0) {
     481                 :          0 :             oldUnconfTxs[bucketindex]--;
     482                 :          0 :         } else {
     483   [ #  #  #  #  :          0 :             LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error, mempool tx removed from >25 blocks,bucketIndex=%u already\n",
             #  #  #  # ]
     484                 :            :                      bucketindex);
     485                 :            :         }
     486                 :          0 :     }
     487                 :            :     else {
     488                 :          0 :         unsigned int blockIndex = entryHeight % unconfTxs.size();
     489         [ #  # ]:          0 :         if (unconfTxs[blockIndex][bucketindex] > 0) {
     490                 :          0 :             unconfTxs[blockIndex][bucketindex]--;
     491                 :          0 :         } else {
     492   [ #  #  #  #  :          0 :             LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error, mempool tx removed from blockIndex=%u,bucketIndex=%u already\n",
             #  #  #  # ]
     493                 :            :                      blockIndex, bucketindex);
     494                 :            :         }
     495                 :            :     }
     496   [ #  #  #  # ]:          0 :     if (!inBlock && (unsigned int)blocksAgo >= scale) { // Only counts as a failure if not confirmed for entire period
     497         [ #  # ]:          0 :         assert(scale != 0);
     498                 :          0 :         unsigned int periodsAgo = blocksAgo / scale;
     499   [ #  #  #  # ]:          0 :         for (size_t i = 0; i < periodsAgo && i < failAvg.size(); i++) {
     500                 :          0 :             failAvg[i][bucketindex]++;
     501                 :          0 :         }
     502                 :          0 :     }
     503                 :          0 : }
     504                 :            : 
     505                 :            : // This function is called from CTxMemPool::removeUnchecked to ensure
     506                 :            : // txs removed from the mempool for any reason are no longer
     507                 :            : // tracked. Txs that were part of a block have already been removed in
     508                 :            : // processBlockTx to ensure they are never double tracked, but it is
     509                 :            : // of no harm to try to remove them again.
     510                 :          0 : bool CBlockPolicyEstimator::removeTx(uint256 hash, bool inBlock)
     511                 :            : {
     512                 :          0 :     LOCK(m_cs_fee_estimator);
     513         [ #  # ]:          0 :     return _removeTx(hash, inBlock);
     514                 :          0 : }
     515                 :            : 
     516                 :          0 : bool CBlockPolicyEstimator::_removeTx(const uint256& hash, bool inBlock)
     517                 :            : {
     518                 :          0 :     AssertLockHeld(m_cs_fee_estimator);
     519                 :          0 :     std::map<uint256, TxStatsInfo>::iterator pos = mapMemPoolTxs.find(hash);
     520         [ #  # ]:          0 :     if (pos != mapMemPoolTxs.end()) {
     521                 :          0 :         feeStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock);
     522                 :          0 :         shortStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock);
     523                 :          0 :         longStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock);
     524                 :          0 :         mapMemPoolTxs.erase(hash);
     525                 :          0 :         return true;
     526                 :            :     } else {
     527                 :          0 :         return false;
     528                 :            :     }
     529                 :          0 : }
     530                 :            : 
     531                 :          4 : CBlockPolicyEstimator::CBlockPolicyEstimator(const fs::path& estimation_filepath, const bool read_stale_estimates)
     532                 :          1 :     : m_estimation_filepath{estimation_filepath}
     533                 :            : {
     534                 :            :     static_assert(MIN_BUCKET_FEERATE > 0, "Min feerate must be nonzero");
     535                 :          1 :     size_t bucketIndex = 0;
     536                 :            : 
     537         [ +  + ]:        190 :     for (double bucketBoundary = MIN_BUCKET_FEERATE; bucketBoundary <= MAX_BUCKET_FEERATE; bucketBoundary *= FEE_SPACING, bucketIndex++) {
     538         [ +  - ]:        189 :         buckets.push_back(bucketBoundary);
     539         [ +  - ]:        189 :         bucketMap[bucketBoundary] = bucketIndex;
     540                 :        189 :     }
     541         [ +  - ]:          1 :     buckets.push_back(INF_FEERATE);
     542         [ +  - ]:          1 :     bucketMap[INF_FEERATE] = bucketIndex;
     543         [ +  - ]:          1 :     assert(bucketMap.size() == buckets.size());
     544                 :            : 
     545   [ +  -  +  - ]:          1 :     feeStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE));
     546   [ +  -  +  - ]:          1 :     shortStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE));
     547   [ +  -  +  - ]:          1 :     longStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE));
     548                 :            : 
     549   [ +  -  +  - ]:          1 :     AutoFile est_file{fsbridge::fopen(m_estimation_filepath, "rb")};
     550                 :            : 
     551   [ +  -  -  + ]:          1 :     if (est_file.IsNull()) {
     552   [ +  -  +  -  :          1 :         LogPrintf("%s is not found. Continue anyway.\n", fs::PathToString(m_estimation_filepath));
             +  -  +  - ]
     553                 :          1 :         return;
     554                 :            :     }
     555                 :            : 
     556         [ #  # ]:          0 :     std::chrono::hours file_age = GetFeeEstimatorFileAge();
     557   [ #  #  #  #  :          0 :     if (file_age > MAX_FILE_AGE && !read_stale_estimates) {
                   #  # ]
     558   [ #  #  #  #  :          0 :         LogPrintf("Fee estimation file %s too old (age=%lld > %lld hours) and will not be used to avoid serving stale estimates.\n", fs::PathToString(m_estimation_filepath), Ticks<std::chrono::hours>(file_age), Ticks<std::chrono::hours>(MAX_FILE_AGE));
          #  #  #  #  #  
                #  #  # ]
     559                 :          0 :         return;
     560                 :            :     }
     561                 :            : 
     562   [ #  #  #  # ]:          0 :     if (!Read(est_file)) {
     563   [ #  #  #  #  :          0 :         LogPrintf("Failed to read fee estimates from %s. Continue anyway.\n", fs::PathToString(m_estimation_filepath));
             #  #  #  # ]
     564                 :          0 :     }
     565         [ -  + ]:          1 : }
     566                 :            : 
     567                 :          1 : CBlockPolicyEstimator::~CBlockPolicyEstimator() = default;
     568                 :            : 
     569                 :          0 : void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate)
     570                 :            : {
     571                 :          0 :     LOCK(m_cs_fee_estimator);
     572         [ #  # ]:          0 :     unsigned int txHeight = entry.GetHeight();
     573   [ #  #  #  # ]:          0 :     uint256 hash = entry.GetTx().GetHash();
     574   [ #  #  #  # ]:          0 :     if (mapMemPoolTxs.count(hash)) {
     575   [ #  #  #  #  :          0 :         LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error mempool tx %s already being tracked\n",
          #  #  #  #  #  
                #  #  # ]
     576                 :            :                  hash.ToString());
     577                 :          0 :         return;
     578                 :            :     }
     579                 :            : 
     580         [ #  # ]:          0 :     if (txHeight != nBestSeenHeight) {
     581                 :            :         // Ignore side chains and re-orgs; assuming they are random they don't
     582                 :            :         // affect the estimate.  We'll potentially double count transactions in 1-block reorgs.
     583                 :            :         // Ignore txs if BlockPolicyEstimator is not in sync with ActiveChain().Tip().
     584                 :            :         // It will be synced next time a block is processed.
     585                 :          0 :         return;
     586                 :            :     }
     587                 :            : 
     588                 :            :     // Only want to be updating estimates when our blockchain is synced,
     589                 :            :     // otherwise we'll miscalculate how many blocks its taking to get included.
     590         [ #  # ]:          0 :     if (!validFeeEstimate) {
     591                 :          0 :         untrackedTxs++;
     592                 :          0 :         return;
     593                 :            :     }
     594                 :          0 :     trackedTxs++;
     595                 :            : 
     596                 :            :     // Feerates are stored and reported as BTC-per-kb:
     597   [ #  #  #  #  :          0 :     CFeeRate feeRate(entry.GetFee(), entry.GetTxSize());
                   #  # ]
     598                 :            : 
     599         [ #  # ]:          0 :     mapMemPoolTxs[hash].blockHeight = txHeight;
     600   [ #  #  #  # ]:          0 :     unsigned int bucketIndex = feeStats->NewTx(txHeight, (double)feeRate.GetFeePerK());
     601         [ #  # ]:          0 :     mapMemPoolTxs[hash].bucketIndex = bucketIndex;
     602   [ #  #  #  # ]:          0 :     unsigned int bucketIndex2 = shortStats->NewTx(txHeight, (double)feeRate.GetFeePerK());
     603         [ #  # ]:          0 :     assert(bucketIndex == bucketIndex2);
     604   [ #  #  #  # ]:          0 :     unsigned int bucketIndex3 = longStats->NewTx(txHeight, (double)feeRate.GetFeePerK());
     605         [ #  # ]:          0 :     assert(bucketIndex == bucketIndex3);
     606         [ #  # ]:          0 : }
     607                 :            : 
     608                 :          0 : bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry)
     609                 :            : {
     610                 :          0 :     AssertLockHeld(m_cs_fee_estimator);
     611         [ #  # ]:          0 :     if (!_removeTx(entry->GetTx().GetHash(), true)) {
     612                 :            :         // This transaction wasn't being tracked for fee estimation
     613                 :          0 :         return false;
     614                 :            :     }
     615                 :            : 
     616                 :            :     // How many blocks did it take for miners to include this transaction?
     617                 :            :     // blocksToConfirm is 1-based, so a transaction included in the earliest
     618                 :            :     // possible block has confirmation count of 1
     619                 :          0 :     int blocksToConfirm = nBlockHeight - entry->GetHeight();
     620         [ #  # ]:          0 :     if (blocksToConfirm <= 0) {
     621                 :            :         // This can't happen because we don't process transactions from a block with a height
     622                 :            :         // lower than our greatest seen height
     623   [ #  #  #  #  :          0 :         LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error Transaction had negative blocksToConfirm\n");
             #  #  #  # ]
     624                 :          0 :         return false;
     625                 :            :     }
     626                 :            : 
     627                 :            :     // Feerates are stored and reported as BTC-per-kb:
     628                 :          0 :     CFeeRate feeRate(entry->GetFee(), entry->GetTxSize());
     629                 :            : 
     630                 :          0 :     feeStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK());
     631                 :          0 :     shortStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK());
     632                 :          0 :     longStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK());
     633                 :          0 :     return true;
     634                 :          0 : }
     635                 :            : 
     636                 :          1 : void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight,
     637                 :            :                                          std::vector<const CTxMemPoolEntry*>& entries)
     638                 :            : {
     639                 :          1 :     LOCK(m_cs_fee_estimator);
     640         [ +  - ]:          1 :     if (nBlockHeight <= nBestSeenHeight) {
     641                 :            :         // Ignore side chains and re-orgs; assuming they are random
     642                 :            :         // they don't affect the estimate.
     643                 :            :         // And if an attacker can re-org the chain at will, then
     644                 :            :         // you've got much bigger problems than "attacker can influence
     645                 :            :         // transaction fees."
     646                 :          1 :         return;
     647                 :            :     }
     648                 :            : 
     649                 :            :     // Must update nBestSeenHeight in sync with ClearCurrent so that
     650                 :            :     // calls to removeTx (via processBlockTx) correctly calculate age
     651                 :            :     // of unconfirmed txs to remove from tracking.
     652                 :          0 :     nBestSeenHeight = nBlockHeight;
     653                 :            : 
     654                 :            :     // Update unconfirmed circular buffer
     655                 :          0 :     feeStats->ClearCurrent(nBlockHeight);
     656                 :          0 :     shortStats->ClearCurrent(nBlockHeight);
     657                 :          0 :     longStats->ClearCurrent(nBlockHeight);
     658                 :            : 
     659                 :            :     // Decay all exponential averages
     660                 :          0 :     feeStats->UpdateMovingAverages();
     661                 :          0 :     shortStats->UpdateMovingAverages();
     662                 :          0 :     longStats->UpdateMovingAverages();
     663                 :            : 
     664                 :          0 :     unsigned int countedTxs = 0;
     665                 :            :     // Update averages with data points from current block
     666         [ #  # ]:          0 :     for (const auto& entry : entries) {
     667   [ #  #  #  # ]:          0 :         if (processBlockTx(nBlockHeight, entry))
     668                 :          0 :             countedTxs++;
     669                 :            :     }
     670                 :            : 
     671   [ #  #  #  # ]:          0 :     if (firstRecordedHeight == 0 && countedTxs > 0) {
     672                 :          0 :         firstRecordedHeight = nBestSeenHeight;
     673   [ #  #  #  #  :          0 :         LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy first recorded height %u\n", firstRecordedHeight);
          #  #  #  #  #  
                      # ]
     674                 :          0 :     }
     675                 :            : 
     676                 :            : 
     677   [ #  #  #  #  :          0 :     LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy estimates updated by %u of %u block txs, since last block %u of %u tracked, mempool map size %u, max target %u from %s\n",
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
     678                 :            :              countedTxs, entries.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size(),
     679                 :            :              MaxUsableEstimate(), HistoricalBlockSpan() > BlockSpan() ? "historical" : "current");
     680                 :            : 
     681                 :          0 :     trackedTxs = 0;
     682                 :          0 :     untrackedTxs = 0;
     683         [ -  + ]:          1 : }
     684                 :            : 
     685                 :          0 : CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) const
     686                 :            : {
     687                 :            :     // It's not possible to get reasonable estimates for confTarget of 1
     688         [ #  # ]:          0 :     if (confTarget <= 1)
     689                 :          0 :         return CFeeRate(0);
     690                 :            : 
     691                 :          0 :     return estimateRawFee(confTarget, DOUBLE_SUCCESS_PCT, FeeEstimateHorizon::MED_HALFLIFE);
     692                 :          0 : }
     693                 :            : 
     694                 :          0 : CFeeRate CBlockPolicyEstimator::estimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult* result) const
     695                 :            : {
     696                 :          0 :     TxConfirmStats* stats = nullptr;
     697                 :          0 :     double sufficientTxs = SUFFICIENT_FEETXS;
     698   [ #  #  #  # ]:          0 :     switch (horizon) {
     699                 :            :     case FeeEstimateHorizon::SHORT_HALFLIFE: {
     700                 :          0 :         stats = shortStats.get();
     701                 :          0 :         sufficientTxs = SUFFICIENT_TXS_SHORT;
     702                 :          0 :         break;
     703                 :            :     }
     704                 :            :     case FeeEstimateHorizon::MED_HALFLIFE: {
     705                 :          0 :         stats = feeStats.get();
     706                 :          0 :         break;
     707                 :            :     }
     708                 :            :     case FeeEstimateHorizon::LONG_HALFLIFE: {
     709                 :          0 :         stats = longStats.get();
     710                 :          0 :         break;
     711                 :            :     }
     712                 :            :     } // no default case, so the compiler can warn about missing cases
     713         [ #  # ]:          0 :     assert(stats);
     714                 :            : 
     715                 :          0 :     LOCK(m_cs_fee_estimator);
     716                 :            :     // Return failure if trying to analyze a target we're not tracking
     717   [ #  #  #  #  :          0 :     if (confTarget <= 0 || (unsigned int)confTarget > stats->GetMaxConfirms())
                   #  # ]
     718         [ #  # ]:          0 :         return CFeeRate(0);
     719         [ #  # ]:          0 :     if (successThreshold > 1)
     720         [ #  # ]:          0 :         return CFeeRate(0);
     721                 :            : 
     722         [ #  # ]:          0 :     double median = stats->EstimateMedianVal(confTarget, sufficientTxs, successThreshold, nBestSeenHeight, result);
     723                 :            : 
     724         [ #  # ]:          0 :     if (median < 0)
     725         [ #  # ]:          0 :         return CFeeRate(0);
     726                 :            : 
     727         [ #  # ]:          0 :     return CFeeRate(llround(median));
     728                 :          0 : }
     729                 :            : 
     730                 :          0 : unsigned int CBlockPolicyEstimator::HighestTargetTracked(FeeEstimateHorizon horizon) const
     731                 :            : {
     732                 :          0 :     LOCK(m_cs_fee_estimator);
     733   [ #  #  #  # ]:          0 :     switch (horizon) {
     734                 :            :     case FeeEstimateHorizon::SHORT_HALFLIFE: {
     735         [ #  # ]:          0 :         return shortStats->GetMaxConfirms();
     736                 :            :     }
     737                 :            :     case FeeEstimateHorizon::MED_HALFLIFE: {
     738         [ #  # ]:          0 :         return feeStats->GetMaxConfirms();
     739                 :            :     }
     740                 :            :     case FeeEstimateHorizon::LONG_HALFLIFE: {
     741         [ #  # ]:          0 :         return longStats->GetMaxConfirms();
     742                 :            :     }
     743                 :            :     } // no default case, so the compiler can warn about missing cases
     744                 :          0 :     assert(false);
     745                 :          0 : }
     746                 :            : 
     747                 :          0 : unsigned int CBlockPolicyEstimator::BlockSpan() const
     748                 :            : {
     749         [ #  # ]:          0 :     if (firstRecordedHeight == 0) return 0;
     750         [ #  # ]:          0 :     assert(nBestSeenHeight >= firstRecordedHeight);
     751                 :            : 
     752                 :          0 :     return nBestSeenHeight - firstRecordedHeight;
     753                 :          0 : }
     754                 :            : 
     755                 :          0 : unsigned int CBlockPolicyEstimator::HistoricalBlockSpan() const
     756                 :            : {
     757         [ #  # ]:          0 :     if (historicalFirst == 0) return 0;
     758         [ #  # ]:          0 :     assert(historicalBest >= historicalFirst);
     759                 :            : 
     760         [ #  # ]:          0 :     if (nBestSeenHeight - historicalBest > OLDEST_ESTIMATE_HISTORY) return 0;
     761                 :            : 
     762                 :          0 :     return historicalBest - historicalFirst;
     763                 :          0 : }
     764                 :            : 
     765                 :          0 : unsigned int CBlockPolicyEstimator::MaxUsableEstimate() const
     766                 :            : {
     767                 :            :     // Block spans are divided by 2 to make sure there are enough potential failing data points for the estimate
     768                 :          0 :     return std::min(longStats->GetMaxConfirms(), std::max(BlockSpan(), HistoricalBlockSpan()) / 2);
     769                 :            : }
     770                 :            : 
     771                 :            : /** Return a fee estimate at the required successThreshold from the shortest
     772                 :            :  * time horizon which tracks confirmations up to the desired target.  If
     773                 :            :  * checkShorterHorizon is requested, also allow short time horizon estimates
     774                 :            :  * for a lower target to reduce the given answer */
     775                 :          0 : double CBlockPolicyEstimator::estimateCombinedFee(unsigned int confTarget, double successThreshold, bool checkShorterHorizon, EstimationResult *result) const
     776                 :            : {
     777                 :          0 :     double estimate = -1;
     778   [ #  #  #  # ]:          0 :     if (confTarget >= 1 && confTarget <= longStats->GetMaxConfirms()) {
     779                 :            :         // Find estimate from shortest time horizon possible
     780         [ #  # ]:          0 :         if (confTarget <= shortStats->GetMaxConfirms()) { // short horizon
     781                 :          0 :             estimate = shortStats->EstimateMedianVal(confTarget, SUFFICIENT_TXS_SHORT, successThreshold, nBestSeenHeight, result);
     782                 :          0 :         }
     783         [ #  # ]:          0 :         else if (confTarget <= feeStats->GetMaxConfirms()) { // medium horizon
     784                 :          0 :             estimate = feeStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, successThreshold, nBestSeenHeight, result);
     785                 :          0 :         }
     786                 :            :         else { // long horizon
     787                 :          0 :             estimate = longStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, successThreshold, nBestSeenHeight, result);
     788                 :            :         }
     789         [ #  # ]:          0 :         if (checkShorterHorizon) {
     790                 :          0 :             EstimationResult tempResult;
     791                 :            :             // If a lower confTarget from a more recent horizon returns a lower answer use it.
     792         [ #  # ]:          0 :             if (confTarget > feeStats->GetMaxConfirms()) {
     793                 :          0 :                 double medMax = feeStats->EstimateMedianVal(feeStats->GetMaxConfirms(), SUFFICIENT_FEETXS, successThreshold, nBestSeenHeight, &tempResult);
     794   [ #  #  #  #  :          0 :                 if (medMax > 0 && (estimate == -1 || medMax < estimate)) {
                   #  # ]
     795                 :          0 :                     estimate = medMax;
     796         [ #  # ]:          0 :                     if (result) *result = tempResult;
     797                 :          0 :                 }
     798                 :          0 :             }
     799         [ #  # ]:          0 :             if (confTarget > shortStats->GetMaxConfirms()) {
     800                 :          0 :                 double shortMax = shortStats->EstimateMedianVal(shortStats->GetMaxConfirms(), SUFFICIENT_TXS_SHORT, successThreshold, nBestSeenHeight, &tempResult);
     801   [ #  #  #  #  :          0 :                 if (shortMax > 0 && (estimate == -1 || shortMax < estimate)) {
                   #  # ]
     802                 :          0 :                     estimate = shortMax;
     803         [ #  # ]:          0 :                     if (result) *result = tempResult;
     804                 :          0 :                 }
     805                 :          0 :             }
     806                 :          0 :         }
     807                 :          0 :     }
     808                 :          0 :     return estimate;
     809                 :            : }
     810                 :            : 
     811                 :            : /** Ensure that for a conservative estimate, the DOUBLE_SUCCESS_PCT is also met
     812                 :            :  * at 2 * target for any longer time horizons.
     813                 :            :  */
     814                 :          0 : double CBlockPolicyEstimator::estimateConservativeFee(unsigned int doubleTarget, EstimationResult *result) const
     815                 :            : {
     816                 :          0 :     double estimate = -1;
     817                 :          0 :     EstimationResult tempResult;
     818         [ #  # ]:          0 :     if (doubleTarget <= shortStats->GetMaxConfirms()) {
     819                 :          0 :         estimate = feeStats->EstimateMedianVal(doubleTarget, SUFFICIENT_FEETXS, DOUBLE_SUCCESS_PCT, nBestSeenHeight, result);
     820                 :          0 :     }
     821         [ #  # ]:          0 :     if (doubleTarget <= feeStats->GetMaxConfirms()) {
     822                 :          0 :         double longEstimate = longStats->EstimateMedianVal(doubleTarget, SUFFICIENT_FEETXS, DOUBLE_SUCCESS_PCT, nBestSeenHeight, &tempResult);
     823         [ #  # ]:          0 :         if (longEstimate > estimate) {
     824                 :          0 :             estimate = longEstimate;
     825         [ #  # ]:          0 :             if (result) *result = tempResult;
     826                 :          0 :         }
     827                 :          0 :     }
     828                 :          0 :     return estimate;
     829                 :            : }
     830                 :            : 
     831                 :            : /** estimateSmartFee returns the max of the feerates calculated with a 60%
     832                 :            :  * threshold required at target / 2, an 85% threshold required at target and a
     833                 :            :  * 95% threshold required at 2 * target.  Each calculation is performed at the
     834                 :            :  * shortest time horizon which tracks the required target.  Conservative
     835                 :            :  * estimates, however, required the 95% threshold at 2 * target be met for any
     836                 :            :  * longer time horizons also.
     837                 :            :  */
     838                 :          0 : CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, FeeCalculation *feeCalc, bool conservative) const
     839                 :            : {
     840                 :          0 :     LOCK(m_cs_fee_estimator);
     841                 :            : 
     842         [ #  # ]:          0 :     if (feeCalc) {
     843                 :          0 :         feeCalc->desiredTarget = confTarget;
     844                 :          0 :         feeCalc->returnedTarget = confTarget;
     845                 :          0 :     }
     846                 :            : 
     847                 :          0 :     double median = -1;
     848                 :          0 :     EstimationResult tempResult;
     849                 :            : 
     850                 :            :     // Return failure if trying to analyze a target we're not tracking
     851   [ #  #  #  #  :          0 :     if (confTarget <= 0 || (unsigned int)confTarget > longStats->GetMaxConfirms()) {
                   #  # ]
     852         [ #  # ]:          0 :         return CFeeRate(0);  // error condition
     853                 :            :     }
     854                 :            : 
     855                 :            :     // It's not possible to get reasonable estimates for confTarget of 1
     856         [ #  # ]:          0 :     if (confTarget == 1) confTarget = 2;
     857                 :            : 
     858         [ #  # ]:          0 :     unsigned int maxUsableEstimate = MaxUsableEstimate();
     859         [ #  # ]:          0 :     if ((unsigned int)confTarget > maxUsableEstimate) {
     860                 :          0 :         confTarget = maxUsableEstimate;
     861                 :          0 :     }
     862         [ #  # ]:          0 :     if (feeCalc) feeCalc->returnedTarget = confTarget;
     863                 :            : 
     864   [ #  #  #  # ]:          0 :     if (confTarget <= 1) return CFeeRate(0); // error condition
     865                 :            : 
     866         [ #  # ]:          0 :     assert(confTarget > 0); //estimateCombinedFee and estimateConservativeFee take unsigned ints
     867                 :            :     /** true is passed to estimateCombined fee for target/2 and target so
     868                 :            :      * that we check the max confirms for shorter time horizons as well.
     869                 :            :      * This is necessary to preserve monotonically increasing estimates.
     870                 :            :      * For non-conservative estimates we do the same thing for 2*target, but
     871                 :            :      * for conservative estimates we want to skip these shorter horizons
     872                 :            :      * checks for 2*target because we are taking the max over all time
     873                 :            :      * horizons so we already have monotonically increasing estimates and
     874                 :            :      * the purpose of conservative estimates is not to let short term
     875                 :            :      * fluctuations lower our estimates by too much.
     876                 :            :      */
     877         [ #  # ]:          0 :     double halfEst = estimateCombinedFee(confTarget/2, HALF_SUCCESS_PCT, true, &tempResult);
     878         [ #  # ]:          0 :     if (feeCalc) {
     879                 :          0 :         feeCalc->est = tempResult;
     880                 :          0 :         feeCalc->reason = FeeReason::HALF_ESTIMATE;
     881                 :          0 :     }
     882                 :          0 :     median = halfEst;
     883         [ #  # ]:          0 :     double actualEst = estimateCombinedFee(confTarget, SUCCESS_PCT, true, &tempResult);
     884         [ #  # ]:          0 :     if (actualEst > median) {
     885                 :          0 :         median = actualEst;
     886         [ #  # ]:          0 :         if (feeCalc) {
     887                 :          0 :             feeCalc->est = tempResult;
     888                 :          0 :             feeCalc->reason = FeeReason::FULL_ESTIMATE;
     889                 :          0 :         }
     890                 :          0 :     }
     891         [ #  # ]:          0 :     double doubleEst = estimateCombinedFee(2 * confTarget, DOUBLE_SUCCESS_PCT, !conservative, &tempResult);
     892         [ #  # ]:          0 :     if (doubleEst > median) {
     893                 :          0 :         median = doubleEst;
     894         [ #  # ]:          0 :         if (feeCalc) {
     895                 :          0 :             feeCalc->est = tempResult;
     896                 :          0 :             feeCalc->reason = FeeReason::DOUBLE_ESTIMATE;
     897                 :          0 :         }
     898                 :          0 :     }
     899                 :            : 
     900   [ #  #  #  # ]:          0 :     if (conservative || median == -1) {
     901         [ #  # ]:          0 :         double consEst =  estimateConservativeFee(2 * confTarget, &tempResult);
     902         [ #  # ]:          0 :         if (consEst > median) {
     903                 :          0 :             median = consEst;
     904         [ #  # ]:          0 :             if (feeCalc) {
     905                 :          0 :                 feeCalc->est = tempResult;
     906                 :          0 :                 feeCalc->reason = FeeReason::CONSERVATIVE;
     907                 :          0 :             }
     908                 :          0 :         }
     909                 :          0 :     }
     910                 :            : 
     911   [ #  #  #  # ]:          0 :     if (median < 0) return CFeeRate(0); // error condition
     912                 :            : 
     913         [ #  # ]:          0 :     return CFeeRate(llround(median));
     914                 :          0 : }
     915                 :            : 
     916                 :          0 : void CBlockPolicyEstimator::Flush() {
     917                 :          0 :     FlushUnconfirmed();
     918                 :          0 :     FlushFeeEstimates();
     919                 :          0 : }
     920                 :            : 
     921                 :          0 : void CBlockPolicyEstimator::FlushFeeEstimates()
     922                 :            : {
     923         [ #  # ]:          0 :     AutoFile est_file{fsbridge::fopen(m_estimation_filepath, "wb")};
     924   [ #  #  #  #  :          0 :     if (est_file.IsNull() || !Write(est_file)) {
             #  #  #  # ]
     925   [ #  #  #  #  :          0 :         LogPrintf("Failed to write fee estimates to %s. Continue anyway.\n", fs::PathToString(m_estimation_filepath));
             #  #  #  # ]
     926                 :          0 :     } else {
     927   [ #  #  #  #  :          0 :         LogPrintf("Flushed fee estimates to %s.\n", fs::PathToString(m_estimation_filepath.filename()));
          #  #  #  #  #  
                      # ]
     928                 :            :     }
     929                 :          0 : }
     930                 :            : 
     931                 :          0 : bool CBlockPolicyEstimator::Write(AutoFile& fileout) const
     932                 :            : {
     933                 :            :     try {
     934   [ #  #  #  # ]:          0 :         LOCK(m_cs_fee_estimator);
     935         [ #  # ]:          0 :         fileout << 149900; // version required to read: 0.14.99 or later
     936         [ #  # ]:          0 :         fileout << CLIENT_VERSION; // version that wrote the file
     937         [ #  # ]:          0 :         fileout << nBestSeenHeight;
     938         [ #  # ]:          0 :         if (BlockSpan() > HistoricalBlockSpan()/2) {
     939   [ #  #  #  # ]:          0 :             fileout << firstRecordedHeight << nBestSeenHeight;
     940                 :          0 :         }
     941                 :            :         else {
     942   [ #  #  #  # ]:          0 :             fileout << historicalFirst << historicalBest;
     943                 :            :         }
     944   [ #  #  #  # ]:          0 :         fileout << Using<VectorFormatter<EncodedDoubleFormatter>>(buckets);
     945         [ #  # ]:          0 :         feeStats->Write(fileout);
     946         [ #  # ]:          0 :         shortStats->Write(fileout);
     947         [ #  # ]:          0 :         longStats->Write(fileout);
     948         [ #  # ]:          0 :     }
     949                 :            :     catch (const std::exception&) {
     950   [ #  #  #  #  :          0 :         LogPrintf("CBlockPolicyEstimator::Write(): unable to write policy estimator data (non-fatal)\n");
                   #  # ]
     951                 :          0 :         return false;
     952         [ #  # ]:          0 :     }
     953                 :          0 :     return true;
     954                 :          0 : }
     955                 :            : 
     956                 :          0 : bool CBlockPolicyEstimator::Read(AutoFile& filein)
     957                 :            : {
     958                 :            :     try {
     959   [ #  #  #  # ]:          0 :         LOCK(m_cs_fee_estimator);
     960                 :            :         int nVersionRequired, nVersionThatWrote;
     961   [ #  #  #  # ]:          0 :         filein >> nVersionRequired >> nVersionThatWrote;
     962         [ #  # ]:          0 :         if (nVersionRequired > CLIENT_VERSION) {
     963   [ #  #  #  #  :          0 :             throw std::runtime_error(strprintf("up-version (%d) fee estimate file", nVersionRequired));
                   #  # ]
     964                 :            :         }
     965                 :            : 
     966                 :            :         // Read fee estimates file into temporary variables so existing data
     967                 :            :         // structures aren't corrupted if there is an exception.
     968                 :            :         unsigned int nFileBestSeenHeight;
     969         [ #  # ]:          0 :         filein >> nFileBestSeenHeight;
     970                 :            : 
     971         [ #  # ]:          0 :         if (nVersionRequired < 149900) {
     972   [ #  #  #  #  :          0 :             LogPrintf("%s: incompatible old fee estimation data (non-fatal). Version: %d\n", __func__, nVersionRequired);
                   #  # ]
     973                 :          0 :         } else { // New format introduced in 149900
     974                 :            :             unsigned int nFileHistoricalFirst, nFileHistoricalBest;
     975   [ #  #  #  # ]:          0 :             filein >> nFileHistoricalFirst >> nFileHistoricalBest;
     976         [ #  # ]:          0 :             if (nFileHistoricalFirst > nFileHistoricalBest || nFileHistoricalBest > nFileBestSeenHeight) {
     977         [ #  # ]:          0 :                 throw std::runtime_error("Corrupt estimates file. Historical block range for estimates is invalid");
     978                 :            :             }
     979                 :          0 :             std::vector<double> fileBuckets;
     980   [ #  #  #  # ]:          0 :             filein >> Using<VectorFormatter<EncodedDoubleFormatter>>(fileBuckets);
     981                 :          0 :             size_t numBuckets = fileBuckets.size();
     982         [ #  # ]:          0 :             if (numBuckets <= 1 || numBuckets > 1000) {
     983         [ #  # ]:          0 :                 throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets");
     984                 :            :             }
     985                 :            : 
     986   [ #  #  #  # ]:          0 :             std::unique_ptr<TxConfirmStats> fileFeeStats(new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE));
     987   [ #  #  #  # ]:          0 :             std::unique_ptr<TxConfirmStats> fileShortStats(new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE));
     988   [ #  #  #  # ]:          0 :             std::unique_ptr<TxConfirmStats> fileLongStats(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE));
     989         [ #  # ]:          0 :             fileFeeStats->Read(filein, nVersionThatWrote, numBuckets);
     990         [ #  # ]:          0 :             fileShortStats->Read(filein, nVersionThatWrote, numBuckets);
     991         [ #  # ]:          0 :             fileLongStats->Read(filein, nVersionThatWrote, numBuckets);
     992                 :            : 
     993                 :            :             // Fee estimates file parsed correctly
     994                 :            :             // Copy buckets from file and refresh our bucketmap
     995         [ #  # ]:          0 :             buckets = fileBuckets;
     996                 :          0 :             bucketMap.clear();
     997         [ #  # ]:          0 :             for (unsigned int i = 0; i < buckets.size(); i++) {
     998         [ #  # ]:          0 :                 bucketMap[buckets[i]] = i;
     999                 :          0 :             }
    1000                 :            : 
    1001                 :            :             // Destroy old TxConfirmStats and point to new ones that already reference buckets and bucketMap
    1002                 :          0 :             feeStats = std::move(fileFeeStats);
    1003                 :          0 :             shortStats = std::move(fileShortStats);
    1004                 :          0 :             longStats = std::move(fileLongStats);
    1005                 :            : 
    1006                 :          0 :             nBestSeenHeight = nFileBestSeenHeight;
    1007                 :          0 :             historicalFirst = nFileHistoricalFirst;
    1008                 :          0 :             historicalBest = nFileHistoricalBest;
    1009                 :          0 :         }
    1010         [ #  # ]:          0 :     }
    1011                 :            :     catch (const std::exception& e) {
    1012   [ #  #  #  #  :          0 :         LogPrintf("CBlockPolicyEstimator::Read(): unable to read policy estimator data (non-fatal): %s\n",e.what());
                   #  # ]
    1013                 :          0 :         return false;
    1014         [ #  # ]:          0 :     }
    1015                 :          0 :     return true;
    1016                 :          0 : }
    1017                 :            : 
    1018                 :          0 : void CBlockPolicyEstimator::FlushUnconfirmed()
    1019                 :            : {
    1020                 :          0 :     const auto startclear{SteadyClock::now()};
    1021                 :          0 :     LOCK(m_cs_fee_estimator);
    1022                 :          0 :     size_t num_entries = mapMemPoolTxs.size();
    1023                 :            :     // Remove every entry in mapMemPoolTxs
    1024         [ #  # ]:          0 :     while (!mapMemPoolTxs.empty()) {
    1025                 :          0 :         auto mi = mapMemPoolTxs.begin();
    1026         [ #  # ]:          0 :         _removeTx(mi->first, false); // this calls erase() on mapMemPoolTxs
    1027                 :            :     }
    1028                 :          0 :     const auto endclear{SteadyClock::now()};
    1029   [ #  #  #  #  :          0 :     LogPrint(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %gs\n", num_entries, Ticks<SecondsDouble>(endclear - startclear));
          #  #  #  #  #  
             #  #  #  #  
                      # ]
    1030                 :          0 : }
    1031                 :            : 
    1032                 :          0 : std::chrono::hours CBlockPolicyEstimator::GetFeeEstimatorFileAge()
    1033                 :            : {
    1034                 :          0 :     auto file_time = std::filesystem::last_write_time(m_estimation_filepath);
    1035                 :          0 :     auto now = std::filesystem::file_time_type::clock::now();
    1036                 :          0 :     return std::chrono::duration_cast<std::chrono::hours>(now - file_time);
    1037                 :            : }
    1038                 :            : 
    1039                 :          0 : static std::set<double> MakeFeeSet(const CFeeRate& min_incremental_fee,
    1040                 :            :                                    double max_filter_fee_rate,
    1041                 :            :                                    double fee_filter_spacing)
    1042                 :            : {
    1043                 :          0 :     std::set<double> fee_set;
    1044                 :            : 
    1045         [ #  # ]:          0 :     const CAmount min_fee_limit{std::max(CAmount(1), min_incremental_fee.GetFeePerK() / 2)};
    1046         [ #  # ]:          0 :     fee_set.insert(0);
    1047         [ #  # ]:          0 :     for (double bucket_boundary = min_fee_limit;
    1048                 :          0 :          bucket_boundary <= max_filter_fee_rate;
    1049                 :          0 :          bucket_boundary *= fee_filter_spacing) {
    1050                 :            : 
    1051         [ #  # ]:          0 :         fee_set.insert(bucket_boundary);
    1052                 :          0 :     }
    1053                 :            : 
    1054                 :          0 :     return fee_set;
    1055         [ #  # ]:          0 : }
    1056                 :            : 
    1057                 :          0 : FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee)
    1058                 :          0 :     : m_fee_set{MakeFeeSet(minIncrementalFee, MAX_FILTER_FEERATE, FEE_FILTER_SPACING)}
    1059                 :            : {
    1060                 :          0 : }
    1061                 :            : 
    1062                 :          0 : CAmount FeeFilterRounder::round(CAmount currentMinFee)
    1063                 :            : {
    1064                 :          0 :     AssertLockNotHeld(m_insecure_rand_mutex);
    1065                 :          0 :     std::set<double>::iterator it = m_fee_set.lower_bound(currentMinFee);
    1066   [ #  #  #  # ]:          0 :     if (it == m_fee_set.end() ||
    1067         [ #  # ]:          0 :         (it != m_fee_set.begin() &&
    1068                 :          0 :          WITH_LOCK(m_insecure_rand_mutex, return insecure_rand.rand32()) % 3 != 0)) {
    1069                 :          0 :         --it;
    1070                 :          0 :     }
    1071                 :          0 :     return static_cast<CAmount>(*it);
    1072                 :            : }

Generated by: LCOV version 1.14