[WIP] [FN] Fee estimator

src/Stratis.Bitcoin.Features.ColdStaking.Tests/ColdStakingControllerTest.cs

@@ -138,8 +138,7 @@ private void CreateMempoolManager()
         {
             this.mempoolSettings = new MempoolSettings(this.nodeSettings);
             this.consensusSettings = new ConsensusSettings(this.nodeSettings);
-            this.txMemPool = new TxMempool(this.dateTimeProvider, new BlockPolicyEstimator(
-                new MempoolSettings(this.nodeSettings), this.loggerFactory, this.nodeSettings), this.loggerFactory, this.nodeSettings);
+            this.txMemPool = new TxMempool(this.dateTimeProvider, new BitcoinBlockPolicyEstimator(this.loggerFactory, this.nodeSettings), this.loggerFactory, this.nodeSettings);
             this.chainIndexer = new ChainIndexer(this.Network);
             this.nodeDeployments = new NodeDeployments(this.Network, this.chainIndexer);
 

src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs

@@ -181,7 +181,7 @@ public static async Task<List<Block>> MineBlocksAsync(TestChainContext testChain
         /// </summary>
         private static async Task<List<Block>> MineBlocksAsync(TestChainContext testChainContext, int count, Script receiver, bool mutateLastBlock)
         {
-            var blockPolicyEstimator = new BlockPolicyEstimator(new MempoolSettings(testChainContext.NodeSettings), testChainContext.LoggerFactory, testChainContext.NodeSettings);
+            var blockPolicyEstimator = new BitcoinBlockPolicyEstimator(testChainContext.LoggerFactory, testChainContext.NodeSettings);
             var mempool = new TxMempool(testChainContext.DateTimeProvider, blockPolicyEstimator, testChainContext.LoggerFactory, testChainContext.NodeSettings);
             var mempoolLock = new MempoolSchedulerLock();
 

src/Stratis.Bitcoin.Features.MemoryPool.Tests/FeeTests.cs

@@ -17,8 +17,7 @@ public void BlockPolicyEstimates()
         {
             var dateTimeSet = new DateTimeProviderSet();
             NodeSettings settings = NodeSettings.Default(KnownNetworks.TestNet);
-            var mpool = new TxMempool(DateTimeProvider.Default,
-                new BlockPolicyEstimator(new MempoolSettings(settings), settings.LoggerFactory, settings), settings.LoggerFactory, settings);
+            var mpool = new TxMempool(DateTimeProvider.Default, new BitcoinBlockPolicyEstimator(settings.LoggerFactory, settings), settings.LoggerFactory, settings);
             var entry = new TestMemPoolEntryHelper();
             var basefee = new Money(2000);
             var deltaFee = new Money(100);
@@ -51,8 +50,8 @@ public void BlockPolicyEstimates()
             int answerFound;
 
             // Loop through 200 blocks
-            // At a decay .998 and 4 fee transactions per block
-            // This makes the tx count about 1.33 per bucket, above the 1 threshold
+            // At a decay .9952 and 4 fee transactions per block
+            // This makes the tx count about 2.5 per bucket, well above the 0.1 threshold
             while (blocknum < 200)
             {
                 for (int j = 0; j < 10; j++)
@@ -82,21 +81,15 @@ public void BlockPolicyEstimates()
                 }
                 mpool.RemoveForBlock(block, ++blocknum);
                 block.Clear();
-                if (blocknum == 30)
+                // Check after just a few txs that combining buckets works as expected
+                if (blocknum == 3)
                 {
-                    // At this point we should need to combine 5 buckets to get enough data points
-                    // So estimateFee(1,2,3) should fail and estimateFee(4) should return somewhere around
-                    // 8*baserate.  estimateFee(4) %'s are 100,100,100,100,90 = average 98%
+                    // At this point we should need to combine 3 buckets to get enough data points
+                    // So estimateFee(1) should fail and estimateFee(2) should return somewhere around
+                    // 9*baserate.  estimateFee(2) %'s are 100,100,90 = average 97%
                     Assert.True(mpool.EstimateFee(1) == new FeeRate(0));
-                    Assert.True(mpool.EstimateFee(2) == new FeeRate(0));
-                    Assert.True(mpool.EstimateFee(3) == new FeeRate(0));
-                    Assert.True(mpool.EstimateFee(4).FeePerK < 8 * baseRate.FeePerK + deltaFee);
-                    Assert.True(mpool.EstimateFee(4).FeePerK > 8 * baseRate.FeePerK - deltaFee);
-
-                    Assert.True(mpool.EstimateSmartFee(1, out answerFound) == mpool.EstimateFee(4) && answerFound == 4);
-                    Assert.True(mpool.EstimateSmartFee(3, out answerFound) == mpool.EstimateFee(4) && answerFound == 4);
-                    Assert.True(mpool.EstimateSmartFee(4, out answerFound) == mpool.EstimateFee(4) && answerFound == 4);
-                    Assert.True(mpool.EstimateSmartFee(8, out answerFound) == mpool.EstimateFee(8) && answerFound == 8);
+                    Assert.True(mpool.EstimateFee(2).FeePerK < 9 * baseRate.FeePerK + deltaFee);
+                    Assert.True(mpool.EstimateFee(2).FeePerK > 9 * baseRate.FeePerK - deltaFee);
                 }
             }
 
@@ -115,15 +108,16 @@ public void BlockPolicyEstimates()
                     Assert.True(origFeeEst[i - 1] <= origFeeEst[i - 2]);
                 }
                 int mult = 11 - i;
-                if (i > 1)
+                if (i % 2 == 0) //At scale 2, test logic is only correct for even targets
                 {
                     Assert.True(origFeeEst[i - 1] < mult * baseRate.FeePerK + deltaFee);
                     Assert.True(origFeeEst[i - 1] > mult * baseRate.FeePerK - deltaFee);
                 }
-                else
-                {
-                    Assert.True(origFeeEst[i - 1] == new FeeRate(0).FeePerK);
-                }
+            }
+            // Fill out rest of the original estimates
+            for (int i = 10; i <= 48; i++)
+            {
+                origFeeEst.Add(mpool.EstimateFee(i).FeePerK);
             }
 
             // Mine 50 more blocks with no transactions happening, estimates shouldn't change
@@ -132,7 +126,7 @@ public void BlockPolicyEstimates()
                 mpool.RemoveForBlock(block, ++blocknum);
 
             Assert.True(mpool.EstimateFee(1) == new FeeRate(0));
-            for (int i = 2; i < 10; i++)
+            for (int i = 2; i < 9; i++)
             {
                 Assert.True(mpool.EstimateFee(i).FeePerK < origFeeEst[i - 1] + deltaFee);
                 Assert.True(mpool.EstimateFee(i).FeePerK > origFeeEst[i - 1] - deltaFee);
@@ -156,10 +150,9 @@ public void BlockPolicyEstimates()
                 mpool.RemoveForBlock(block, ++blocknum);
             }
 
-            for (int i = 1; i < 10; i++)
+            for (int i = 1; i < 9; i++)
             {
                 Assert.True(mpool.EstimateFee(i) == new FeeRate(0) || mpool.EstimateFee(i).FeePerK > origFeeEst[i - 1] - deltaFee);
-                Assert.True(mpool.EstimateSmartFee(i, out answerFound).FeePerK > origFeeEst[answerFound - 1] - deltaFee);
             }
 
             // Mine all those transactions
@@ -177,14 +170,15 @@ public void BlockPolicyEstimates()
             mpool.RemoveForBlock(block, 265);
             block.Clear();
             Assert.True(mpool.EstimateFee(1) == new FeeRate(0));
-            for (int i = 2; i < 10; i++)
+            for (int i = 2; i < 9; i++)
             {
-                Assert.True(mpool.EstimateFee(i).FeePerK > origFeeEst[i - 1] - deltaFee);
+                Assert.True(mpool.EstimateFee(i) == new FeeRate(0) || 
+                    mpool.EstimateFee(i).FeePerK > origFeeEst[i - 1] - deltaFee);
             }
 
-            // Mine 200 more blocks where everything is mined every block
+            // Mine 600 more blocks where everything is mined every block
             // Estimates should be below original estimates
-            while (blocknum < 465)
+            while (blocknum < 865)
             {
                 for (int j = 0; j < 10; j++)
                 { // For each fee multiple
@@ -203,23 +197,10 @@ public void BlockPolicyEstimates()
                 block.Clear();
             }
             Assert.True(mpool.EstimateFee(1) == new FeeRate(0));
-            for (int i = 2; i < 10; i++)
+            for (int i = 2; i < 9; i++)
             {
                 Assert.True(mpool.EstimateFee(i).FeePerK < origFeeEst[i - 1] - deltaFee);
-            }
-
-            // Test that if the mempool is limited, estimateSmartFee won't return a value below the mempool min fee
-            // and that estimateSmartPriority returns essentially an infinite value
-            mpool.AddUnchecked(txf.GetHash(), entry.Fee(feeV[5]).Time(dateTimeSet.GetTime()).Priority(0).Height(blocknum).FromTx(txf, mpool));
-            // evict that transaction which should set a mempool min fee of minRelayTxFee + feeV[5]
-            mpool.TrimToSize(1);
-            Assert.True(mpool.GetMinFee(1).FeePerK > feeV[5]);
-            for (int i = 1; i < 10; i++)
-            {
-                Assert.True(mpool.EstimateSmartFee(i, out answerFound).FeePerK >= mpool.EstimateFee(i).FeePerK);
-                Assert.True(mpool.EstimateSmartFee(i, out answerFound).FeePerK >= mpool.GetMinFee(1).FeePerK);
-                Assert.True(mpool.EstimateSmartPriority(i, out answerFound) == BlockPolicyEstimator.InfPriority);
-            }
+            }            
         }
 
         public class DateTimeProviderSet : DateTimeProvider

src/Stratis.Bitcoin.Features.MemoryPool.Tests/MemoryPoolTransactionTests.cs

@@ -20,6 +20,7 @@ public void MempoolRemoveTest()
 
             // Parent transaction with three children,
             // and three grand-children:
+            // TODO: Use network factory methods for all Transaction instantiation
             var txParent = new Transaction();
 
             txParent.AddInput(new TxIn());
@@ -47,7 +48,7 @@ public void MempoolRemoveTest()
             }
 
             NodeSettings settings = NodeSettings.Default(KnownNetworks.TestNet);
-            var testPool = new TxMempool(DateTimeProvider.Default, new BlockPolicyEstimator(new MempoolSettings(settings), settings.LoggerFactory, settings), settings.LoggerFactory, settings);
+            var testPool = new TxMempool(DateTimeProvider.Default, new BitcoinBlockPolicyEstimator(settings.LoggerFactory, settings), settings.LoggerFactory, settings);
 
             // Nothing in pool, remove should do nothing:
             long poolSize = testPool.Size;
@@ -115,7 +116,7 @@ private void CheckSort(TxMempool pool, List<TxMempoolEntry> sortedSource, List<s
         public void MempoolIndexingTest()
         {
             NodeSettings settings = NodeSettings.Default(KnownNetworks.TestNet);
-            var pool = new TxMempool(DateTimeProvider.Default, new BlockPolicyEstimator(new MempoolSettings(settings), settings.LoggerFactory, settings), settings.LoggerFactory, settings);
+            var pool = new TxMempool(DateTimeProvider.Default, new BitcoinBlockPolicyEstimator(settings.LoggerFactory, settings), settings.LoggerFactory, settings);
             var entry = new TestMemPoolEntryHelper();
 
             /* 3rd highest fee */
@@ -298,7 +299,7 @@ public void MempoolIndexingTest()
         public void MempoolAncestorIndexingTest()
         {
             NodeSettings settings = NodeSettings.Default(KnownNetworks.TestNet);
-            var pool = new TxMempool(DateTimeProvider.Default, new BlockPolicyEstimator(new MempoolSettings(settings), settings.LoggerFactory, settings), settings.LoggerFactory, settings);
+            var pool = new TxMempool(DateTimeProvider.Default, new BitcoinBlockPolicyEstimator(settings.LoggerFactory, settings), settings.LoggerFactory, settings);
             var entry = new TestMemPoolEntryHelper();
 
             /* 3rd highest fee */
@@ -393,7 +394,7 @@ public void MempoolSizeLimitTest()
         {
             NodeSettings settings = NodeSettings.Default(KnownNetworks.TestNet);
             var dateTimeSet = new DateTimeProviderSet();
-            var pool = new TxMempool(dateTimeSet, new BlockPolicyEstimator(new MempoolSettings(settings), settings.LoggerFactory, settings), settings.LoggerFactory, settings);
+            var pool = new TxMempool(dateTimeSet, new BitcoinBlockPolicyEstimator(settings.LoggerFactory, settings), settings.LoggerFactory, settings);
             var entry = new TestMemPoolEntryHelper();
             entry.Priority(10.0);
 
@@ -528,7 +529,7 @@ public override DateTime GetUtcNow()
         public void MempoolConcurrencyTest()
         {
             NodeSettings settings = NodeSettings.Default(KnownNetworks.TestNet);
-            var pool = new TxMempool(DateTimeProvider.Default, new BlockPolicyEstimator(new MempoolSettings(settings), settings.LoggerFactory, settings), settings.LoggerFactory, settings);
+            var pool = new TxMempool(DateTimeProvider.Default, new BitcoinBlockPolicyEstimator(settings.LoggerFactory, settings), settings.LoggerFactory, settings);
             var scheduler = new SchedulerLock();
             var rand = new Random();
 

src/Stratis.Bitcoin.Features.MemoryPool.Tests/MempoolManagerTest.cs

@@ -42,7 +42,7 @@ private MempoolManager TxExpiryManager
                 };
                 NodeSettings nodeSettings = NodeSettings.Default(KnownNetworks.TestNet);
 
-                var blockPolicyEstimator = new BlockPolicyEstimator(settings, loggerFactory, nodeSettings);
+                var blockPolicyEstimator = new BitcoinBlockPolicyEstimator(loggerFactory, nodeSettings);
                 var mempool = new TxMempool(dateTime, blockPolicyEstimator, loggerFactory, nodeSettings);
 
                 var mockTxMempool = new Mock<TxMempool>();

src/Stratis.Bitcoin.Features.MemoryPool.Tests/MempoolPersistenceTest.cs

@@ -282,7 +282,7 @@ private MempoolManager CreateTestMempool(NodeSettings settings, out TxMempool tx
             NodeSettings nodeSettings = NodeSettings.Default(settings.Network);
             ILoggerFactory loggerFactory = nodeSettings.LoggerFactory;
             var consensusSettings = new ConsensusSettings(nodeSettings);
-            txMemPool = new TxMempool(dateTimeProvider, new BlockPolicyEstimator(new MempoolSettings(nodeSettings), loggerFactory, nodeSettings), loggerFactory, nodeSettings);
+            txMemPool = new TxMempool(dateTimeProvider, new BitcoinBlockPolicyEstimator(loggerFactory, nodeSettings), loggerFactory, nodeSettings);
             var mempoolLock = new MempoolSchedulerLock();
             var coins = new InMemoryCoinView(settings.Network.GenesisHash);
             var chain = new ChainIndexer(settings.Network);

src/Stratis.Bitcoin.Features.MemoryPool.Tests/TestChainFactory.cs

@@ -108,7 +108,7 @@ public static async Task<ITestChainContext> CreatePosAsync(Network network, Scri
             await consensus.InitializeAsync(genesis).ConfigureAwait(false);
 
             var mempoolSettings = new MempoolSettings(nodeSettings);
-            var blockPolicyEstimator = new BlockPolicyEstimator(mempoolSettings, loggerFactory, nodeSettings);
+            var blockPolicyEstimator = new BitcoinBlockPolicyEstimator(loggerFactory, nodeSettings);
             var mempool = new TxMempool(dateTimeProvider, blockPolicyEstimator, loggerFactory, nodeSettings);
             var mempoolLock = new MempoolSchedulerLock();
 
@@ -188,7 +188,7 @@ public static async Task<ITestChainContext> CreateAsync(Network network, Script
             chainState.BlockStoreTip = genesis;
             await consensus.InitializeAsync(genesis).ConfigureAwait(false);
 
-            var blockPolicyEstimator = new BlockPolicyEstimator(new MempoolSettings(nodeSettings), loggerFactory, nodeSettings);
+            var blockPolicyEstimator = new BitcoinBlockPolicyEstimator(loggerFactory, nodeSettings);
             var mempool = new TxMempool(dateTimeProvider, blockPolicyEstimator, loggerFactory, nodeSettings);
             var mempoolLock = new MempoolSchedulerLock();