diff --git a/electrum-proxy.sln b/electrum-proxy.sln new file mode 100644 index 000000000..b3f6cb1ea --- /dev/null +++ b/electrum-proxy.sln @@ -0,0 +1,110 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.10.35201.131 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "GWallet.Backend", "src\GWallet.Backend\GWallet.Backend.fsproj", "{96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "build", "build", "{9DFD61F8-2CED-47F1-BB3A-48A383D4751D}" + ProjectSection(SolutionItems) = preProject + scripts\bump.fsx = scripts\bump.fsx + scripts\configure.fsx = scripts\configure.fsx + configure.sh = configure.sh + scripts\find.fsx = scripts\find.fsx + scripts\fsxHelper.fs = scripts\fsxHelper.fs + scripts\githubActions.fs = scripts\githubActions.fs + scripts\make.fsx = scripts\make.fsx + scripts\make.sh = scripts\make.sh + Makefile = Makefile + scripts\sanitycheck.fsx = scripts\sanitycheck.fsx + scripts\snap_release.fsx = scripts\snap_release.fsx + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docs", "docs", "{C90A30F5-1423-44B2-A8D4-ED5FEDD4E36F}" + ProjectSection(SolutionItems) = preProject + CONTRIBUTING.md = CONTRIBUTING.md + ReadMe.md = ReadMe.md + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Fsdk", "Fsdk", "{6EE07541-91A1-42C2-A21F-2809BBDC2F50}" + ProjectSection(SolutionItems) = preProject + scripts\fsx\Fsdk\Git.fs = scripts\fsx\Fsdk\Git.fs + scripts\fsx\Fsdk\Misc.fs = scripts\fsx\Fsdk\Misc.fs + scripts\fsx\Fsdk\Network.fs = scripts\fsx\Fsdk\Network.fs + scripts\fsx\Fsdk\Process.fs = scripts\fsx\Fsdk\Process.fs + scripts\fsx\Fsdk\Unix.fs = scripts\fsx\Fsdk\Unix.fs + EndProjectSection +EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "ElectrumProxy", "src\ElectrumProxy\ElectrumProxy.fsproj", "{9F313452-F0F3-4A6A-8391-CF9239C5242D}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|ARM = Debug|ARM + Debug|iPhone = Debug|iPhone + Debug|iPhoneSimulator = Debug|iPhoneSimulator + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|ARM = Release|ARM + Release|iPhone = Release|iPhone + Release|iPhoneSimulator = Release|iPhoneSimulator + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Debug|ARM.ActiveCfg = Debug|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Debug|ARM.Build.0 = Debug|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Debug|iPhone.ActiveCfg = Debug|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Debug|iPhone.Build.0 = Debug|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Debug|iPhoneSimulator.ActiveCfg = Debug|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Debug|iPhoneSimulator.Build.0 = Debug|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Debug|x64.ActiveCfg = Debug|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Debug|x64.Build.0 = Debug|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Debug|x86.ActiveCfg = Debug|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Debug|x86.Build.0 = Debug|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Release|Any CPU.Build.0 = Release|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Release|ARM.ActiveCfg = Release|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Release|ARM.Build.0 = Release|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Release|iPhone.ActiveCfg = Release|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Release|iPhone.Build.0 = Release|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Release|iPhoneSimulator.ActiveCfg = Release|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Release|iPhoneSimulator.Build.0 = Release|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Release|x64.ActiveCfg = Release|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Release|x64.Build.0 = Release|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Release|x86.ActiveCfg = Release|Any CPU + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}.Release|x86.Build.0 = Release|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Debug|ARM.ActiveCfg = Debug|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Debug|ARM.Build.0 = Debug|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Debug|iPhone.ActiveCfg = Debug|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Debug|iPhone.Build.0 = Debug|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Debug|iPhoneSimulator.ActiveCfg = Debug|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Debug|iPhoneSimulator.Build.0 = Debug|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Debug|x64.ActiveCfg = Debug|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Debug|x64.Build.0 = Debug|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Debug|x86.ActiveCfg = Debug|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Debug|x86.Build.0 = Debug|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Release|Any CPU.Build.0 = Release|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Release|ARM.ActiveCfg = Release|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Release|ARM.Build.0 = Release|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Release|iPhone.ActiveCfg = Release|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Release|iPhone.Build.0 = Release|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Release|iPhoneSimulator.ActiveCfg = Release|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Release|iPhoneSimulator.Build.0 = Release|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Release|x64.ActiveCfg = Release|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Release|x64.Build.0 = Release|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Release|x86.ActiveCfg = Release|Any CPU + {9F313452-F0F3-4A6A-8391-CF9239C5242D}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {9B7D9375-3711-4242-B4B1-3F7CD6241287} + EndGlobalSection +EndGlobal diff --git a/geewallet.sln b/geewallet.sln index 5b13e1b8b..d3aa9921c 100644 --- a/geewallet.sln +++ b/geewallet.sln @@ -1,27 +1,26 @@ Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.27130.0 +# Visual Studio Version 17 +VisualStudioVersion = 17.10.35201.131 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "GWallet.Backend", "src\GWallet.Backend\GWallet.Backend.fsproj", "{96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}" +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "GWallet.Backend", "src\GWallet.Backend\GWallet.Backend.fsproj", "{96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2}" EndProject -Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "GWallet.Backend.Tests", "src\GWallet.Backend.Tests\GWallet.Backend.Tests.fsproj", "{F9448076-88BE-4045-8704-A652D133E036}" +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "GWallet.Backend.Tests", "src\GWallet.Backend.Tests\GWallet.Backend.Tests.fsproj", "{F9448076-88BE-4045-8704-A652D133E036}" EndProject -Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "GWallet.Frontend.Console", "src\GWallet.Frontend.Console\GWallet.Frontend.Console.fsproj", "{8413EEF5-69F5-499F-AE01-754E9541EF90}" +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "GWallet.Frontend.Console", "src\GWallet.Frontend.Console\GWallet.Frontend.Console.fsproj", "{8413EEF5-69F5-499F-AE01-754E9541EF90}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "build", "build", "{9DFD61F8-2CED-47F1-BB3A-48A383D4751D}" ProjectSection(SolutionItems) = preProject - configure.sh = configure.sh - Makefile = Makefile + scripts\bump.fsx = scripts\bump.fsx scripts\configure.fsx = scripts\configure.fsx + configure.sh = configure.sh + scripts\find.fsx = scripts\find.fsx + scripts\fsxHelper.fs = scripts\fsxHelper.fs + scripts\githubActions.fs = scripts\githubActions.fs scripts\make.fsx = scripts\make.fsx scripts\make.sh = scripts\make.sh - scripts\bump.fsx = scripts\bump.fsx + Makefile = Makefile scripts\sanitycheck.fsx = scripts\sanitycheck.fsx - scripts\fsxHelper.fs = scripts\fsxHelper.fs scripts\snap_release.fsx = scripts\snap_release.fsx - scripts\githubActions.fs = scripts\githubActions.fs - scripts\find.fsx = scripts\find.fsx - scripts\bump.fsx = scripts\bump.fsx EndProjectSection EndProject Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "GWallet.Frontend.XF.Mac", "src\GWallet.Frontend.XF.Mac\GWallet.Frontend.XF.Mac.fsproj", "{9E020D62-9160-49AC-A9CD-476CADAE0B87}" @@ -44,14 +43,16 @@ Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "GWallet.Frontend.XF.iOS", " EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Fsdk", "Fsdk", "{6EE07541-91A1-42C2-A21F-2809BBDC2F50}" ProjectSection(SolutionItems) = preProject + scripts\fsx\Fsdk\Git.fs = scripts\fsx\Fsdk\Git.fs scripts\fsx\Fsdk\Misc.fs = scripts\fsx\Fsdk\Misc.fs - scripts\fsx\Fsdk\Unix.fs = scripts\fsx\Fsdk\Unix.fs - scripts\fsx\Fsdk\Process.fs = scripts\fsx\Fsdk\Process.fs scripts\fsx\Fsdk\Network.fs = scripts\fsx\Fsdk\Network.fs - scripts\fsx\Fsdk\Git.fs = scripts\fsx\Fsdk\Git.fs + scripts\fsx\Fsdk\Process.fs = scripts\fsx\Fsdk\Process.fs + scripts\fsx\Fsdk\Unix.fs = scripts\fsx\Fsdk\Unix.fs EndProjectSection EndProject -Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "GWallet.Frontend.ConsoleApp", "src\GWallet.Frontend.ConsoleApp\GWallet.Frontend.ConsoleApp.fsproj", "{EFACE810-A402-4673-B8B5-4517E698EACE}" +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "GWallet.Frontend.ConsoleApp", "src\GWallet.Frontend.ConsoleApp\GWallet.Frontend.ConsoleApp.fsproj", "{EFACE810-A402-4673-B8B5-4517E698EACE}" +EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "ElectrumProxy", "ElectrumProxy\ElectrumProxy.fsproj", "{2CE9C122-CB05-4143-9070-78968074E6CC}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -306,6 +307,30 @@ Global {EFACE810-A402-4673-B8B5-4517E698EACE}.Release|x64.Build.0 = Release|Any CPU {EFACE810-A402-4673-B8B5-4517E698EACE}.Release|x86.ActiveCfg = Release|Any CPU {EFACE810-A402-4673-B8B5-4517E698EACE}.Release|x86.Build.0 = Release|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Debug|ARM.ActiveCfg = Debug|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Debug|ARM.Build.0 = Debug|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Debug|iPhone.ActiveCfg = Debug|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Debug|iPhone.Build.0 = Debug|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Debug|iPhoneSimulator.ActiveCfg = Debug|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Debug|iPhoneSimulator.Build.0 = Debug|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Debug|x64.ActiveCfg = Debug|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Debug|x64.Build.0 = Debug|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Debug|x86.ActiveCfg = Debug|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Debug|x86.Build.0 = Debug|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Release|Any CPU.Build.0 = Release|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Release|ARM.ActiveCfg = Release|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Release|ARM.Build.0 = Release|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Release|iPhone.ActiveCfg = Release|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Release|iPhone.Build.0 = Release|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Release|iPhoneSimulator.ActiveCfg = Release|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Release|iPhoneSimulator.Build.0 = Release|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Release|x64.ActiveCfg = Release|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Release|x64.Build.0 = Release|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Release|x86.ActiveCfg = Release|Any CPU + {2CE9C122-CB05-4143-9070-78968074E6CC}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/scripts/sanitycheck.fsx b/scripts/sanitycheck.fsx index 3506ba182..9c016a4b7 100755 --- a/scripts/sanitycheck.fsx +++ b/scripts/sanitycheck.fsx @@ -93,7 +93,8 @@ let FindOffendingPrintfUsage () = "scripts{0}" + "src{1}GWallet.Frontend.Console{0}" + "src{1}GWallet.Backend.Tests{0}" + - "src{1}GWallet.Backend{1}FSharpUtil.fs", + "src{1}GWallet.Backend{1}FSharpUtil.fs{0}" + + "src{1}ElectrumProxy", Path.PathSeparator, Path.DirectorySeparatorChar ) diff --git a/src/ElectrumProxy/ElectrumProxy.fsproj b/src/ElectrumProxy/ElectrumProxy.fsproj new file mode 100644 index 000000000..fa43de7da --- /dev/null +++ b/src/ElectrumProxy/ElectrumProxy.fsproj @@ -0,0 +1,22 @@ + + + + Exe + net6.0 + + + + + + + + + + + + + + + + + diff --git a/src/ElectrumProxy/Program.fs b/src/ElectrumProxy/Program.fs new file mode 100644 index 000000000..3154feeda --- /dev/null +++ b/src/ElectrumProxy/Program.fs @@ -0,0 +1,47 @@ +module Program + +open System.Net.Sockets + +open StreamJsonRpc + + +[] +let main (args: string[]) = + let port = int args.[0] + + let listener = new TcpListener(System.Net.IPAddress.Any, port) + listener.Start(); + + GWallet.Backend.Caching.Instance.SaveServerRankingsToDiskOnEachUpdate <- false + + async { + while true do + use! tcpClient = listener.AcceptTcpClientAsync() |> Async.AwaitTask + use networkStream = tcpClient.GetStream() + + use formatter = new SystemTextJsonFormatter() + use handler = new NewLineDelimitedMessageHandler(networkStream, networkStream, formatter) + formatter.JsonSerializerOptions.PropertyNamingPolicy <- Server.PascalCaseToSnakeCaseNamingPolicy() + + use jsonRpc = new JsonRpc(handler) + use server = new Server.ElectrumProxyServer() + let serverOptions = JsonRpcTargetOptions(EventNameTransform=System.Func<_, _>(server.EventNameTransform)) + jsonRpc.AddLocalRpcTarget(server, serverOptions) + +#if DEBUG + jsonRpc.TraceSource.Listeners.Add(new System.Diagnostics.TextWriterTraceListener(System.Console.OpenStandardError())) + |> ignore + jsonRpc.TraceSource.Switch.Level <- System.Diagnostics.SourceLevels.All +#endif + + jsonRpc.Disconnected.Add(fun args -> + eprintfn "Disconnected. Reason=%A; Description=%A; Exception=%A" args.Reason args.Description args.Exception) + + jsonRpc.StartListening() + do! jsonRpc.Completion |> Async.AwaitTask + } + |> Async.RunSynchronously + + GWallet.Backend.Caching.Instance.SaveServerStatsToDisk() + + 0 diff --git a/src/ElectrumProxy/Server.fs b/src/ElectrumProxy/Server.fs new file mode 100644 index 000000000..71085b7e5 --- /dev/null +++ b/src/ElectrumProxy/Server.fs @@ -0,0 +1,213 @@ +module Server + +open System +open System.Text +open System.Threading.Tasks + +open StreamJsonRpc + +open GWallet.Backend + +type PascalCaseToSnakeCaseNamingPolicy() = + inherit Json.JsonNamingPolicy() + + static let capitalizedWordRegex = RegularExpressions.Regex "[A-Z][a-z0-9]*" + + override self.ConvertName name = + let evaluator (regexMatch: RegularExpressions.Match) = + let lowercase = regexMatch.Value.ToLower() + if regexMatch.Index = 0 then lowercase else "_" + lowercase + capitalizedWordRegex.Replace(name, Text.RegularExpressions.MatchEvaluator evaluator) + +let supportedProtocolVersion = "1.3" + +let ScriptHashToAddress (scriptHash: string) = + let scriptId = NBitcoin.WitScriptId scriptHash + scriptId.GetAddress NBitcoin.Network.Main + +let private QueryElectrum<'R when 'R: equality> (job: Async->Async<'R>) : Async<'R> = + UtxoCoin.Server.Query Currency.BTC (UtxoCoin.QuerySettings.Default ServerSelectionMode.Fast) job None + +let private QueryMultiple<'R when 'R: equality> + (electrumJob: Async->Async<'R>) + (additionalServers: List>) : Async<'R> = + let updateServer serverMatchFunc stat = + if additionalServers |> List.exists (fun each -> serverMatchFunc each.Details) |> not then + Caching.Instance.SaveServerLastStat serverMatchFunc stat + + let faultTolerantClient = + FaultTolerantParallelClient updateServer + let query = faultTolerantClient.Query + let querySettings = UtxoCoin.Server.FaultTolerantParallelClientDefaultSettings ServerSelectionMode.Fast None + query + querySettings + (List.append + (UtxoCoin.Server.GetRandomizedFuncs Currency.BTC electrumJob) + additionalServers) + +type ElectrumProxyServer() as self = + static let blockchainHeadersSubscriptionInterval = TimeSpan.FromMinutes 1.0 + + let blockchainHeadersSubscriptionEvent = new Event() + + let cts = new Threading.CancellationTokenSource(-1) + let blockchainHeadersSubscription = lazy( + Async.Start( + async { + while true do + do! Async.Sleep blockchainHeadersSubscriptionInterval + let! blockchinTip = self.GetBlockchainTip() + blockchainHeadersSubscriptionEvent.Trigger blockchinTip + }, cts.Token)) + + let bitcoreNodeAddress = "https://api.bitcore.io" + let bitcoreNodeClient = new BitcoreNodeClient(bitcoreNodeAddress) + let blockbokClients = + [ + for i=1 to 5 do + let address = sprintf "https://btc%d.trezor.io" i + yield address, lazy(new BlockbookClient(address)) + ] + + // Cache results of "blockchain.scripthash.get_history" requests. Invalidate cache only when + // new block(s) are added to the blockchain. + let mutable blockchainHeight = 0UL + let mutable scripthashHistoryCache = Map.empty> + + interface IDisposable with + override self.Dispose() = + (bitcoreNodeClient :> IDisposable).Dispose() + for _, lazyClient in blockbokClients do + if lazyClient.IsValueCreated then (lazyClient.Value :> IDisposable).Dispose() + cts.Cancel() + + member self.EventNameTransform (name: string): string = + match name with + | "BlockchainHeadersSubscription" -> "blockchain.headers.subscribe" + | _ -> name + + [] + member self.ServerVersion (_clientVersion: string) (_protocolVersion: string) = + supportedProtocolVersion + + [] + member self.ServerPing () = () + + [] + member self.BlockchainBlockHeader (height: uint64) : Task = + QueryElectrum + (fun asyncClient -> async { + let! client = asyncClient + let! result = client.BlockchainBlockHeader height + return result.Result + } ) + |> Async.StartAsTask + + [] + member self.BlockchainBlockHeaders (start_height: uint64) (count: uint64) : Task = + QueryElectrum + (fun asyncClient -> async { + let! client = asyncClient + let! result = client.BlockchainBlockHeaders start_height count + return result.Result + } ) + |> Async.StartAsTask + + [] + member self.BlockchainScripthashGetHistory (scripthash: string) : Task> = + let electrumJob = + (fun (asyncClient: Async) -> async { + let! client = asyncClient + let! result = client.BlockchainScriptHashGetHistory scripthash + return result.Result + } ) + let bitcoreNodeServer: Server> = + { + Details = { + ServerInfo = { + NetworkPath = bitcoreNodeAddress + ConnectionType = { ConnectionType.Encrypted = true; Protocol = Protocol.Http } + } + CommunicationHistory = None + } + Retrieval = fun _timeouts -> async { + let address = ScriptHashToAddress scripthash + return! bitcoreNodeClient.GetAddressTransactions (address.ToString()) + } + } + + let blockbookServers = + [ + for serverAddress, lazyClient in blockbokClients do + yield { + Details = { + ServerInfo = { + NetworkPath = serverAddress + ConnectionType = { ConnectionType.Encrypted = true; Protocol = Protocol.Http } + } + CommunicationHistory = None + } + Retrieval = fun _timeouts -> async { + let address = ScriptHashToAddress scripthash + return! lazyClient.Value.GetAddressTransactions (address.ToString()) + } + } + ] + + async { + match scripthashHistoryCache |> Map.tryFind scripthash with + | Some value -> return value + | None -> + let! result = + QueryMultiple + electrumJob + (bitcoreNodeServer :: blockbookServers) + lock + scripthashHistoryCache + (fun () -> scripthashHistoryCache <- scripthashHistoryCache |> Map.add scripthash result) + return result + } + |> Async.StartAsTask + + member private self.GetBlockchainTip() : Async = + QueryElectrum + (fun asyncClient -> async { + let! client = asyncClient + let! result = client.BlockchainHeadersSubscribe() + let height = result.Result.Height + if height > blockchainHeight then + blockchainHeight <- height + lock + scripthashHistoryCache + (fun () -> scripthashHistoryCache <- Map.empty) + return result.Result + } ) + + [] + member this.BlockchainHeadersSubscription = blockchainHeadersSubscriptionEvent.Publish + + [] + member self.BlockchainHeadersSubscribe () : Task = + let task = self.GetBlockchainTip() |> Async.StartAsTask + blockchainHeadersSubscription.Value + task + + [] + member self.BlockchainTransactionGet (txHash: string) : Task = + QueryElectrum + (fun asyncClient -> async { + let! client = asyncClient + let! result = client.BlockchainTransactionGet txHash + return result.Result + } ) + |> Async.StartAsTask + + [] + member self.BlockchainTransactionBroadcast (rawTx: string) : Task = + QueryElectrum + (fun asyncClient -> async { + let! client = asyncClient + let! result = client.BlockchainTransactionBroadcast rawTx + return result.Result + } ) + |> Async.StartAsTask diff --git a/src/GWallet.Backend.Tests/AsyncCancellation.fs b/src/GWallet.Backend.Tests/AsyncCancellation.fs index 60d46efd8..7d4f99fcb 100644 --- a/src/GWallet.Backend.Tests/AsyncCancellation.fs +++ b/src/GWallet.Backend.Tests/AsyncCancellation.fs @@ -27,7 +27,7 @@ type FaultTolerantParallelClientAsyncCancellation() = } CommunicationHistory = None } - Retrieval = job + Retrieval = fun _timeout -> job } let dummy_func_to_not_save_server_because_it_is_irrelevant_for_this_test = (fun _ _ -> ()) diff --git a/src/GWallet.Backend.Tests/ElectrumIntegrationTests.fs b/src/GWallet.Backend.Tests/ElectrumIntegrationTests.fs index a8f63aa65..252e1b248 100644 --- a/src/GWallet.Backend.Tests/ElectrumIntegrationTests.fs +++ b/src/GWallet.Backend.Tests/ElectrumIntegrationTests.fs @@ -61,7 +61,8 @@ type ElectrumIntegrationTests() = // because we want the server incompatibilities to show up here (even if GWallet clients bypass // them in order not to crash) try - let stratumClient = ElectrumClient.StratumServer server + let timeout = { Timeout = TimeSpan.FromSeconds 5.0; ConnectTimeout = TimeSpan.FromSeconds 10.0 } + let stratumClient = ElectrumClient.StratumServer server timeout let result = query stratumClient |> Async.RunSynchronously diff --git a/src/GWallet.Backend.Tests/FaultTolerance.fs b/src/GWallet.Backend.Tests/FaultTolerance.fs index 7e73f500e..19ba36d14 100644 --- a/src/GWallet.Backend.Tests/FaultTolerance.fs +++ b/src/GWallet.Backend.Tests/FaultTolerance.fs @@ -69,7 +69,7 @@ type FaultTolerance() = } CommunicationHistory = None } - Retrieval = job + Retrieval = fun _timeout -> job } [] @@ -629,7 +629,7 @@ type FaultTolerance() = Some ({ Status = fault; TimeSpan = TimeSpan.FromSeconds 1.0 }, dummy_date_for_cache) } - Retrieval = async { return someResult1 } + Retrieval = fun _ -> async { return someResult1 } } let server2 = { Details = @@ -643,7 +643,7 @@ type FaultTolerance() = TimeSpan = TimeSpan.FromSeconds 2.0 }, dummy_date_for_cache) } - Retrieval = async { return someResult2 } + Retrieval = fun _ -> async { return someResult2 } } let retrievedData = (FaultTolerantParallelClient dummy_func_to_not_save_server_because_it_is_irrelevant_for_this_test).Query @@ -678,7 +678,7 @@ type FaultTolerance() = CommunicationHistory = Some ({ Status = fault; TimeSpan = TimeSpan.FromSeconds 2.0 }, dummy_date_for_cache) } - Retrieval = async { return someResult1 } + Retrieval = fun _ -> async { return someResult1 } } let server2 = { Details = @@ -691,7 +691,7 @@ type FaultTolerance() = CommunicationHistory = Some ({ Status = fault; TimeSpan = TimeSpan.FromSeconds 1.0 }, dummy_date_for_cache) } - Retrieval = async { return someResult2 } + Retrieval = fun _ -> async { return someResult2 } } let retrievedData = (FaultTolerantParallelClient dummy_func_to_not_save_server_because_it_is_irrelevant_for_this_test).Query @@ -726,7 +726,7 @@ type FaultTolerance() = TimeSpan = TimeSpan.FromSeconds 2.0 }, dummy_date_for_cache) } - Retrieval = async { return someResult1 } + Retrieval = fun _ -> async { return someResult1 } } let server2 = { Details = @@ -738,7 +738,7 @@ type FaultTolerance() = } CommunicationHistory = None } - Retrieval = async { return someResult2 } + Retrieval = fun _ -> async { return someResult2 } } let retrievedData = (FaultTolerantParallelClient dummy_func_to_not_save_server_because_it_is_irrelevant_for_this_test).Query @@ -773,7 +773,7 @@ type FaultTolerance() = CommunicationHistory = Some ({ Status = fault; TimeSpan = TimeSpan.FromSeconds 1.0 }, dummy_date_for_cache) } - Retrieval = async { return someResult1 } + Retrieval = fun _ -> async { return someResult1 } } let server2 = { Details = @@ -785,7 +785,7 @@ type FaultTolerance() = } CommunicationHistory = None } - Retrieval = async { return someResult2 } + Retrieval = fun _ -> async { return someResult2 } } let retrievedData = (FaultTolerantParallelClient dummy_func_to_not_save_server_because_it_is_irrelevant_for_this_test).Query @@ -821,7 +821,7 @@ type FaultTolerance() = CommunicationHistory = Some ({ Status = fault; TimeSpan = TimeSpan.FromSeconds 1.0 }, dummy_date_for_cache) } - Retrieval = async { return someResult1 } + Retrieval = fun _ -> async { return someResult1 } } let server2 = { Details = @@ -833,7 +833,7 @@ type FaultTolerance() = } CommunicationHistory = None } - Retrieval = async { return someResult2 } + Retrieval = fun _ -> async { return someResult2 } } let server3 = { Details = @@ -847,7 +847,7 @@ type FaultTolerance() = TimeSpan = TimeSpan.FromSeconds 1.0 }, dummy_date_for_cache) } - Retrieval = async { return someResult3 } + Retrieval = fun _ -> async { return someResult3 } } let defaultSettings = FaultTolerance.DefaultSettingsForNoConsistencyNoParallelismAndNoRetries None @@ -897,7 +897,7 @@ type FaultTolerance() = TimeSpan = TimeSpan.FromSeconds 1.0 }, dummy_date_for_cache) } - Retrieval = async { return raise SomeSpecificException } + Retrieval = fun _ -> async { return raise SomeSpecificException } } let server2 = { Details = @@ -911,7 +911,7 @@ type FaultTolerance() = TimeSpan = TimeSpan.FromSeconds 2.0 }, dummy_date_for_cache) } - Retrieval = async { return raise SomeSpecificException } + Retrieval = fun _ -> async { return raise SomeSpecificException } } let server3 = { Details = @@ -925,7 +925,7 @@ type FaultTolerance() = TimeSpan = TimeSpan.FromSeconds 3.0 }, dummy_date_for_cache) } - Retrieval = async { return someResult3 } + Retrieval = fun _ -> async { return someResult3 } } let fault = some_fault_with_no_last_successful_comm_because_irrelevant_for_this_test let server4 = { @@ -940,7 +940,7 @@ type FaultTolerance() = TimeSpan = TimeSpan.FromSeconds 1.0 }, dummy_date_for_cache) } - Retrieval = async { return someResult4 } + Retrieval = fun _ -> async { return someResult4 } } @@ -991,7 +991,7 @@ type FaultTolerance() = TimeSpan = TimeSpan.FromSeconds 1.0 }, dummy_date_for_cache) } - Retrieval = async { return raise SomeSpecificException } + Retrieval = fun _ -> async { return raise SomeSpecificException } } let server2 = { Details = @@ -1005,7 +1005,7 @@ type FaultTolerance() = TimeSpan = TimeSpan.FromSeconds 2.0 }, dummy_date_for_cache) } - Retrieval = async { return raise SomeSpecificException } + Retrieval = fun _ -> async { return raise SomeSpecificException } } let server3 = { Details = @@ -1019,7 +1019,7 @@ type FaultTolerance() = TimeSpan = TimeSpan.FromSeconds 3.0 }, dummy_date_for_cache) } - Retrieval = async { return raise SomeSpecificException } + Retrieval = fun _ -> async { return raise SomeSpecificException } } let server4 = { @@ -1034,7 +1034,7 @@ type FaultTolerance() = TimeSpan = TimeSpan.FromSeconds 4.0 }, dummy_date_for_cache) } - Retrieval = async { return someResult4 } + Retrieval = fun _ -> async { return someResult4 } } let server5 = { Details = @@ -1048,7 +1048,7 @@ type FaultTolerance() = TimeSpan = TimeSpan.FromSeconds 5.0 }, dummy_date_for_cache) } - Retrieval = async { return someResult5 } + Retrieval = fun _ -> async { return someResult5 } } let defaultSettings = FaultTolerance.DefaultSettingsForNoConsistencyNoParallelismAndNoRetries None diff --git a/src/GWallet.Backend.Tests/ParallelizationAndOptimization.fs b/src/GWallet.Backend.Tests/ParallelizationAndOptimization.fs index afa5565ef..ca706418b 100644 --- a/src/GWallet.Backend.Tests/ParallelizationAndOptimization.fs +++ b/src/GWallet.Backend.Tests/ParallelizationAndOptimization.fs @@ -28,7 +28,7 @@ type ParallelizationAndOptimization() = } CommunicationHistory = None } - Retrieval = job + Retrieval = fun _timeout -> job } let dummy_func_to_not_save_server_because_it_is_irrelevant_for_this_test = (fun _ _ -> ()) @@ -232,7 +232,7 @@ type ParallelizationAndOptimization() = TimeSpan = TimeSpan.FromSeconds 2.0 }, dummy_date_for_cache) } - Retrieval = async { return someResult1 } + Retrieval = fun _ -> async { return someResult1 } } let server2 = { Details = @@ -246,7 +246,7 @@ type ParallelizationAndOptimization() = TimeSpan = TimeSpan.FromSeconds 1.0 }, dummy_date_for_cache) } - Retrieval = async { return someResult2 } + Retrieval = fun _ -> async { return someResult2 } } let retrievedData = (FaultTolerantParallelClient dummy_func_to_not_save_server_because_it_is_irrelevant_for_this_test).Query @@ -301,7 +301,7 @@ type ParallelizationAndOptimization() = TimeSpan = TimeSpan.FromSeconds 1.0 }, dummy_date_for_cache) } - Retrieval = async { return raise SomeExceptionDuringParallelWork } + Retrieval = fun _ -> async { return raise SomeExceptionDuringParallelWork } } let server2 = { Details = @@ -315,7 +315,7 @@ type ParallelizationAndOptimization() = TimeSpan = TimeSpan.FromSeconds 2.0 }, dummy_date_for_cache) } - Retrieval = async { return someResult2 } + Retrieval = fun _ -> async { return someResult2 } } let server3 = { Details = @@ -327,7 +327,7 @@ type ParallelizationAndOptimization() = } CommunicationHistory = None } - Retrieval = async { return someResult3 } + Retrieval = fun _ -> async { return someResult3 } } let defaultSettings = FaultTolerance.DefaultSettingsForNoConsistencyNoParallelismAndNoRetries None diff --git a/src/GWallet.Backend.Tests/ServerReference.fs b/src/GWallet.Backend.Tests/ServerReference.fs index a87110563..e4ff46d89 100644 --- a/src/GWallet.Backend.Tests/ServerReference.fs +++ b/src/GWallet.Backend.Tests/ServerReference.fs @@ -510,7 +510,7 @@ type ServerReference() = Assert.Fail "https server should be fault, not successful" [] - member __.``duplicate servers are removed``() = + member __.``no duplicate servers are in the collection``() = let sameRandomHostname = "xfoihror3uo3wmio" let serverA = { @@ -532,16 +532,14 @@ type ServerReference() = } let servers = Map.empty.Add (dummy_currency_because_irrelevant_for_this_test, - seq { yield serverA; yield serverB }) - let serverDetails = ServerRegistry.Serialize servers - let deserializedServers = - ((ServerRegistry.Deserialize serverDetails).TryFind dummy_currency_because_irrelevant_for_this_test).Value - |> List.ofSeq + seq { yield serverA } |> ServerRegistry.AddServer serverB) - Assert.That(deserializedServers.Length, Is.EqualTo 1) + let serversForCurrency = servers.[dummy_currency_because_irrelevant_for_this_test] + + Assert.That(serversForCurrency |> Seq.length, Is.EqualTo 1) [] - member __.``non-duplicate servers are not removed``() = + member __.``non-duplicate servers are added to colection``() = let serverA = { ServerInfo = @@ -562,16 +560,15 @@ type ServerReference() = } let servers = Map.empty.Add - (dummy_currency_because_irrelevant_for_this_test, seq { yield serverA; yield serverB }) - let serverDetails = ServerRegistry.Serialize servers - let deserializedServers = - ((ServerRegistry.Deserialize serverDetails).TryFind dummy_currency_because_irrelevant_for_this_test).Value - |> List.ofSeq + (dummy_currency_because_irrelevant_for_this_test, + seq { yield serverA } |> ServerRegistry.AddServer serverB) - Assert.That(deserializedServers.Length, Is.EqualTo 2) + let serversForCurrency = servers.[dummy_currency_because_irrelevant_for_this_test] + + Assert.That(serversForCurrency |> Seq.length, Is.EqualTo 2) member private __.SerializeAndDeserialize (serverA: ServerDetails) (serverB: ServerDetails): List = - let servers = seq { yield serverA; yield serverB } + let servers = seq { yield serverA } |> ServerRegistry.AddServer serverB let serverRanking = Map.empty.Add (dummy_currency_because_irrelevant_for_this_test, servers) let serverDetails = ServerRegistry.Serialize serverRanking ((ServerRegistry.Deserialize serverDetails).TryFind dummy_currency_because_irrelevant_for_this_test).Value diff --git a/src/GWallet.Backend/Caching.fs b/src/GWallet.Backend/Caching.fs index 2736032d3..38c207f0e 100644 --- a/src/GWallet.Backend/Caching.fs +++ b/src/GWallet.Backend/Caching.fs @@ -316,6 +316,11 @@ module Caching = address newCache)) + // When saving server rankings to disk, removal of duplicates and serializing/deserializing is performed, + // which puts load on CPU. This is acceptable for geewallet, since request rate is low, but not for + // ElectrumProxy, which has to process hundreds of request at a time. + member val SaveServerRankingsToDiskOnEachUpdate = true with get, set + member __.ClearAll () = SaveNetworkDataToDisk CachedNetworkData.Empty SaveServerRankingsToDisk Map.empty @@ -522,7 +527,7 @@ module Caching = if transactionCurrency <> feeCurrency && (not Config.EthTokenEstimationCouldBeBuggyAsInNotAccurate) then self.StoreTransactionRecord address feeCurrency txId feeAmount - member __.SaveServerLastStat (serverMatchFunc: ServerDetails->bool) + member self.SaveServerLastStat (serverMatchFunc: ServerDetails->bool) (stat: HistoryFact): unit = lock cacheFiles.ServerStats (fun _ -> let currency,serverInfo,previousLastSuccessfulCommunication = @@ -557,15 +562,21 @@ module Caching = | None -> Seq.empty | Some servers -> servers - let newServersForCurrency = - Seq.append (seq { yield newServerDetails }) serversForCurrency + let newServersForCurrency = ServerRegistry.AddServer newServerDetails serversForCurrency let newServerList = sessionServerRanking.Add(currency, newServersForCurrency) - let newCachedValue = SaveServerRankingsToDisk newServerList + let newCachedValue = + if self.SaveServerRankingsToDiskOnEachUpdate then + SaveServerRankingsToDisk newServerList + else + newServerList sessionServerRanking <- newCachedValue ) + member __.SaveServerStatsToDisk(): unit = + SaveServerRankingsToDisk sessionServerRanking |> ignore + member __.GetServers (currency: Currency): seq = lock cacheFiles.ServerStats (fun _ -> match sessionServerRanking.TryFind currency with diff --git a/src/GWallet.Backend/Config.fs b/src/GWallet.Backend/Config.fs index 9ccbf1ccf..5c8487b6c 100644 --- a/src/GWallet.Backend/Config.fs +++ b/src/GWallet.Backend/Config.fs @@ -10,6 +10,17 @@ open Fsdk open GWallet.Backend.FSharpUtil.UwpHacks +type NetworkTimeouts = + { + Timeout: TimeSpan + ConnectTimeout: TimeSpan + } + member self.Double() = + { + Timeout = self.Timeout + self.Timeout + ConnectTimeout = self.ConnectTimeout + self.ConnectTimeout + } + // TODO: make internal when tests don't depend on this anymore module Config = @@ -63,10 +74,7 @@ module Config = return simpleVersion } - // FIXME: make FaultTolerantParallelClient accept funcs that receive this as an arg, maybe 2x-ing it when a full - // round of failures has happened, as in, all servers failed - let internal DEFAULT_NETWORK_TIMEOUT = TimeSpan.FromSeconds 30.0 - let internal DEFAULT_NETWORK_CONNECT_TIMEOUT = TimeSpan.FromSeconds 5.0 + let internal DEFAULT_NETWORK_TIMEOUTS = { Timeout = TimeSpan.FromSeconds 5.0; ConnectTimeout = TimeSpan.FromSeconds 1.0 } let internal NUMBER_OF_RETRIES_TO_SAME_SERVERS = 3u diff --git a/src/GWallet.Backend/Ether/EtherServer.fs b/src/GWallet.Backend/Ether/EtherServer.fs index 3e86bbb9f..fe3351c0d 100644 --- a/src/GWallet.Backend/Ether/EtherServer.fs +++ b/src/GWallet.Backend/Ether/EtherServer.fs @@ -85,8 +85,8 @@ module Server = || ex.Message.Contains(SPrintF1 " %i." errorCode) let exMsg = "Could not communicate with EtherServer" - let PerformEtherRemoteCallWithTimeout<'T,'R> (job: Async<'R>): Async<'R> = async { - let! maybeResult = FSharpUtil.WithTimeout Config.DEFAULT_NETWORK_TIMEOUT job + let PerformEtherRemoteCallWithTimeout<'T,'R> (job: Async<'R>) (timeout: TimeSpan): Async<'R> = async { + let! maybeResult = FSharpUtil.WithTimeout timeout job match maybeResult with | None -> return raise <| ServerTimedOutException("Timeout when trying to communicate with Ether server") @@ -411,12 +411,13 @@ module Server = let Web3ServerToRetrievalFunc (server: ServerDetails) (web3ClientFunc: SomeWeb3->Async<'R>) currency + (timeouts: NetworkTimeouts) : Async<'R> = let HandlePossibleEtherFailures (job: Async<'R>): Async<'R> = async { try - let! result = PerformEtherRemoteCallWithTimeout job + let! result = PerformEtherRemoteCallWithTimeout job timeouts.Timeout return result with | ex -> @@ -428,9 +429,9 @@ module Server = let connectionTimeout = match currency with | Currency.ETC when etcEcosystemIsMomentarilyCentralized -> - Config.DEFAULT_NETWORK_TIMEOUT + Config.DEFAULT_NETWORK_TIMEOUT + timeouts.Double().Timeout | _ -> - Config.DEFAULT_NETWORK_TIMEOUT + timeouts.Timeout async { let web3Server = Web3Server (connectionTimeout, server) diff --git a/src/GWallet.Backend/Ether/TokenManager.fs b/src/GWallet.Backend/Ether/TokenManager.fs index ed00b7f22..023870150 100644 --- a/src/GWallet.Backend/Ether/TokenManager.fs +++ b/src/GWallet.Backend/Ether/TokenManager.fs @@ -47,6 +47,6 @@ module TokenManager = // this is a dummy instance we need in order to pass it to base class of StandardTokenService, but not // really used online; FIXME: propose "Web3-less" overload to Nethereum - let private dummyOfflineWeb3 = Web3 Config.DEFAULT_NETWORK_TIMEOUT + let private dummyOfflineWeb3 = Web3 Config.DEFAULT_NETWORK_TIMEOUTS.Timeout type OfflineTokenServiceWrapper(currency: Currency) = inherit TokenServiceWrapper(dummyOfflineWeb3, currency) diff --git a/src/GWallet.Backend/FaultTolerantParallelClient.fs b/src/GWallet.Backend/FaultTolerantParallelClient.fs index 1a71f0a28..3273905ec 100644 --- a/src/GWallet.Backend/FaultTolerantParallelClient.fs +++ b/src/GWallet.Backend/FaultTolerantParallelClient.fs @@ -153,11 +153,12 @@ type internal Runner<'Resource when 'Resource: equality> = (cancelState: ClientCancelState) (shouldReportUncanceledJobs: bool) (maybeExceptionHandler: Optionunit>) + (timeouts: NetworkTimeouts) : Async> = async { try try - let! res = server.Retrieval + let! res = server.Retrieval timeouts return SuccessfulValue res finally stopwatch.Stop() @@ -196,13 +197,14 @@ type internal Runner<'Resource when 'Resource: equality> = (cancelState: ClientCancelState) (updateServer: ('K->bool)->HistoryFact->unit) (server: Server<'K,'Resource>) + (timeouts: NetworkTimeouts) : ServerJob<'K,'Resource> = let job = async { let stopwatch = Stopwatch() stopwatch.Start() let! runResult = - Runner.Run<'K,'Ex> server stopwatch cancelState shouldReportUncanceledJobs exceptionHandler + Runner.Run<'K,'Ex> server stopwatch cancelState shouldReportUncanceledJobs exceptionHandler timeouts match runResult with | SuccessfulValue result -> @@ -234,13 +236,14 @@ type internal Runner<'Resource when 'Resource: equality> = (updateServerFunc: ('K->bool)->HistoryFact->unit) (funcs: List>) (cancelState: ClientCancelState) + (timeouts: NetworkTimeouts) : List>*List> = let launchFunc = Runner.CreateAsyncJobFromFunc<'K,'Ex> shouldReportUncanceledJobs exceptionHandler cancelState updateServerFunc let jobs = funcs - |> Seq.map launchFunc + |> Seq.map (fun each -> launchFunc each timeouts) |> List.ofSeq if parallelJobs < uint32 jobs.Length then List.splitAt (int parallelJobs) jobs @@ -289,6 +292,9 @@ type FaultTolerantParallelClient<'K,'E when 'K: equality and 'K :> ICommunicatio if typeof<'E> = typeof then raise (ArgumentException("'E cannot be System.Exception, use a derived one", "'E")) + /// it is doubled when all servers have failed + let mutable timeouts = Config.DEFAULT_NETWORK_TIMEOUTS + let MeasureConsistency (results: List<'R>) = results |> Seq.countBy id |> Seq.sortByDescending (fun (_,count: int) -> count) |> List.ofSeq @@ -483,6 +489,7 @@ type FaultTolerantParallelClient<'K,'E when 'K: equality and 'K :> ICommunicatio updateServer funcs cancelState + timeouts ) let startedTasks,jobsToLaunchLater = @@ -763,8 +770,13 @@ type FaultTolerantParallelClient<'K,'E when 'K: equality and 'K :> ICommunicatio 0u cancellationTokenSourceOption async { - let! res = job - return res + try + let! res = job + return res + with + | ex when FSharpUtil.FindException(ex).IsSome -> + timeouts <- timeouts.Double() + return raise <| FSharpUtil.ReRaise ex } member self.QueryWithCancellation<'R when 'R : equality> diff --git a/src/GWallet.Backend/GWallet.Backend.fsproj b/src/GWallet.Backend/GWallet.Backend.fsproj index 75e953bc2..7dd4c768e 100644 --- a/src/GWallet.Backend/GWallet.Backend.fsproj +++ b/src/GWallet.Backend/GWallet.Backend.fsproj @@ -35,6 +35,9 @@ + + + @@ -86,5 +89,6 @@ + diff --git a/src/GWallet.Backend/JsonRpcTcpClient.fs b/src/GWallet.Backend/JsonRpcTcpClient.fs index 9069a1065..a7d55405b 100644 --- a/src/GWallet.Backend/JsonRpcTcpClient.fs +++ b/src/GWallet.Backend/JsonRpcTcpClient.fs @@ -35,7 +35,7 @@ type ServerNameResolvedToInvalidAddressException = { inherit CommunicationUnsuccessfulException (info, context) } -type JsonRpcTcpClient (host: string, port: uint32) = +type JsonRpcTcpClient (host: string, port: uint32, timeouts: NetworkTimeouts) = let ResolveAsync (hostName: string): Async> = async { // FIXME: loop over all addresses? @@ -47,7 +47,7 @@ type JsonRpcTcpClient (host: string, port: uint32) = let ResolveHost(): Async = async { try - let! maybeTimedOutipAddress = ResolveAsync host |> FSharpUtil.WithTimeout Config.DEFAULT_NETWORK_TIMEOUT + let! maybeTimedOutipAddress = ResolveAsync host |> FSharpUtil.WithTimeout timeouts.Timeout match maybeTimedOutipAddress with | Some ipAddressOption -> match ipAddressOption with @@ -77,14 +77,14 @@ type JsonRpcTcpClient (host: string, port: uint32) = let rpcTcpClientInnerRequest = let tcpClient = - JsonRpcSharp.TcpClient.JsonRpcClient(ResolveHost, int port, Config.DEFAULT_NETWORK_CONNECT_TIMEOUT) + JsonRpcSharp.TcpClient.JsonRpcClient(ResolveHost, int port, timeouts.ConnectTimeout) fun jsonRequest -> tcpClient.RequestAsync jsonRequest member __.Host with get() = host member __.Request (request: string): Async = async { try - let! stringOption = rpcTcpClientInnerRequest request |> FSharpUtil.WithTimeout Config.DEFAULT_NETWORK_TIMEOUT + let! stringOption = rpcTcpClientInnerRequest request |> FSharpUtil.WithTimeout timeouts.Timeout let str = match stringOption with | Some s -> s diff --git a/src/GWallet.Backend/Server.fs b/src/GWallet.Backend/Server.fs index 8511bac2c..49ae57d33 100644 --- a/src/GWallet.Backend/Server.fs +++ b/src/GWallet.Backend/Server.fs @@ -90,33 +90,21 @@ module ServerRegistry = let listMap = Map.toList map tryFind listMap serverPredicate - let internal RemoveDupes (servers: seq) = - let rec removeDupesInternal (servers: seq) (serversMap: Map) = - match Seq.tryHead servers with - | None -> Seq.empty - | Some server -> - let tail = Seq.tail servers - match serversMap.TryGetValue server.ServerInfo.NetworkPath with - | false,_ -> - removeDupesInternal tail serversMap - | true,serverInMap -> - let serverToAppend = - match server.CommunicationHistory,serverInMap.CommunicationHistory with - | None,_ -> serverInMap - | _,None -> server - | Some (_, lastComm),Some (_, lastCommInMap) -> - if lastComm > lastCommInMap then - server - else - serverInMap - let newMap = serversMap.Remove serverToAppend.ServerInfo.NetworkPath - Seq.append (seq { yield serverToAppend }) (removeDupesInternal tail newMap) - - let initialServersMap = - servers - |> Seq.map (fun server -> server.ServerInfo.NetworkPath, server) - |> Map.ofSeq - removeDupesInternal servers initialServersMap + let AddServer (newServer: ServerDetails) (servers: seq) : seq = + let serversArray = Seq.toArray servers + match Array.tryFindIndex (fun each -> each.ServerInfo.NetworkPath = newServer.ServerInfo.NetworkPath) serversArray with + | Some index -> + let existingServer = serversArray.[index] + match newServer.CommunicationHistory, existingServer.CommunicationHistory with + | None, _ -> () + | _, None -> + serversArray.[index] <- newServer + | Some (_, newLastComm),Some (_, existingLastCommInMap) when newLastComm > existingLastCommInMap -> + serversArray.[index] <- newServer + | _ -> () + serversArray :> seq + | None -> + Array.append serversArray (Array.singleton newServer) :> seq let internal RemoveBlackListed (cs: Currency*seq): seq = let isBlackListed currency server = @@ -134,7 +122,7 @@ module ServerRegistry = Seq.filter (fun server -> not (isBlackListed currency server)) servers let RemoveCruft (cs: Currency*seq): seq = - cs |> RemoveBlackListed |> RemoveDupes + cs |> RemoveBlackListed let internal Sort (servers: seq): seq = let sort server = @@ -186,7 +174,12 @@ module ServerRegistry = | None -> Seq.empty | Some servers -> servers - let allServers = (currency, Seq.append allServersFrom1 allServersFrom2) + let mergedServers = + Seq.fold + (fun servers newServer -> AddServer newServer servers) + allServersFrom1 + allServersFrom2 + let allServers = (currency, mergedServers) |> RemoveCruft |> Sort @@ -202,7 +195,7 @@ module ServerRegistry = [] type Server<'K,'R when 'K: equality and 'K :> ICommunicationHistory> = { Details: 'K - Retrieval: Async<'R> } + Retrieval: NetworkTimeouts -> Async<'R> } override self.Equals yObj = match yObj with | :? Server<'K,'R> as y -> diff --git a/src/GWallet.Backend/UtxoCoin/BitcoreNodeClient.fs b/src/GWallet.Backend/UtxoCoin/BitcoreNodeClient.fs new file mode 100644 index 000000000..d2a61c701 --- /dev/null +++ b/src/GWallet.Backend/UtxoCoin/BitcoreNodeClient.fs @@ -0,0 +1,24 @@ +namespace GWallet.Backend + +open System.Text.Json + +open Fsdk.FSharpUtil + +open GWallet.Backend +open GWallet.Backend.UtxoCoin +open GWallet.Backend.FSharpUtil.UwpHacks + + +/// see https://github.com/bitpay/bitcore/blob/master/packages/bitcore-node/docs/api-documentation.md +type BitcoreNodeClient(serverAddress: string) = + inherit RestAPIClient(serverAddress, 1u) + + member self.GetAddressTransactions(address: string): Async> = + async { + let request = SPrintF1 "/api/BTC/mainnet/address/%s/txs" address + let! response = self.Request request + let json = JsonDocument.Parse response + return [| for entry in json.RootElement.EnumerateArray() -> + { TxHash = entry.GetProperty("mintTxid").GetString(); + Height = entry.GetProperty("mintHeight").GetUInt64() } |] + } diff --git a/src/GWallet.Backend/UtxoCoin/BlockbookClient.fs b/src/GWallet.Backend/UtxoCoin/BlockbookClient.fs new file mode 100644 index 000000000..aadf4c9ef --- /dev/null +++ b/src/GWallet.Backend/UtxoCoin/BlockbookClient.fs @@ -0,0 +1,24 @@ +namespace GWallet.Backend + +open System.Text.Json + +open Fsdk.FSharpUtil + +open GWallet.Backend +open GWallet.Backend.UtxoCoin +open GWallet.Backend.FSharpUtil.UwpHacks + + +/// Client for Blockbook API used by Trezor. See https://github.com/trezor/blockbook/blob/master/docs/api.md +type BlockbookClient(serverAddress: string) = + inherit RestAPIClient(serverAddress, 1u) + + member self.GetAddressTransactions(address: string): Async> = + async { + let request = SPrintF1 "/api/v2/utxo/%s?confirmed=true" address + let! response = self.Request request + let json = JsonDocument.Parse response + return [| for entry in json.RootElement.EnumerateArray() -> + { TxHash = entry.GetProperty("txid").GetString(); + Height = entry.GetProperty("height").GetUInt64() } |] + } diff --git a/src/GWallet.Backend/UtxoCoin/ElectrumClient.fs b/src/GWallet.Backend/UtxoCoin/ElectrumClient.fs index fda5132fa..d9a85665c 100644 --- a/src/GWallet.Backend/UtxoCoin/ElectrumClient.fs +++ b/src/GWallet.Backend/UtxoCoin/ElectrumClient.fs @@ -7,8 +7,8 @@ open GWallet.Backend.FSharpUtil.UwpHacks module ElectrumClient = - let private Init (fqdn: string) (port: uint32): Async = - let jsonRpcClient = new JsonRpcTcpClient(fqdn, port) + let private Init (fqdn: string) (port: uint32) (timeouts: NetworkTimeouts): Async = + let jsonRpcClient = new JsonRpcTcpClient(fqdn, port, timeouts) let stratumClient = new StratumClient(jsonRpcClient) // this is the last version of Electrum released at the time of writing this module @@ -44,12 +44,12 @@ module ElectrumClient = return stratumClient } - let StratumServer (electrumServer: ServerDetails): Async = + let StratumServer (electrumServer: ServerDetails) (timeouts: NetworkTimeouts): Async = match electrumServer.ServerInfo.ConnectionType with | { Encrypted = true; Protocol = _ } -> failwith "Incompatibility filter for non-encryption didn't work?" | { Encrypted = false; Protocol = Http } -> failwith "HTTP server for UtxoCoin?" | { Encrypted = false; Protocol = Tcp port } -> - Init electrumServer.ServerInfo.NetworkPath port + Init electrumServer.ServerInfo.NetworkPath port timeouts let GetBalances (scriptHashes: List) (stratumServer: Async) = async { // FIXME: we should rather implement this method in terms of: diff --git a/src/GWallet.Backend/UtxoCoin/RestApiClient.fs b/src/GWallet.Backend/UtxoCoin/RestApiClient.fs new file mode 100644 index 000000000..9e6fb6165 --- /dev/null +++ b/src/GWallet.Backend/UtxoCoin/RestApiClient.fs @@ -0,0 +1,54 @@ +namespace GWallet.Backend + +open System +open System.Net.Http + +open Fsdk.FSharpUtil + +open GWallet.Backend +open GWallet.Backend.UtxoCoin +open GWallet.Backend.FSharpUtil.UwpHacks + + +/// Abstract base class for clients for REST APIs such as Bitcore or Blockbook +[] +type RestAPIClient(serverAddress: string, ?maxConcurrentRequests: uint32) = + let httpClient = new HttpClient(BaseAddress=Uri serverAddress, Timeout=Config.DEFAULT_NETWORK_TIMEOUTS.Timeout) + + let mutable lastRequestTime = DateTime.Now + let minTimeBetweenRequests = 0.1 + let semaphore = new System.Threading.SemaphoreSlim(defaultArg maxConcurrentRequests 1u |> int) + + interface IDisposable with + override self.Dispose (): unit = + httpClient.Dispose() + semaphore.Dispose() + + member internal self.Request(request: string): Async = + async { + try + try + do! semaphore.WaitAsync() |> Async.AwaitTask + let diff = (DateTime.Now - lastRequestTime).TotalSeconds + if diff < minTimeBetweenRequests then + do! Async.Sleep <| int ((minTimeBetweenRequests - diff) * 1000.0) + let! result = httpClient.GetStringAsync request |> Async.AwaitTask + lastRequestTime <- DateTime.Now + return result + finally + semaphore.Release() |> ignore + with + | ex -> + match FindException ex with + | Some httpRequestExn -> + // maybe only discard server on several specific errors? + let msg = SPrintF2 "%s: %s" (httpRequestExn.GetType().FullName) httpRequestExn.Message + return raise <| ServerDiscardedException(msg, httpRequestExn) + | _ -> () + match FindException ex with + | Some taskCancelledExn -> + let msg = SPrintF1 "Timeout: %s" taskCancelledExn.Message + return raise <| ServerDiscardedException(msg, taskCancelledExn) + | _ -> () + return raise (ReRaise ex) + } diff --git a/src/GWallet.Backend/UtxoCoin/StratumClient.fs b/src/GWallet.Backend/UtxoCoin/StratumClient.fs index a9f0112b6..693bd81c5 100644 --- a/src/GWallet.Backend/UtxoCoin/StratumClient.fs +++ b/src/GWallet.Backend/UtxoCoin/StratumClient.fs @@ -71,6 +71,49 @@ type BlockchainTransactionBroadcastResult = Result: string; } +type BlockchainBlockHeaderResult = + { + Id: int; + Result: string; + } + +type BlockchainBlockHeadersInnerResult = + { + Count: uint64 + Hex: string + Max: uint64 + } + +type BlockchainBlockHeadersResult = + { + Id: int; + Result: BlockchainBlockHeadersInnerResult; + } + +type BlockchainScriptHashGetHistoryInnerResult = + { + Height: uint64 + TxHash: string + } + +type BlockchainScriptHashGetHistoryResult = + { + Id: int; + Result: array; + } + +type BlockchainHeadersSubscribeInnerResult = + { + Height: uint64 + Hex: string + } + +type BlockchainHeadersSubscribeResult = + { + Id: int; + Result: BlockchainHeadersSubscribeInnerResult; + } + type ErrorInnerResult = { Message: string; @@ -216,6 +259,45 @@ type StratumClient (jsonRpcClient: JsonRpcTcpClient) = | true -> StratumClient.DeserializeInternal result + member self.BlockchainBlockHeader (height: uint64): Async = + let obj = { + Id = 0; + Method = "blockchain.block.header"; + Params = [height] + } + let json = Serialize obj + + async { + let! resObj,_ = self.Request json + return resObj + } + + member self.BlockchainBlockHeaders (start_height: uint64) (count: uint64): Async = + let obj = { + Id = 0; + Method = "blockchain.block.headers"; + Params = [start_height; count] + } + let json = Serialize obj + + async { + let! resObj,_ = self.Request json + return resObj + } + + member self.BlockchainHeadersSubscribe (): Async = + let obj = { + Id = 0; + Method = "blockchain.headers.subscribe"; + Params = [] + } + let json = Serialize obj + + async { + let! resObj,_ = self.Request json + return resObj + } + member self.BlockchainScriptHashGetBalance address: Async = let obj = { Id = 0; @@ -229,6 +311,19 @@ type StratumClient (jsonRpcClient: JsonRpcTcpClient) = return resObj } + member self.BlockchainScriptHashGetHistory address: Async = + let obj = { + Id = 0; + Method = "blockchain.scripthash.get_history"; + Params = [address] + } + let json = Serialize obj + + async { + let! resObj,_ = self.Request json + return resObj + } + static member private CreateVersion(versionStr: string): Version = let correctedVersion = if (versionStr.EndsWith("+")) then diff --git a/src/GWallet.Backend/UtxoCoin/UtxoCoinServer.fs b/src/GWallet.Backend/UtxoCoin/UtxoCoinServer.fs index f5eb49dc3..4d35de51a 100644 --- a/src/GWallet.Backend/UtxoCoin/UtxoCoinServer.fs +++ b/src/GWallet.Backend/UtxoCoin/UtxoCoinServer.fs @@ -21,7 +21,7 @@ module Server = | ServerSelectionMode.Fast -> 3u | ServerSelectionMode.Analysis -> 2u - let private FaultTolerantParallelClientDefaultSettings (mode: ServerSelectionMode) + let FaultTolerantParallelClientDefaultSettings (mode: ServerSelectionMode) maybeConsistencyConfig = let consistencyConfig = match maybeConsistencyConfig with @@ -66,9 +66,10 @@ module Server = let ElectrumServerToRetrievalFunc (server: ServerDetails) (electrumClientFunc: Async->Async<'R>) + (timeouts: NetworkTimeouts) : Async<'R> = async { try - let stratumClient = ElectrumClient.StratumServer server + let stratumClient = ElectrumClient.StratumServer server timeouts return! electrumClientFunc stratumClient // NOTE: try to make this 'with' block be in sync with the one in EtherServer:GetWeb3Funcs() @@ -93,7 +94,7 @@ module Server = electrumServers serverFuncs - let private GetRandomizedFuncs<'R> (currency: Currency) + let GetRandomizedFuncs<'R> (currency: Currency) (electrumClientFunc: Async->Async<'R>) : List> = diff --git a/src/GWallet.Frontend.XF.Android/GWallet.Frontend.XF.Android.fsproj b/src/GWallet.Frontend.XF.Android/GWallet.Frontend.XF.Android.fsproj index 7be4ac652..78b86fc51 100644 --- a/src/GWallet.Frontend.XF.Android/GWallet.Frontend.XF.Android.fsproj +++ b/src/GWallet.Frontend.XF.Android/GWallet.Frontend.XF.Android.fsproj @@ -20,6 +20,10 @@ Assets + pixel_4a_-_api_31 + Emulator + Pixel 4a - API 31 (Android 12.0 - API 31) + pixel_4a_-_api_31 true @@ -295,6 +299,9 @@ ..\..\packages\Microsoft.Extensions.Logging.Abstractions.1.0.2\lib\netstandard1.1\Microsoft.Extensions.Logging.Abstractions.dll + + ..\..\packages\Microsoft.Bcl.AsyncInterfaces.8.0.0\lib\netstandard2.0\Microsoft.Bcl.AsyncInterfaces.dll + ..\..\packages\NBitcoin.6.0.17\lib\netstandard2.1\NBitcoin.dll @@ -357,6 +364,12 @@ True ..\..\packages\System.Memory.4.5.5\lib\netstandard2.0\System.Memory.dll + + ..\..\packages\System.Text.Json.8.0.5\lib\netstandard2.0\System.Text.Json.dll + + + ..\..\packages\System.Text.Encodings.Web.8.0.0\lib\netstandard2.0\System.Text.Encodings.Web.dll + ..\..\packages\Xamarin.Android.Arch.Core.Common.1.1.1.3\lib\monoandroid90\Xamarin.Android.Arch.Core.Common.dll @@ -512,7 +525,7 @@ GWallet.Frontend.XF - {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2} + {96F9B3E5-11F8-4F5F-AADC-51D0D995B3D2} GWallet.Backend @@ -671,7 +684,6 @@ - MSB3277 - + \ No newline at end of file diff --git a/src/GWallet.Frontend.XF.Android/packages.config b/src/GWallet.Frontend.XF.Android/packages.config index ed414b2f5..0edb53565 100644 --- a/src/GWallet.Frontend.XF.Android/packages.config +++ b/src/GWallet.Frontend.XF.Android/packages.config @@ -9,6 +9,7 @@ + @@ -69,7 +70,9 @@ + +