From b2f576343da7ecbc88d274a7d6d023de3bbff359 Mon Sep 17 00:00:00 2001 From: Ken Sedgwick Date: Thu, 25 Jul 2024 14:02:27 -0700 Subject: [PATCH 01/18] Add Util::BacktraceException to capture and display backtraces --- ChangeLog | 10 ++++ Makefile.am | 1 + README.md | 1 + Util/BacktraceException.hpp | 109 ++++++++++++++++++++++++++++++++++++ 4 files changed, 121 insertions(+) create mode 100644 Util/BacktraceException.hpp diff --git a/ChangeLog b/ChangeLog index b8416f2a8..4cf36e638 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [Unreleased] + +### Added + +- Added `Util::BacktraceException` which captures backtraces where an + exception is thrown and then formats them for debugging when they + are displayed with `what()`. The backtraces are more useful if the + following configuration is used: `./configure CXXFLAGS="-g -Og"` but + this results in larger, less optimized binaries. + ## [0.13.3] - 2024-08-09: "Blinded by the Light" This point release fixes an important bug by restoring the earned fee diff --git a/Makefile.am b/Makefile.am index c64ed4acb..750638325 100644 --- a/Makefile.am +++ b/Makefile.am @@ -516,6 +516,7 @@ libclboss_la_SOURCES = \ Stats/RunningMean.cpp \ Stats/RunningMean.hpp \ Stats/WeightedMedian.hpp \ + Util/BacktraceException.hpp \ Util/Bech32.cpp \ Util/Bech32.hpp \ Util/Compiler.hpp \ diff --git a/README.md b/README.md index 1dc1d88f4..b0b444547 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ would be 20x larger!), but if it matters to you, you can override the CLBOSS default via `CXXFLAGS`, such as: ./configure CXXFLAGS="-g -O2" # or whatever flags you like + ./configure CXXFLAGS="-g -Og" # recommended for debugging And if your build machine has more than 1 core, you probably want to pass in the `-j` option to `make`, too: diff --git a/Util/BacktraceException.hpp b/Util/BacktraceException.hpp new file mode 100644 index 000000000..313efb834 --- /dev/null +++ b/Util/BacktraceException.hpp @@ -0,0 +1,109 @@ +#ifndef UTIL_BACKTRACE_EXCEPTION_HPP +#define UTIL_BACKTRACE_EXCEPTION_HPP + +#include +#include +#include +#include +#include +#include +#include + +namespace Util { + +/** class Util::BacktraceException + * + * @brief A wrapper for an exception E which additionally stores a + * backtrace when it is constructed. The backtrace formatting is + * deferred until `what()` is called on the handled exception. + */ + +struct PcloseDeleter { + void operator()(FILE* fp) const { + if (fp) { + pclose(fp); + } + } +}; + +template +class BacktraceException : public T { +public: + template + BacktraceException(Args&&... args) + : T(std::forward(args)...), full_message_(T::what()) { + capture_backtrace(); + } + + const char* what() const noexcept override { + if (!formatted_) { + formatted_ = true; + full_message_ = create_full_message(T::what()); + } + return full_message_.c_str(); + } + +private: + static constexpr size_t MAX_FRAMES = 100; + mutable bool formatted_ = false; + mutable std::string full_message_; + mutable void* backtrace_addresses_[MAX_FRAMES]; + mutable size_t stack_depth; + + void capture_backtrace() { + memset(backtrace_addresses_, 0, sizeof(backtrace_addresses_)); + stack_depth = backtrace(backtrace_addresses_, sizeof(backtrace_addresses_) / sizeof(void*)); + } + + std::string format_backtrace() const { + char** symbols = backtrace_symbols(backtrace_addresses_, stack_depth); + std::ostringstream oss; + for (size_t i = 0; i < stack_depth; ++i) { + oss << '#' << std::left << std::setfill(' ') << std::setw(2) << i << ' '; + auto line = addr2line(backtrace_addresses_[i]); + if (line.find("??") != std::string::npos) { + // If addr2line doesn't find a good + // answer use basic information + // instead. + oss << symbols[i] << std::endl; + } else { + oss << line; + } + } + free(symbols); + return oss.str(); + } + + // Unfortunately there is no simple way to get a high quality + // backtrace using in-process libraries. Instead for now we + // popen an addr2line process and use it's output. + std::string addr2line(void* addr) const { + char cmd[512]; + snprintf(cmd, sizeof(cmd), + "addr2line -C -f -p -e %s %p", program_invocation_name, addr); + + std::array buffer; + std::string result; + std::unique_ptr pipe(popen(cmd, "r")); + + if (!pipe) { + return " -- error: unable to open addr2line"; + } + + while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) { + result += buffer.data(); + } + + return result; + } + + std::string create_full_message(const std::string& message) const { + std::ostringstream oss; + oss << message << "\nBacktrace:\n" << format_backtrace(); + return oss.str(); + } +}; + +} // namespace Util + +#endif /* UTIL_BACKTRACE_EXCEPTION_HPP */ From a639a59d33aed645c4fb24ccfaf6ee26d1e8eafd Mon Sep 17 00:00:00 2001 From: Ken Sedgwick Date: Thu, 25 Jul 2024 17:00:18 -0700 Subject: [PATCH 02/18] Use BacktraceException for appropriate (most) exceptions --- Bitcoin/Tx.cpp | 4 ++-- Bitcoin/addr_to_scriptPubKey.hpp | 5 +++-- Bitcoin/sighash.hpp | 4 ++-- Boltz/ConnectionIF.hpp | 5 +++-- Boss/Mod/Initiator.cpp | 2 +- Boss/Mod/Rpc.cpp | 8 ++++---- Boss/Mod/Rpc.hpp | 2 +- Boss/Mod/UnmanagedManager.cpp | 2 +- Boss/Signer.cpp | 12 ++++++------ Boss/open_rpc_socket.cpp | 9 +++++---- Boss/random_engine.cpp | 3 ++- DnsSeed/Detail/decode_bech32_node.cpp | 7 ++++--- Ev/ThreadPool.cpp | 3 ++- Ev/runcmd.cpp | 3 ++- Jsmn/Object.cpp | 2 +- Jsmn/Object.hpp | 6 +++--- Jsmn/ParseError.hpp | 5 +++-- Ln/Amount.cpp | 5 +++-- Ln/NodeId.cpp | 2 +- Ln/Preimage.cpp | 2 +- Ln/Scid.cpp | 3 ++- Net/IPAddr.hpp | 5 +++-- Net/SocketFd.cpp | 5 +++-- Ripemd160/Hash.cpp | 2 +- Secp256k1/Detail/context.cpp | 3 ++- Secp256k1/PrivKey.cpp | 4 ++-- Secp256k1/PrivKey.hpp | 5 +++-- Secp256k1/PubKey.cpp | 4 ++-- Secp256k1/PubKey.hpp | 5 +++-- Secp256k1/Signature.cpp | 2 +- Secp256k1/Signature.hpp | 6 ++++-- Sha256/Hash.cpp | 2 +- Sqlite3/Db.cpp | 7 ++++--- Sqlite3/Detail/binds.cpp | 9 +++++---- Sqlite3/Query.cpp | 3 ++- Sqlite3/Result.cpp | 3 ++- Sqlite3/Tx.cpp | 3 ++- Stats/WeightedMedian.hpp | 5 +++-- Util/Str.hpp | 6 +++--- Uuid.cpp | 2 +- 40 files changed, 99 insertions(+), 76 deletions(-) diff --git a/Bitcoin/Tx.cpp b/Bitcoin/Tx.cpp index 5fdb29535..431ae7721 100644 --- a/Bitcoin/Tx.cpp +++ b/Bitcoin/Tx.cpp @@ -88,9 +88,9 @@ Tx::Tx(std::string const& s) { auto is = std::istringstream(std::move(str)); is >> *this; if (!is.good()) - throw std::invalid_argument("Bitcoin::Tx: invalid hex string input."); + throw Util::BacktraceException("Bitcoin::Tx: invalid hex string input."); if (is.get() != std::char_traits::eof()) - throw std::invalid_argument("Bitcoin::Tx: input string too long."); + throw Util::BacktraceException("Bitcoin::Tx: input string too long."); } Bitcoin::TxId Tx::get_txid() const { diff --git a/Bitcoin/addr_to_scriptPubKey.hpp b/Bitcoin/addr_to_scriptPubKey.hpp index 5f799a07e..93248e43e 100644 --- a/Bitcoin/addr_to_scriptPubKey.hpp +++ b/Bitcoin/addr_to_scriptPubKey.hpp @@ -1,6 +1,7 @@ #ifndef BITCOIN_ADDR_TO_SCRIPTPUBKEY_HPP #define BITCOIN_ADDR_TO_SCRIPTPUBKEY_HPP +#include"Util/BacktraceException.hpp" #include #include #include @@ -8,10 +9,10 @@ namespace Bitcoin { -struct UnknownAddrType : public std::invalid_argument { +struct UnknownAddrType : public Util::BacktraceException { public: UnknownAddrType() - : std::invalid_argument("Bitcoin::UnknownAddrType") { } + : Util::BacktraceException("Bitcoin::UnknownAddrType") { } }; /** Bitcoin::addr_to_scriptPubKey diff --git a/Bitcoin/sighash.hpp b/Bitcoin/sighash.hpp index a3b1e5e5d..2eb9a1fb6 100644 --- a/Bitcoin/sighash.hpp +++ b/Bitcoin/sighash.hpp @@ -19,10 +19,10 @@ enum SighashFlags , SIGHASH_ANYONECANPAY = 0x80 }; -struct InvalidSighash : public std::invalid_argument { +struct InvalidSighash : public Util::BacktraceException { InvalidSighash() =delete; InvalidSighash(std::string const& msg) - : std::invalid_argument( + : Util::BacktraceException( std::string("Bitcoin::InvalidSighash: ") + msg ) { } }; diff --git a/Boltz/ConnectionIF.hpp b/Boltz/ConnectionIF.hpp index a03c8f5ce..e85abc288 100644 --- a/Boltz/ConnectionIF.hpp +++ b/Boltz/ConnectionIF.hpp @@ -1,6 +1,7 @@ #ifndef BOLTZ_CONNECTIONIF_HPP #define BOLTZ_CONNECTIONIF_HPP +#include"Util/BacktraceException.hpp" #include #include #include @@ -35,10 +36,10 @@ class ConnectionIF { * @brief thrown when communications with * the BOLTZ server has problems. */ -class ApiError : public std::runtime_error { +class ApiError : public Util::BacktraceException { public: ApiError(std::string const& e - ) : std::runtime_error(e) { } + ) : Util::BacktraceException(e) { } }; } diff --git a/Boss/Mod/Initiator.cpp b/Boss/Mod/Initiator.cpp index c6735aa30..08b0df36b 100644 --- a/Boss/Mod/Initiator.cpp +++ b/Boss/Mod/Initiator.cpp @@ -104,7 +104,7 @@ class Initiator::Impl { , meth , is.c_str() ).then([]() { - throw std::runtime_error("Unexpected result."); + throw Util::BacktraceException("Unexpected result."); return Ev::lift(); }); } diff --git a/Boss/Mod/Rpc.cpp b/Boss/Mod/Rpc.cpp index 02f67b350..524669544 100644 --- a/Boss/Mod/Rpc.cpp +++ b/Boss/Mod/Rpc.cpp @@ -60,7 +60,7 @@ std::string RpcError::make_error_message( std::string const& command RpcError::RpcError( std::string command_ , Jsmn::Object error_ - ) : std::runtime_error(make_error_message(command_, error_)) + ) : Util::BacktraceException(make_error_message(command_, error_)) , command(command_) , error(error_) { } @@ -200,13 +200,13 @@ class Rpc::Impl { return std::size_t(0); } if (res < 0) - throw std::runtime_error( + throw Util::BacktraceException( std::string("Rpc: read: ") + strerror(errno) ); if (res == 0) /* Unexpected end of file! */ - throw std::runtime_error( + throw Util::BacktraceException( "Rpc: read: unexpected end-of-file " "in RPC socket." ); @@ -263,7 +263,7 @@ class Rpc::Impl { )) break; if (res < 0) - throw std::runtime_error( + throw Util::BacktraceException( std::string("Rpc: write: ") + strerror(errno) ); diff --git a/Boss/Mod/Rpc.hpp b/Boss/Mod/Rpc.hpp index 41bccfbc1..41d090904 100644 --- a/Boss/Mod/Rpc.hpp +++ b/Boss/Mod/Rpc.hpp @@ -13,7 +13,7 @@ namespace S { class Bus; } namespace Boss { namespace Mod { -struct RpcError : public std::runtime_error { +struct RpcError : public Util::BacktraceException { private: static std::string make_error_message( std::string const& diff --git a/Boss/Mod/UnmanagedManager.cpp b/Boss/Mod/UnmanagedManager.cpp index f2ce0b478..4f12bfb43 100644 --- a/Boss/Mod/UnmanagedManager.cpp +++ b/Boss/Mod/UnmanagedManager.cpp @@ -272,7 +272,7 @@ class UnmanagedManager::Impl { for (auto const& tag : tags) { auto it = tag_informs.find(tag); if (it == tag_informs.end()) - throw std::runtime_error( + throw Util::BacktraceException( std::string("Unknown tag: ") + tag ); } diff --git a/Boss/Signer.cpp b/Boss/Signer.cpp index ee3b3d4d3..57262a326 100644 --- a/Boss/Signer.cpp +++ b/Boss/Signer.cpp @@ -77,7 +77,7 @@ void try_create_privkey_file( std::string const& privkey_filename /* Benign failure, file already exists. */ return; if (!fd) - throw std::runtime_error( + throw Util::BacktraceException( std::string("Boss::Signer: creat(\"") + privkey_filename + "\"): " + strerror(errno) @@ -88,7 +88,7 @@ void try_create_privkey_file( std::string const& privkey_filename auto res = Util::Rw::write_all(fd.get(), &sk, sizeof(sk)); if (!res) { unlink_noerr(privkey_filename); - throw std::runtime_error( + throw Util::BacktraceException( std::string("Boss::Signer: write(\"") + privkey_filename + "\"): " + strerror(errno) @@ -101,7 +101,7 @@ Secp256k1::PrivKey read_privkey_file(std::string const& privkey_filename) { auto fd = Net::Fd(open(privkey_filename.c_str(), O_RDONLY)); if (!fd) - throw std::runtime_error( + throw Util::BacktraceException( std::string("Boss::Signer: open(\"") + privkey_filename + "\"): " + strerror(errno) @@ -111,13 +111,13 @@ read_privkey_file(std::string const& privkey_filename) { auto size = sizeof(sk); auto res = Util::Rw::read_all(fd.get(), &sk, size); if (!res) - throw std::runtime_error( + throw Util::BacktraceException( std::string("Boss::Signer: read(\"") + privkey_filename + "\"): " + strerror(errno) ); if (size != sizeof(sk)) - throw std::runtime_error( + throw Util::BacktraceException( std::string("Boss::Signer: read(\"") + privkey_filename + "\"): unexpected end-of-file" @@ -196,7 +196,7 @@ class Signer::Impl { /* Check it matches. */ auto actual_pk = Secp256k1::PubKey(self->sk); if (actual_pk != pubkey) - throw std::runtime_error( + throw Util::BacktraceException( std::string("Boss::Signer: ") + self->privkey_filename + ": not matched to database!" diff --git a/Boss/open_rpc_socket.cpp b/Boss/open_rpc_socket.cpp index f3e941149..1c6fb3800 100644 --- a/Boss/open_rpc_socket.cpp +++ b/Boss/open_rpc_socket.cpp @@ -1,5 +1,6 @@ #include"Boss/open_rpc_socket.hpp" #include"Net/Fd.hpp" +#include"Util/BacktraceException.hpp" #include #include #include @@ -15,19 +16,19 @@ Net::Fd open_rpc_socket( std::string const& lightning_dir ) { auto chdir_res = chdir(lightning_dir.c_str()); if (chdir_res < 0) - throw std::runtime_error( + throw Util::BacktraceException( std::string("chdir: ") + strerror(errno) ); auto fd = Net::Fd(socket(AF_UNIX, SOCK_STREAM, 0)); if (!fd) - throw std::runtime_error( + throw Util::BacktraceException( std::string("socket: ") + strerror(errno) ); auto addr = sockaddr_un(); if (rpc_file.length() + 1 > sizeof(addr.sun_path)) - throw std::runtime_error( + throw Util::BacktraceException( std::string("sizeof(sun_path): ") + strerror(ENOSPC) ); @@ -42,7 +43,7 @@ Net::Fd open_rpc_socket( std::string const& lightning_dir ); } while (connect_res < 0 && errno == EINTR); if (connect_res < 0) - throw std::runtime_error( + throw Util::BacktraceException( std::string("connect: ") + strerror(errno) ); diff --git a/Boss/random_engine.cpp b/Boss/random_engine.cpp index 72f124168..505bffc1c 100644 --- a/Boss/random_engine.cpp +++ b/Boss/random_engine.cpp @@ -1,5 +1,6 @@ #include"Boss/random_engine.hpp" #include"Net/Fd.hpp" +#include"Util/BacktraceException.hpp" #include #include #include @@ -18,7 +19,7 @@ std::default_random_engine initialize_random_engine() { dr = Net::Fd(open("/dev/random", O_RDONLY)); } while (!dr && errno == EINTR); if (!dr) - throw std::runtime_error( + throw Util::BacktraceException( std::string("open /dev/random: ") + strerror(errno) ); diff --git a/DnsSeed/Detail/decode_bech32_node.cpp b/DnsSeed/Detail/decode_bech32_node.cpp index 7bb0c0d5a..148089e39 100644 --- a/DnsSeed/Detail/decode_bech32_node.cpp +++ b/DnsSeed/Detail/decode_bech32_node.cpp @@ -1,4 +1,5 @@ #include"DnsSeed/Detail/decode_bech32_node.hpp" +#include"Util/BacktraceException.hpp" #include"Util/Bech32.hpp" #include #include @@ -12,19 +13,19 @@ std::vector decode_bech32_node(std::string const& s) { auto bitstream = std::vector(); if (!Util::Bech32::decode(hrp, bitstream, s)) - throw std::runtime_error( + throw Util::BacktraceException( std::string("Not a bech32 string: ") + s ); /* 264 bits needed, but bech32 encodes in batches of 5 bits so * we round up to 265. */ if (bitstream.size() != 33 * 8 + 1) - throw std::runtime_error( + throw Util::BacktraceException( std::string("Data part wrong size: ") + s ); /* The last bit should be 0. */ if (bitstream[264] == 1) - throw std::runtime_error( + throw Util::BacktraceException( std::string("Odd node id: ") + s ); /* And should be removed. */ diff --git a/Ev/ThreadPool.cpp b/Ev/ThreadPool.cpp index d68636668..fce64cb72 100644 --- a/Ev/ThreadPool.cpp +++ b/Ev/ThreadPool.cpp @@ -12,6 +12,7 @@ #include #include #include"Ev/ThreadPool.hpp" +#include"Util/BacktraceException.hpp" #include"Util/make_unique.hpp" namespace { @@ -182,7 +183,7 @@ class ThreadPool::Impl { int pipes[2]; auto pipe_res = pipe(pipes); if (pipe_res < 0) { - throw std::runtime_error(std::string("Ev::ThreadPool: pipe:") + throw Util::BacktraceException(std::string("Ev::ThreadPool: pipe:") + strerror(errno) ); } diff --git a/Ev/runcmd.cpp b/Ev/runcmd.cpp index 26c992d0b..52cd97292 100644 --- a/Ev/runcmd.cpp +++ b/Ev/runcmd.cpp @@ -1,6 +1,7 @@ #include"Ev/Io.hpp" #include"Ev/runcmd.hpp" #include"Net/Fd.hpp" +#include"Util/BacktraceException.hpp" #include"Util/make_unique.hpp" #include #include @@ -62,7 +63,7 @@ class RunCmd { static void error(std::unique_ptr self, std::string msg) { try { - throw std::runtime_error( + throw Util::BacktraceException>( std::string("pipecmd: ") + msg + ": " + strerror(errno) ); diff --git a/Jsmn/Object.cpp b/Jsmn/Object.cpp index 8333f604c..f520745b0 100644 --- a/Jsmn/Object.cpp +++ b/Jsmn/Object.cpp @@ -386,7 +386,7 @@ std::istream& operator>>(std::istream& is, Jsmn::Object& o) { if (!is || is.eof() || !is.get(buf[0])) { if (!started) return is; - throw std::runtime_error("Unexpected end-of-file."); + throw Util::BacktraceException("Unexpected end-of-file."); } started = true; diff --git a/Jsmn/Object.hpp b/Jsmn/Object.hpp index e2bca4b7d..db7dacb9e 100644 --- a/Jsmn/Object.hpp +++ b/Jsmn/Object.hpp @@ -7,6 +7,7 @@ #include #include #include +#include"Util/BacktraceException.hpp" #include #include @@ -16,10 +17,9 @@ namespace Jsmn { class ParserExposedBuffer; } namespace Jsmn { /* Thrown when converting or using as incorrect type. */ -/* FIXME: Use a backtrace-catching exception. */ -class TypeError : public std::invalid_argument { +class TypeError : public Util::BacktraceException { public: - TypeError() : std::invalid_argument("Incorrect type.") { } + TypeError() : Util::BacktraceException("Incorrect type.") { } }; /* Represents an object that has been parsed from a JSON string. */ diff --git a/Jsmn/ParseError.hpp b/Jsmn/ParseError.hpp index abcfd9edd..16f94660e 100644 --- a/Jsmn/ParseError.hpp +++ b/Jsmn/ParseError.hpp @@ -1,6 +1,7 @@ #ifndef JSMN_PARSEERROR_HPP #define JSMN_PARSEERROR_HPP +#include"Util/BacktraceException.hpp" #include #include #include @@ -8,7 +9,7 @@ namespace Jsmn { /* Thrown on JSON parsing failure. */ -class ParseError : public std::runtime_error { +class ParseError : public Util::BacktraceException { private: std::string input; unsigned int i; @@ -20,7 +21,7 @@ class ParseError : public std::runtime_error { ParseError() =delete; ParseError( std::string input_ , unsigned int i_ - ) : std::runtime_error(enmessage(input_, i_)) + ) : Util::BacktraceException(enmessage(input_, i_)) , input(std::move(input_)) , i(i_) { diff --git a/Ln/Amount.cpp b/Ln/Amount.cpp index c7a32d007..90c841a01 100644 --- a/Ln/Amount.cpp +++ b/Ln/Amount.cpp @@ -1,4 +1,5 @@ #include"Ln/Amount.hpp" +#include"Util/BacktraceException.hpp" #include #include #include @@ -44,12 +45,12 @@ Amount::object(Jsmn::Object const& o) { else if (o.is_string()) return Amount(std::string(o)); else - throw std::invalid_argument("Ln::Amount json object invalid."); + throw Util::BacktraceException("Ln::Amount json object invalid."); } Amount::Amount(std::string const& s) { if (!valid_string(s)) - throw std::invalid_argument("Ln::Amount string invalid."); + throw Util::BacktraceException("Ln::Amount string invalid."); auto is = std::istringstream(std::string(s.begin(), s.end() - 4)); is >> v; } diff --git a/Ln/NodeId.cpp b/Ln/NodeId.cpp index ff559d59d..921ab6325 100644 --- a/Ln/NodeId.cpp +++ b/Ln/NodeId.cpp @@ -19,7 +19,7 @@ namespace Ln { NodeId::NodeId(std::string const& s) { /* Parse. */ if (!valid_string(s)) - throw std::range_error( + throw Util::BacktraceException( std::string("Ln::NodeId: not node ID: ") + s ); diff --git a/Ln/Preimage.cpp b/Ln/Preimage.cpp index 91b01d495..8c4920b91 100644 --- a/Ln/Preimage.cpp +++ b/Ln/Preimage.cpp @@ -29,7 +29,7 @@ bool Preimage::valid_string(std::string const& s) { Preimage::Preimage(std::string const& s) : pimpl(std::make_shared()) { auto buf = Util::Str::hexread(s); if (buf.size() != 32) - throw std::invalid_argument("Ln::Preimage: wrong size."); + throw Util::BacktraceException("Ln::Preimage: wrong size."); for (auto i = 0; i < 32; ++i) pimpl->data[i] = buf[i]; } diff --git a/Ln/Scid.cpp b/Ln/Scid.cpp index a964e2951..78770caff 100644 --- a/Ln/Scid.cpp +++ b/Ln/Scid.cpp @@ -1,4 +1,5 @@ #include"Ln/Scid.hpp" +#include"Util/BacktraceException.hpp" #include #include #include @@ -82,7 +83,7 @@ Scid::Scid(std::string const& s) { auto t = std::uint64_t(); auto o = std::uint64_t(); if (!validate_and_parse(s, b, t, o)) - throw std::invalid_argument(std::string("Not an SCID: ") + s); + throw Util::BacktraceException(std::string("Not an SCID: ") + s); val = (b << 40) | (t << 16) | (o << 0) diff --git a/Net/IPAddr.hpp b/Net/IPAddr.hpp index e8aae8e6f..8e880bf23 100644 --- a/Net/IPAddr.hpp +++ b/Net/IPAddr.hpp @@ -1,6 +1,7 @@ #ifndef NET_IPADDR_HPP #define NET_IPADDR_HPP +#include"Util/BacktraceException.hpp" #include #include #include @@ -17,11 +18,11 @@ namespace Net { * * @brief thrown when an invalid IP address is given. */ -class IPAddrInvalid : public std::invalid_argument { +class IPAddrInvalid : public Util::BacktraceException { public: std::string invalid_ip; explicit IPAddrInvalid( std::string const& invalid_ip_ - ) : std::invalid_argument( std::string("Invalid IP: ") + ) : Util::BacktraceException( std::string("Invalid IP: ") + invalid_ip_ ) , invalid_ip(invalid_ip_) diff --git a/Net/SocketFd.cpp b/Net/SocketFd.cpp index 9ed14f367..65c9d7016 100644 --- a/Net/SocketFd.cpp +++ b/Net/SocketFd.cpp @@ -5,6 +5,7 @@ #include #include #include"Net/SocketFd.hpp" +#include"Util/BacktraceException.hpp" namespace Net { @@ -59,7 +60,7 @@ void SocketFd::write(std::vector const& data) { ); } while (res < 0 && errno == EINTR); if (res < 0) - throw std::runtime_error( std::string("Net::SocketFd::write: write: ") + throw Util::BacktraceException( std::string("Net::SocketFd::write: write: ") + strerror(errno) ); ptr += res; @@ -78,7 +79,7 @@ std::vector SocketFd::read(std::size_t size) { ); } while (res < 0 && errno == EINTR); if (res < 0) - throw std::runtime_error( std::string("Net::SocketFd::read: read: ") + throw Util::BacktraceException( std::string("Net::SocketFd::read: read: ") + strerror(errno) ); if (res == 0) { diff --git a/Ripemd160/Hash.cpp b/Ripemd160/Hash.cpp index d2674b749..7694bdbf0 100644 --- a/Ripemd160/Hash.cpp +++ b/Ripemd160/Hash.cpp @@ -19,7 +19,7 @@ bool Hash::valid_string(std::string const& s) { Hash::Hash(std::string const& s) : pimpl(std::make_shared()) { auto buf = Util::Str::hexread(s); if (buf.size() != 20) - throw std::invalid_argument( + throw Util::BacktraceException( "Ripemd160::Hash: incorrect size." ); from_buffer(&buf[0]); diff --git a/Secp256k1/Detail/context.cpp b/Secp256k1/Detail/context.cpp index 557c97ba3..c541a94fe 100644 --- a/Secp256k1/Detail/context.cpp +++ b/Secp256k1/Detail/context.cpp @@ -2,12 +2,13 @@ #include #include #include"Secp256k1/Detail/context.hpp" +#include"Util/BacktraceException.hpp" namespace { /* Called due to illegal inputs to the library. */ void illegal_callback(const char* msg, void*) { - throw std::invalid_argument(std::string("SECP256K1: ") + msg); + throw Util::BacktraceException(std::string("SECP256K1: ") + msg); } std::shared_ptr create_secp256k1_context() { diff --git a/Secp256k1/PrivKey.cpp b/Secp256k1/PrivKey.cpp index 4165dcdac..9561a46f7 100644 --- a/Secp256k1/PrivKey.cpp +++ b/Secp256k1/PrivKey.cpp @@ -68,7 +68,7 @@ PrivKey& PrivKey::operator+=(PrivKey const& o) { auto res = secp256k1_ec_seckey_tweak_add(context.get(), key, o.key); /* FIXME: Use a backtrace-catching exception. */ if (!res) - throw std::out_of_range("Secp256k1::PrivKey += out-of-range"); + throw Util::BacktraceException("Secp256k1::PrivKey += out-of-range"); return *this; } @@ -76,7 +76,7 @@ PrivKey& PrivKey::operator*=(PrivKey const& o) { auto res = secp256k1_ec_seckey_tweak_mul(context.get(), key, o.key); /* FIXME: Use a backtrace-catching exception. */ if (!res) - throw std::out_of_range("Secp256k1::PrivKey += out-of-range"); + throw Util::BacktraceException("Secp256k1::PrivKey += out-of-range"); return *this; } diff --git a/Secp256k1/PrivKey.hpp b/Secp256k1/PrivKey.hpp index 155e5d138..ea1488b3e 100644 --- a/Secp256k1/PrivKey.hpp +++ b/Secp256k1/PrivKey.hpp @@ -1,6 +1,7 @@ #ifndef SECP256K1_PRIVKEY_HPP #define SECP256K1_PRIVKEY_HPP +#include"Util/BacktraceException.hpp" #include #include #include @@ -19,9 +20,9 @@ namespace Secp256k1 { * private key. */ /* FIXME: derive from a backtrace-capturing exception. */ -class InvalidPrivKey : public std::invalid_argument { +class InvalidPrivKey : public Util::BacktraceException { public: - InvalidPrivKey() : std::invalid_argument("Invalid private key.") {} + InvalidPrivKey() : Util::BacktraceException("Invalid private key.") {} }; class PrivKey { diff --git a/Secp256k1/PubKey.cpp b/Secp256k1/PubKey.cpp index c27dad82b..e60502ed0 100644 --- a/Secp256k1/PubKey.cpp +++ b/Secp256k1/PubKey.cpp @@ -67,7 +67,7 @@ class PubKey::Impl { ); /* FIXME: use a backtrace-prserving exception. */ if (!res) - throw std::out_of_range( + throw Util::BacktraceException( "Secp256k1::PubKey::operatoor+=: " "result of adding PubKey out-of-range" ); @@ -83,7 +83,7 @@ class PubKey::Impl { , sk.key ); if (!res) - throw std::out_of_range( + throw Util::BacktraceException( "Secp256k1::PubKey::operatoor*=: " "result of multiplying PrivKey out-of-range" ); diff --git a/Secp256k1/PubKey.hpp b/Secp256k1/PubKey.hpp index 81057d82a..6b7f025cd 100644 --- a/Secp256k1/PubKey.hpp +++ b/Secp256k1/PubKey.hpp @@ -1,6 +1,7 @@ #ifndef SECP256K1_PUBKEY_HPP #define SECP256K1_PUBKEY_HPP +#include"Util/BacktraceException.hpp" #include #include #include @@ -23,9 +24,9 @@ std::ostream& operator<<(std::ostream&, Secp256k1::PubKey const&); namespace Secp256k1 { /* Thrown in case of being fed an invalid public key. */ -class InvalidPubKey : public std::invalid_argument { +class InvalidPubKey : public Util::BacktraceException { public: - InvalidPubKey() : std::invalid_argument("Invalid public key.") { } + InvalidPubKey() : Util::BacktraceException("Invalid public key.") { } }; class PubKey { diff --git a/Secp256k1/Signature.cpp b/Secp256k1/Signature.cpp index 8a22fb30c..ae2b01cf9 100644 --- a/Secp256k1/Signature.cpp +++ b/Secp256k1/Signature.cpp @@ -46,7 +46,7 @@ Signature::Signature( Secp256k1::PrivKey const& sk /* Extremely unlikely to happen. * TODO: backtrace-capturing. */ - throw std::runtime_error("Nonce generation for signing failed."); + throw Util::BacktraceException("Nonce generation for signing failed."); ++(*reinterpret_cast(extra_entropy)); } while (!sig_has_low_r()); } diff --git a/Secp256k1/Signature.hpp b/Secp256k1/Signature.hpp index 7d5c3ce70..2150ef39a 100644 --- a/Secp256k1/Signature.hpp +++ b/Secp256k1/Signature.hpp @@ -1,6 +1,7 @@ #ifndef SECP256K1_SIGNATURE_HPP #define SECP256K1_SIGNATURE_HPP +#include"Util/BacktraceException.hpp" #include #include #include @@ -12,10 +13,11 @@ namespace Sha256 { class Hash; } namespace Secp256k1 { -class BadSignatureEncoding : public std::invalid_argument { + +class BadSignatureEncoding : public Util::BacktraceException { public: BadSignatureEncoding() - : std::invalid_argument("Bad signature encoding") + : Util::BacktraceException("Bad signature encoding") { } }; diff --git a/Sha256/Hash.cpp b/Sha256/Hash.cpp index 0bf9e67c9..1b1fb3e85 100644 --- a/Sha256/Hash.cpp +++ b/Sha256/Hash.cpp @@ -17,7 +17,7 @@ bool Hash::valid_string(std::string const& s) { Hash::Hash(std::string const& s) { auto bytes = Util::Str::hexread(s); if (bytes.size() != 32) - throw std::invalid_argument("Hashes must be 32 bytes."); + throw Util::BacktraceException("Hashes must be 32 bytes."); pimpl = std::make_shared(); for (auto i = std::size_t(0); i < 32; ++i) pimpl->d[i] = bytes[i]; diff --git a/Sqlite3/Db.cpp b/Sqlite3/Db.cpp index 5da5748ab..7029a2bf5 100644 --- a/Sqlite3/Db.cpp +++ b/Sqlite3/Db.cpp @@ -1,5 +1,6 @@ #include"Ev/Io.hpp" #include"Ev/yield.hpp" +#include"Util/BacktraceException.hpp" #include"Sqlite3/Db.hpp" #include"Sqlite3/Tx.hpp" #include @@ -28,7 +29,7 @@ class Db::Impl { connection = nullptr; } else msg = "Not enough memory"; - throw std::runtime_error( + throw Util::BacktraceException( std::string("Sqlite3::Db: sqlite3_open: ") + msg ); @@ -38,7 +39,7 @@ class Db::Impl { auto msg = std::string(sqlite3_errmsg(connection)); sqlite3_close_v2(connection); connection = nullptr; - throw std::runtime_error( + throw Util::BacktraceException( std::string("Sqlite3::Db: sqlite3_extended_result_codes: ") + msg ); @@ -50,7 +51,7 @@ class Db::Impl { auto msg = std::string(sqlite3_errmsg(connection)); sqlite3_close_v2(connection); connection = nullptr; - throw std::runtime_error( + throw Util::BacktraceException( std::string("Sqlite3::Db: " "PRAGMA foreign_keys = ON: " ) + diff --git a/Sqlite3/Detail/binds.cpp b/Sqlite3/Detail/binds.cpp index 7ccc333bc..c171d17f0 100644 --- a/Sqlite3/Detail/binds.cpp +++ b/Sqlite3/Detail/binds.cpp @@ -1,3 +1,4 @@ +#include"Util/BacktraceException.hpp" #include"Sqlite3/Detail/binds.hpp" #include #include @@ -9,14 +10,14 @@ void bind_d(void* stmt, int l, double v) { , l, v ); if (res != SQLITE_OK) - throw std::runtime_error("Sqlite3: bind error."); + throw Util::BacktraceException("Sqlite3: bind error."); } void bind_i(void* stmt, int l, std::int64_t v) { auto res = sqlite3_bind_int64( (sqlite3_stmt*) stmt , l, v ); if (res != SQLITE_OK) - throw std::runtime_error("Sqlite3: bind error."); + throw Util::BacktraceException("Sqlite3: bind error."); } void bind_s(void* stmt, int l, std::string v) { auto res = sqlite3_bind_text( (sqlite3_stmt*) stmt @@ -24,12 +25,12 @@ void bind_s(void* stmt, int l, std::string v) { , SQLITE_TRANSIENT ); if (res != SQLITE_OK) - throw std::runtime_error("Sqlite3: bind error."); + throw Util::BacktraceException("Sqlite3: bind error."); } void bind_null(void* stmt, int l) { auto res = sqlite3_bind_null( (sqlite3_stmt*) stmt, l); if (res != SQLITE_OK) - throw std::runtime_error("Sqlite3: bind error."); + throw Util::BacktraceException("Sqlite3: bind error."); } }} diff --git a/Sqlite3/Query.cpp b/Sqlite3/Query.cpp index 2982073eb..e32adea9d 100644 --- a/Sqlite3/Query.cpp +++ b/Sqlite3/Query.cpp @@ -1,6 +1,7 @@ #include"Sqlite3/Db.hpp" #include"Sqlite3/Query.hpp" #include"Sqlite3/Result.hpp" +#include"Util/BacktraceException.hpp" #include"Util/make_unique.hpp" #include #include @@ -27,7 +28,7 @@ class Query::Impl { int get_location(char const* field) const { auto res = sqlite3_bind_parameter_index(stmt, field); if (res == 0) - throw std::runtime_error( + throw Util::BacktraceException( std::string("Sqlite3::Query::bind: " "no field: " ) + field diff --git a/Sqlite3/Result.cpp b/Sqlite3/Result.cpp index 472c78290..4a2fde841 100644 --- a/Sqlite3/Result.cpp +++ b/Sqlite3/Result.cpp @@ -1,3 +1,4 @@ +#include"Util/BacktraceException.hpp" #include"Sqlite3/Db.hpp" #include"Sqlite3/Result.hpp" #include @@ -30,7 +31,7 @@ bool Result::advance() { else { auto connection = (sqlite3*) db.get_connection(); auto err = std::string(sqlite3_errmsg(connection)); - throw std::runtime_error( + throw Util::BacktraceException( std::string("Sqlite3::Result: ") + err ); } diff --git a/Sqlite3/Tx.cpp b/Sqlite3/Tx.cpp index 31b2eab8c..1c9fe81e8 100644 --- a/Sqlite3/Tx.cpp +++ b/Sqlite3/Tx.cpp @@ -1,6 +1,7 @@ #include"Sqlite3/Db.hpp" #include"Sqlite3/Query.hpp" #include"Sqlite3/Tx.hpp" +#include"Util/BacktraceException.hpp" #include"Util/make_unique.hpp" #include #include @@ -15,7 +16,7 @@ class Tx::Impl { void throw_sqlite3(char const* src) { auto connection = (sqlite3*) db.get_connection(); auto err = std::string(sqlite3_errmsg(connection)); - throw std::runtime_error( + throw Util::BacktraceException( std::string("Sqlite3::Tx: ") + src + ": " + err ); } diff --git a/Stats/WeightedMedian.hpp b/Stats/WeightedMedian.hpp index d5f38581a..6a0c92f45 100644 --- a/Stats/WeightedMedian.hpp +++ b/Stats/WeightedMedian.hpp @@ -1,6 +1,7 @@ #ifndef STATS_WEIGHTEDMEDIAN_HPP #define STATS_WEIGHTEDMEDIAN_HPP +#include"Util/BacktraceException.hpp" #include #include #include @@ -14,9 +15,9 @@ namespace Stats { * @brief thrown when the weighted-median is * extracted but there are no samples. */ -class NoSamples : public std::invalid_argument { +class NoSamples : public Util::BacktraceException { public: - NoSamples() : std::invalid_argument("Stats::NoSamples") { } + NoSamples() : Util::BacktraceException("Stats::NoSamples") { } }; /** class Stats::WeightedMedian diff --git a/Util/Str.hpp b/Util/Str.hpp index 28fbc32af..eaf543025 100644 --- a/Util/Str.hpp +++ b/Util/Str.hpp @@ -5,6 +5,7 @@ * Minor string utilities. */ +#include"Util/BacktraceException.hpp" #include #include #include @@ -20,10 +21,9 @@ std::string hexbyte(std::uint8_t); std::string hexdump(void const* p, std::size_t s); /* Creates a buffer from the given hex string. */ -/* FIXME: use a backtrace-extracting exception. */ -struct HexParseFailure : public std::runtime_error { +struct HexParseFailure : public Util::BacktraceException { HexParseFailure(std::string msg) - : std::runtime_error("hexread: " + msg) { } + : Util::BacktraceException("hexread: " + msg) { } }; std::vector hexread(std::string const&); diff --git a/Uuid.cpp b/Uuid.cpp index 839aef5b6..2552d39d5 100644 --- a/Uuid.cpp +++ b/Uuid.cpp @@ -39,7 +39,7 @@ Uuid::Uuid(std::string const& s) { pimpl = Util::make_unique(); auto buf = Util::Str::hexread(s); if (buf.size() != 16) - throw std::invalid_argument("Uuid: invalid input string."); + throw Util::BacktraceException("Uuid: invalid input string."); std::copy(buf.begin(), buf.end(), pimpl->data); } bool Uuid::valid_string(std::string const& s) { From 7e4f07093ee8fb6a83854662793c14c7eb249091 Mon Sep 17 00:00:00 2001 From: Ken Sedgwick Date: Fri, 26 Jul 2024 12:15:52 -0700 Subject: [PATCH 03/18] Insert exception what() value in logging messages --- Boss/Mod/ChannelCreateDestroyMonitor.cpp | 10 ++++++---- Boss/Mod/ChannelCreator/Carpenter.cpp | 5 +++-- Boss/Mod/ChannelFinderByEarnedFee.cpp | 5 +++-- Boss/Mod/ChannelFinderByListpays.cpp | 5 +++-- Boss/Mod/FeeModderByBalance.cpp | 8 ++++---- Boss/Mod/FeeModderBySize.cpp | 8 ++++---- Boss/Mod/ForwardFeeMonitor.cpp | 5 +++-- Boss/Mod/FundsMover/Attempter.cpp | 8 ++++---- Boss/Mod/FundsMover/Runner.cpp | 10 ++++++---- Boss/Mod/JitRebalancer.cpp | 5 +++-- Boss/Mod/PaymentDeleter.cpp | 14 +++++++------- Boss/Mod/PeerJudge/DataGatherer.cpp | 8 ++++---- 12 files changed, 50 insertions(+), 41 deletions(-) diff --git a/Boss/Mod/ChannelCreateDestroyMonitor.cpp b/Boss/Mod/ChannelCreateDestroyMonitor.cpp index bce644257..d06210753 100644 --- a/Boss/Mod/ChannelCreateDestroyMonitor.cpp +++ b/Boss/Mod/ChannelCreateDestroyMonitor.cpp @@ -149,12 +149,13 @@ void ChannelCreateDestroyMonitor::start() { try { auto payload = params["channel_opened"]; n = Ln::NodeId(std::string(payload["id"])); - } catch (std::runtime_error const&) { + } catch (std::runtime_error const& err) { return Boss::log( bus, Error , "ChannelCreateDestroyMonitor: " "Unexpected channel_opened " - "payload: %s" + "payload: %s: %s" , Util::stringify(params).c_str() + , err.what() ); } /* Is it already in channeled? */ @@ -174,12 +175,13 @@ void ChannelCreateDestroyMonitor::start() { n = Ln::NodeId(std::string(payload["peer_id"])); old_state = std::string(payload["old_state"]); new_state = std::string(payload["new_state"]); - } catch (std::runtime_error const&) { + } catch (std::runtime_error const& err) { return Boss::log( bus, Error , "ChannelCreateDestroyMonitor: " "Unexpected channel_state_changed " - "payload: %s" + "payload: %s: %s" , Util::stringify(params).c_str() + , err.what() ); } /* Only continue if we are leaving the CHANNELD_NORMAL diff --git a/Boss/Mod/ChannelCreator/Carpenter.cpp b/Boss/Mod/ChannelCreator/Carpenter.cpp index d430b6fcd..083978b43 100644 --- a/Boss/Mod/ChannelCreator/Carpenter.cpp +++ b/Boss/Mod/ChannelCreator/Carpenter.cpp @@ -157,14 +157,15 @@ Carpenter::construct(std::map plan) { report << ", "; report << node; } - } catch (std::invalid_argument const&) { + } catch (std::invalid_argument const& ex) { auto os = std::ostringstream(); os << res; return Boss::log( bus, Error , "ChannelCreator::Carpenter: " "Unexpected result from " - "multifundchannel: %s" + "multifundchannel: %s: %s" , os.str().c_str() + , ex.what() ); } diff --git a/Boss/Mod/ChannelFinderByEarnedFee.cpp b/Boss/Mod/ChannelFinderByEarnedFee.cpp index b3ee311fa..cf06ee774 100644 --- a/Boss/Mod/ChannelFinderByEarnedFee.cpp +++ b/Boss/Mod/ChannelFinderByEarnedFee.cpp @@ -131,12 +131,13 @@ class ChannelFinderByEarnedFee::Impl { continue; props.emplace(std::move(dest)); } - } catch (std::exception const&) { + } catch (std::exception const& ex) { return Boss::log( bus, Error , "ChannelFinderByEarnedFees: " "Unexpected result from " - "`listchannels`: %s" + "`listchannels`: %s: %s" , Util::stringify(res).c_str() + , ex.what() ); } res = Jsmn::Object(); diff --git a/Boss/Mod/ChannelFinderByListpays.cpp b/Boss/Mod/ChannelFinderByListpays.cpp index c3dd950b9..cad88b166 100644 --- a/Boss/Mod/ChannelFinderByListpays.cpp +++ b/Boss/Mod/ChannelFinderByListpays.cpp @@ -159,12 +159,13 @@ Ev::Io ChannelFinderByListpays::extract_payees_loop() { ++pit->second; ++count; return extract_payees_loop(); - } catch (std::exception const&) { + } catch (std::exception const& ex) { return Boss::log( bus, Error , "ChannelFinderByListpays: " "Unexpected result from `listpays` " - "`pays` field: %s" + "`pays` field: %s: %s" , Util::stringify(pay).c_str() + , ex.what() ); } }); diff --git a/Boss/Mod/FeeModderByBalance.cpp b/Boss/Mod/FeeModderByBalance.cpp index 79c292fd6..ec33c92e7 100644 --- a/Boss/Mod/FeeModderByBalance.cpp +++ b/Boss/Mod/FeeModderByBalance.cpp @@ -186,14 +186,14 @@ class FeeModderByBalance::Impl { ); break; } - } catch (std::exception const&) { + } catch (std::exception const& ex) { found = false; act = Boss::log( bus, Error , "FeeModderByBalance: " "Unexpected result from " - "listpeerchannels: %s" - , Util::stringify(res) - .c_str() + "listpeerchannels: %s: %s" + , Util::stringify(res).c_str() + , ex.what() ); } typedef ChannelSpecs CS; diff --git a/Boss/Mod/FeeModderBySize.cpp b/Boss/Mod/FeeModderBySize.cpp index 930863579..8fc133c2d 100644 --- a/Boss/Mod/FeeModderBySize.cpp +++ b/Boss/Mod/FeeModderBySize.cpp @@ -384,14 +384,14 @@ class FeeModderBySize::Impl { continue; rv.insert(n); } - } catch (std::exception const&) { + } catch (std::exception const& ex) { return Boss::log( bus, Error , "FeeModderBySize: " "get_competitors: " "Unexpected result from " - "listchannels: %s" - , Util::stringify(res) - .c_str() + "listchannels: %s: %s" + , Util::stringify(res).c_str() + , ex.what() ).then([rv]() { return Ev::lift(rv); }); diff --git a/Boss/Mod/ForwardFeeMonitor.cpp b/Boss/Mod/ForwardFeeMonitor.cpp index ddc815711..d3cb0b6cc 100644 --- a/Boss/Mod/ForwardFeeMonitor.cpp +++ b/Boss/Mod/ForwardFeeMonitor.cpp @@ -50,11 +50,12 @@ void ForwardFeeMonitor::start() { - double(payload["received_time"]) ; - } catch (std::runtime_error const& _) { + } catch (std::runtime_error const& err) { return Boss::log( bus, Error , "ForwardFeeMonitor: Unexpected " - "forward_event payload: %s" + "forward_event payload: %s: %s" , Util::stringify(n.params).c_str() + , err.what() ); } diff --git a/Boss/Mod/FundsMover/Attempter.cpp b/Boss/Mod/FundsMover/Attempter.cpp index 1849d033f..66223bbf3 100644 --- a/Boss/Mod/FundsMover/Attempter.cpp +++ b/Boss/Mod/FundsMover/Attempter.cpp @@ -348,15 +348,15 @@ class Attempter::Impl : public std::enable_shared_from_this { data["failcode"] )); } - } catch (std::exception const&) { + } catch (std::exception const& ex) { return std::move(act) + Boss::log( bus, Error , "FundsMover: Attempt: " "Unexpected error from " - "%s: %s" + "%s: %s: %s" , e.command.c_str() - , Util::stringify(e.error) - .c_str() + , Util::stringify(e.error).c_str() + , ex.what() ); } diff --git a/Boss/Mod/FundsMover/Runner.cpp b/Boss/Mod/FundsMover/Runner.cpp index 967c816bf..645b8d7f0 100644 --- a/Boss/Mod/FundsMover/Runner.cpp +++ b/Boss/Mod/FundsMover/Runner.cpp @@ -94,11 +94,12 @@ Ev::Io Runner::gather_info() { )); break; } - } catch (std::exception const&) { + } catch (std::exception const& ex) { return Boss::log( bus, Error , "FundsMover: Unexpected result " - "from listchannels: %s" + "from listchannels: %s: %s" , Util::stringify(res).c_str() + , ex.what() ); } return Ev::lift(); @@ -134,11 +135,12 @@ Ev::Io Runner::gather_info() { )); break; } - } catch (std::exception const&) { + } catch (std::exception const& ex) { return Boss::log( bus, Error , "FundsMover: Unexpected result " - "from listchannels: %s" + "from listchannels: %s: %s" , Util::stringify(res).c_str() + , ex.what() ); } return Ev::lift(); diff --git a/Boss/Mod/JitRebalancer.cpp b/Boss/Mod/JitRebalancer.cpp index 8735064a0..d50ea34a3 100644 --- a/Boss/Mod/JitRebalancer.cpp +++ b/Boss/Mod/JitRebalancer.cpp @@ -337,11 +337,12 @@ class JitRebalancer::Impl::Run::Impl { av.to_us += to_us; av.capacity += capacity; } - } catch (std::exception const&) { + } catch (std::exception const& ex) { return Boss::log( bus, Error , "JitRebalancer: Unexpected " - "result from listpeerchannels: %s" + "result from listpeerchannels: %s: %s" , Util::stringify(res).c_str() + , ex.what() ).then([]() { throw Continue(); return Ev::lift(); diff --git a/Boss/Mod/PaymentDeleter.cpp b/Boss/Mod/PaymentDeleter.cpp index 3a65d701a..a73c43366 100644 --- a/Boss/Mod/PaymentDeleter.cpp +++ b/Boss/Mod/PaymentDeleter.cpp @@ -108,12 +108,12 @@ class PaymentDeleter::Impl { try { pays = res["pays"]; it = pays.begin(); - } catch (std::exception const&) { + } catch (std::exception const& ex) { return Boss::log( bus, Error , "PaymentDeleter: Unexpected " - "result from 'listpays': " - "%s" + "result from 'listpays': %s: %s" , Util::stringify(res).c_str() + , ex.what() ); } return loop(); @@ -156,13 +156,13 @@ class PaymentDeleter::Impl { + delpay(payment_hash, status) + loop() ; - } catch (std::exception const&) { + } catch (std::exception const& ex) { return Boss::log( bus, Error , "PaymentDeleter: " "Unexpected 'pays' entry " - "from 'listpays': %s" - , Util::stringify(pay) - .c_str() + "from 'listpays': %s: %s" + , Util::stringify(pay).c_str() + , ex.what() ); } }); diff --git a/Boss/Mod/PeerJudge/DataGatherer.cpp b/Boss/Mod/PeerJudge/DataGatherer.cpp index 7746248b7..d9e6c11ae 100644 --- a/Boss/Mod/PeerJudge/DataGatherer.cpp +++ b/Boss/Mod/PeerJudge/DataGatherer.cpp @@ -90,13 +90,13 @@ class DataGatherer::Impl { id, total }); } - } catch (std::exception const& e) { + } catch (std::exception const& ex) { infos->clear(); return Boss::log( bus, Error , "PeerJudge: Unexpected " - "listpeers result: %s" - , Util::stringify(peers) - .c_str() + "listpeers result: %s: %s" + , Util::stringify(peers).c_str() + , ex.what() ); } return Ev::lift(); From 060a3f9544f4476d189b2ce3a8636f8d6741b6a9 Mon Sep 17 00:00:00 2001 From: Ken Sedgwick Date: Mon, 5 Aug 2024 19:11:03 -0700 Subject: [PATCH 04/18] factor parse_json into Jsmn::Object static method This makes it easier to generate test cases by using literal JSON. --- Jsmn/Object.cpp | 7 +++++++ Jsmn/Object.hpp | 2 ++ tests/boss/test_forwardfeemonitor.cpp | 13 +++---------- tests/boss/test_jitrebalancer.cpp | 11 ++--------- 4 files changed, 14 insertions(+), 19 deletions(-) diff --git a/Jsmn/Object.cpp b/Jsmn/Object.cpp index f520745b0..ab98a27a8 100644 --- a/Jsmn/Object.cpp +++ b/Jsmn/Object.cpp @@ -302,6 +302,13 @@ Detail::Iterator Object::end() const { return pimpl->end(); } +Object Object::parse_json(char const* txt) { + auto is = std::istringstream(std::string(txt)); + auto js = Jsmn::Object(); + is >> js; + return js; +} + /* Implements indented printing. */ namespace { diff --git a/Jsmn/Object.hpp b/Jsmn/Object.hpp index db7dacb9e..081ba887f 100644 --- a/Jsmn/Object.hpp +++ b/Jsmn/Object.hpp @@ -42,6 +42,8 @@ class Object { Object& operator=(Object const&) =default; Object& operator=(Object&&) =default; + static Object parse_json(char const* txt); + /* Type queries on the object. */ bool is_null() const; bool is_boolean() const; diff --git a/tests/boss/test_forwardfeemonitor.cpp b/tests/boss/test_forwardfeemonitor.cpp index 85a3427cb..094e365ca 100644 --- a/tests/boss/test_forwardfeemonitor.cpp +++ b/tests/boss/test_forwardfeemonitor.cpp @@ -52,13 +52,6 @@ auto const listpeers_result = R"JSON( } )JSON"; -Jsmn::Object parse_json(char const* txt) { - auto is = std::istringstream(std::string(txt)); - auto js = Jsmn::Object(); - is >> js; - return js; -} - } int main() { @@ -103,14 +96,14 @@ int main() { /* Give the peers. */ return bus.raise(Boss::Msg::ListpeersResult{ - Boss::Mod::convert_legacy_listpeers(parse_json(listpeers_result)["peers"]), true + Boss::Mod::convert_legacy_listpeers(Jsmn::Object::parse_json(listpeers_result)["peers"]), true }); }).then([&]() { /* Should ignore non-forward_event. */ forwardfee = nullptr; return bus.raise(Boss::Msg::Notification{ - "not-forward_event", parse_json("{}") + "not-forward_event", Jsmn::Object::parse_json("{}") }); }).then([&]() { return Ev::yield(42); @@ -121,7 +114,7 @@ int main() { forwardfee = nullptr; return bus.raise(Boss::Msg::Notification{ "forward_event", - parse_json(R"JSON( + Jsmn::Object::parse_json(R"JSON( { "forward_event": { "payment_hash": "f5a6a059a25d1e329d9b094aeeec8c2191ca037d3f5b0662e21ae850debe8ea2", diff --git a/tests/boss/test_jitrebalancer.cpp b/tests/boss/test_jitrebalancer.cpp index aeca10691..baaa3c988 100644 --- a/tests/boss/test_jitrebalancer.cpp +++ b/tests/boss/test_jitrebalancer.cpp @@ -169,13 +169,6 @@ class DummyEarningsManager { ) : bus(bus_) { start(); } }; -Jsmn::Object parse_json(char const* txt) { - auto is = std::istringstream(txt); - auto rv = Jsmn::Object(); - is >> rv; - return rv; -} - class DummyRpc { private: S::Bus& bus; @@ -183,7 +176,7 @@ class DummyRpc { Ev::Io respond(void* requester, char const* res) { return bus.raise(Boss::Msg::ResponseRpcCommand{ requester, true, - parse_json(res), + Jsmn::Object::parse_json(res), "" }); } @@ -341,7 +334,7 @@ int main() { assert(deferrer); /* Raise the ListpeersResult. */ - auto res = parse_json(listpeers_result); + auto res = Jsmn::Object::parse_json(listpeers_result); auto peers = res["peers"]; return bus.raise(Boss::Msg::ListpeersResult{ std::move(Boss::Mod::convert_legacy_listpeers(peers)), true From 976f5619a6bbbb2f29f98bbecf1e22bb594f439c Mon Sep 17 00:00:00 2001 From: Ken Sedgwick Date: Mon, 5 Aug 2024 19:51:16 -0700 Subject: [PATCH 05/18] add Jsmn::Object::operator== This makes it easier to write test cases by comparing to expected values --- Jsmn/Object.cpp | 46 ++++++++++++++++++++ Jsmn/Object.hpp | 5 +++ Makefile.am | 1 + tests/jsmn/test_equality.cpp | 81 ++++++++++++++++++++++++++++++++++++ 4 files changed, 133 insertions(+) create mode 100644 tests/jsmn/test_equality.cpp diff --git a/Jsmn/Object.cpp b/Jsmn/Object.cpp index ab98a27a8..5fa16734a 100644 --- a/Jsmn/Object.cpp +++ b/Jsmn/Object.cpp @@ -309,6 +309,52 @@ Object Object::parse_json(char const* txt) { return js; } +bool Object::operator==(Object const& other) const { + if (this == &other) { + return true; + } + + // Compare types first + if (is_null() != other.is_null() || is_boolean() != other.is_boolean() || + is_string() != other.is_string() || is_object() != other.is_object() || + is_array() != other.is_array() || is_number() != other.is_number()) { + return false; + } + + // Compare values based on type + if (is_null()) { + return true; // Both are null + } else if (is_boolean()) { + return static_cast(*this) == static_cast(other); + } else if (is_string()) { + return static_cast(*this) == static_cast(other); + } else if (is_number()) { + return static_cast(*this) == static_cast(other); + } else if (is_object()) { + if (size() != other.size()) { + return false; + } + for (const auto& key : keys()) { + if (!other.has(key) || (*this)[key] != other[key]) { + return false; + } + } + return true; + } else if (is_array()) { + if (size() != other.size()) { + return false; + } + for (std::size_t i = 0; i < size(); ++i) { + if ((*this)[i] != other[i]) { + return false; + } + } + return true; + } + + return false; // Fallback, should not reach here +} + /* Implements indented printing. */ namespace { diff --git a/Jsmn/Object.hpp b/Jsmn/Object.hpp index 081ba887f..63e51218b 100644 --- a/Jsmn/Object.hpp +++ b/Jsmn/Object.hpp @@ -71,6 +71,11 @@ class Object { Object operator[](std::size_t) const; /* Return null if out-of-range. */ /* TODO: Iterator. */ + bool operator==(Object const& other) const; + bool operator!=(Object const& other) const { + return !(*this == other); + } + /* Formal factory. */ friend class ParserExposedBuffer; /* Iterator type. */ diff --git a/Makefile.am b/Makefile.am index 750638325..afbcddce4 100644 --- a/Makefile.am +++ b/Makefile.am @@ -622,6 +622,7 @@ TESTS = \ tests/ev/test_semaphore \ tests/ev/test_throw_in_then \ tests/graph/test_dijkstra \ + tests/jsmn/test_equality \ tests/jsmn/test_iterator \ tests/jsmn/test_parser \ tests/jsmn/test_performance \ diff --git a/tests/jsmn/test_equality.cpp b/tests/jsmn/test_equality.cpp new file mode 100644 index 000000000..02bc4bcd0 --- /dev/null +++ b/tests/jsmn/test_equality.cpp @@ -0,0 +1,81 @@ +#undef NDEBUG +#include "Jsmn/Object.hpp" +#include +#include + +void test_null_equality() { + auto json_null1 = Jsmn::Object::parse_json(R"JSON({ "x": null })JSON"); + auto json_null2 = Jsmn::Object::parse_json(R"JSON({ "x": null })JSON"); + assert(json_null1 == json_null2); + + auto json_non_null = Jsmn::Object::parse_json(R"JSON({ "x": 1 })JSON"); + assert(json_null1 != json_non_null); +} + +void test_boolean_equality() { + auto json_true1 = Jsmn::Object::parse_json(R"JSON({ "x": true })JSON"); + auto json_true2 = Jsmn::Object::parse_json(R"JSON({ "x": true })JSON"); + assert(json_true1 == json_true2); + + auto json_false1 = Jsmn::Object::parse_json(R"JSON({ "x": false })JSON"); + auto json_false2 = Jsmn::Object::parse_json(R"JSON({ "x": false })JSON"); + assert(json_false1 == json_false2); + + assert(json_true1 != json_false1); +} + +void test_string_equality() { + auto json_str1 = Jsmn::Object::parse_json(R"JSON({ "x": "hello" })JSON"); + auto json_str2 = Jsmn::Object::parse_json(R"JSON({ "x": "hello" })JSON"); + assert(json_str1 == json_str2); + + auto json_str3 = Jsmn::Object::parse_json(R"JSON({ "x": "world" })JSON"); + assert(json_str1 != json_str3); +} + +void test_number_equality() { + auto json_num1 = Jsmn::Object::parse_json(R"JSON({ "x": 42 })JSON"); + auto json_num2 = Jsmn::Object::parse_json(R"JSON({ "x": 42 })JSON"); + assert(json_num1 == json_num2); + + auto json_num3 = Jsmn::Object::parse_json(R"JSON({ "x": 3.14 })JSON"); + assert(json_num1 != json_num3); +} + +void test_object_equality() { + auto json_obj1 = Jsmn::Object::parse_json(R"JSON({ "a": 1, "b": 2 })JSON"); + auto json_obj2 = Jsmn::Object::parse_json(R"JSON({ "b": 2, "a": 1 })JSON"); + assert(json_obj1 == json_obj2); + + auto json_obj3 = Jsmn::Object::parse_json(R"JSON({ "a": 1, "b": 3 })JSON"); + assert(json_obj1 != json_obj3); +} + +void test_array_equality() { + auto json_arr1 = Jsmn::Object::parse_json(R"JSON([1, 2, 3])JSON"); + auto json_arr2 = Jsmn::Object::parse_json(R"JSON([1, 2, 3])JSON"); + assert(json_arr1 == json_arr2); + + auto json_arr3 = Jsmn::Object::parse_json(R"JSON([3, 2, 1])JSON"); + assert(json_arr1 != json_arr3); +} + +void test_nested_structure_equality() { + auto json_nested1 = Jsmn::Object::parse_json(R"JSON({ "x": [ { "a": 1 }, { "b": 2 } ] })JSON"); + auto json_nested2 = Jsmn::Object::parse_json(R"JSON({ "x": [ { "a": 1 }, { "b": 2 } ] })JSON"); + assert(json_nested1 == json_nested2); + + auto json_nested3 = Jsmn::Object::parse_json(R"JSON({ "x": [ { "b": 2 }, { "a": 1 } ] })JSON"); + assert(json_nested1 != json_nested3); +} + +int main() { + test_null_equality(); + test_boolean_equality(); + test_string_equality(); + test_number_equality(); + test_object_equality(); + test_array_equality(); + test_nested_structure_equality(); + return 0; +} From 337668a310fbb8ad770e41f5c2c3f8832b2fe4ae Mon Sep 17 00:00:00 2001 From: Ken Sedgwick Date: Mon, 5 Aug 2024 19:15:32 -0700 Subject: [PATCH 06/18] add test_earningstracker for legacy tracking --- Makefile.am | 1 + tests/boss/test_earningstracker.cpp | 202 ++++++++++++++++++++++++++++ 2 files changed, 203 insertions(+) create mode 100644 tests/boss/test_earningstracker.cpp diff --git a/Makefile.am b/Makefile.am index afbcddce4..bd9fc9683 100644 --- a/Makefile.am +++ b/Makefile.am @@ -591,6 +591,7 @@ TESTS = \ tests/boss/test_channelcreator_rearrangerbysize \ tests/boss/test_channelcreator_reprioritizer \ tests/boss/test_earningsrebalancer \ + tests/boss/test_earningstracker \ tests/boss/test_feemodderbypricetheory \ tests/boss/test_forwardfeemonitor \ tests/boss/test_getmanifest \ diff --git a/tests/boss/test_earningstracker.cpp b/tests/boss/test_earningstracker.cpp new file mode 100644 index 000000000..929ae4418 --- /dev/null +++ b/tests/boss/test_earningstracker.cpp @@ -0,0 +1,202 @@ +#undef NDEBUG + +#include"Boss/Mod/EarningsTracker.hpp" +#include"Boss/Msg/DbResource.hpp" +#include"Boss/Msg/ForwardFee.hpp" +#include"Boss/Msg/ProvideStatus.hpp" +#include"Boss/Msg/RequestEarningsInfo.hpp" +#include"Boss/Msg/RequestMoveFunds.hpp" +#include"Boss/Msg/ResponseEarningsInfo.hpp" +#include"Boss/Msg/ResponseMoveFunds.hpp" +#include"Boss/Msg/SolicitStatus.hpp" +#include"Ev/start.hpp" +#include"Jsmn/Object.hpp" +#include"Jsmn/Parser.hpp" +#include"Json/Out.hpp" +#include"Ln/NodeId.hpp" +#include"S/Bus.hpp" +#include"Sqlite3.hpp" + +#include +#include +#include + +namespace { +auto const A = Ln::NodeId("020000000000000000000000000000000000000000000000000000000000000001"); +auto const B = Ln::NodeId("020000000000000000000000000000000000000000000000000000000000000002"); +auto const C = Ln::NodeId("020000000000000000000000000000000000000000000000000000000000000003"); +} + +int main() { + auto bus = S::Bus(); + + /* Module under test */ + Boss::Mod::EarningsTracker mut(bus); + + auto db = Sqlite3::Db(":memory:"); + + Boss::Msg::ProvideStatus lastStatus; + Boss::Msg::ResponseEarningsInfo lastEarningsInfo; + + bus.subscribe([&](Boss::Msg::ProvideStatus const& st) { + lastStatus = st; + return Ev::lift(); + }); + + bus.subscribe([&](Boss::Msg::ResponseEarningsInfo const& ei) { + lastEarningsInfo = ei; + return Ev::lift(); + }); + + auto code = Ev::lift().then([&]() { + return bus.raise(Boss::Msg::DbResource{ db }); + }).then([&]() { + return bus.raise( + Boss::Msg::ForwardFee{ + A, // in_id + B, // out_id + Ln::Amount::sat(1), // fee + 1.0 // resolution_time + }); + }).then([&]() { + return bus.raise(Boss::Msg::SolicitStatus{}); + }).then([&]() { + // std::cerr << lastStatus.value.output() << std::endl; + assert(lastStatus.key == "offchain_earnings_tracker"); + assert( + Jsmn::Object::parse_json(lastStatus.value.output().c_str()) == + Jsmn::Object::parse_json(R"JSON( + { + "020000000000000000000000000000000000000000000000000000000000000001": { + "in_earnings": 1000, + "in_expenditures": 0, + "out_earnings": 0, + "out_expenditures": 0 + }, + "020000000000000000000000000000000000000000000000000000000000000002": { + "in_earnings": 0, + "in_expenditures": 0, + "out_earnings": 1000, + "out_expenditures": 0 + }, + "total": { + "in_earnings": 1000, + "in_expenditures": 0, + "out_earnings": 1000, + "out_expenditures": 0 + } + } + )JSON")); + return Ev::lift(); + }).then([&]() { + return bus.raise( + Boss::Msg::ForwardFee{ + A, // in_id + B, // out_id + Ln::Amount::sat(1), // fee + 1.0 // resolution_time + }); + }).then([&]() { + return bus.raise(Boss::Msg::SolicitStatus{}); + }).then([&]() { + // std::cerr << lastStatus.value.output() << std::endl; + assert(lastStatus.key == "offchain_earnings_tracker"); + assert( + Jsmn::Object::parse_json(lastStatus.value.output().c_str()) == + Jsmn::Object::parse_json(R"JSON( + { + "020000000000000000000000000000000000000000000000000000000000000001": { + "in_earnings": 2000, + "in_expenditures": 0, + "out_earnings": 0, + "out_expenditures": 0 + }, + "020000000000000000000000000000000000000000000000000000000000000002": { + "in_earnings": 0, + "in_expenditures": 0, + "out_earnings": 2000, + "out_expenditures": 0 + }, + "total": { + "in_earnings": 2000, + "in_expenditures": 0, + "out_earnings": 2000, + "out_expenditures": 0 + } + } + )JSON")); + return Ev::lift(); + }).then([&]() { + return bus.raise( + Boss::Msg::RequestMoveFunds{ + NULL, // requester (match ResponseMoveFunds) + C, // source + A, // destination + Ln::Amount::sat(1000), // amount + Ln::Amount::sat(3) // fee_budget + }); + }).then([&]() { + return bus.raise( + Boss::Msg::ResponseMoveFunds{ + NULL, // requester (match RequestMoveFunds) + Ln::Amount::sat(1000), // amount_moved + Ln::Amount::sat(2) // fee_spent + }); + }).then([&]() { + return bus.raise(Boss::Msg::SolicitStatus{}); + }).then([&]() { + // std::cerr << lastStatus.value.output() << std::endl; + assert(lastStatus.key == "offchain_earnings_tracker"); + assert( + Jsmn::Object::parse_json(lastStatus.value.output().c_str()) == + Jsmn::Object::parse_json(R"JSON( + { + "020000000000000000000000000000000000000000000000000000000000000001": { + "in_earnings": 2000, + "in_expenditures": 0, + "out_earnings": 0, + "out_expenditures": 2000 + }, + "020000000000000000000000000000000000000000000000000000000000000002": { + "in_earnings": 0, + "in_expenditures": 0, + "out_earnings": 2000, + "out_expenditures": 0 + }, + "020000000000000000000000000000000000000000000000000000000000000003": { + "in_earnings": 0, + "in_expenditures": 2000, + "out_earnings": 0, + "out_expenditures": 0 + }, + "total": { + "in_earnings": 2000, + "in_expenditures": 2000, + "out_earnings": 2000, + "out_expenditures": 2000 + } + } + )JSON")); + return Ev::lift(); + }).then([&]() { + mock_now = 4000.0; + return bus.raise( + Boss::Msg::RequestEarningsInfo{ + NULL, // requester (match ResponseEarningsInfo) + A // node + }); + }).then([&]() { + return Ev::yield(42); + }).then([&]() { + assert(lastEarningsInfo.node == A); + assert(lastEarningsInfo.in_earnings == Ln::Amount::msat(2000)); + assert(lastEarningsInfo.in_expenditures == Ln::Amount::msat(0)); + assert(lastEarningsInfo.out_earnings == Ln::Amount::msat(0)); + assert(lastEarningsInfo.out_expenditures == Ln::Amount::msat(2000)); + return Ev::lift(); + }).then([&]() { + return Ev::lift(0); + }); + + return Ev::start(std::move(code)); +} From 6ea700908d30b85c0d4567e6e21b16867da2133e Mon Sep 17 00:00:00 2001 From: Ken Sedgwick Date: Tue, 6 Aug 2024 12:11:18 -0700 Subject: [PATCH 07/18] add get_now() and mock_get_now() to EarningsTracker and test_earningstracker A time source is needed for upcoming time buckets change. --- Boss/Mod/EarningsTracker.cpp | 7 ++++--- Boss/Mod/EarningsTracker.hpp | 4 +++- tests/boss/test_earningstracker.cpp | 12 +++++++++++- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/Boss/Mod/EarningsTracker.cpp b/Boss/Mod/EarningsTracker.cpp index a141e0094..a3caea4de 100644 --- a/Boss/Mod/EarningsTracker.cpp +++ b/Boss/Mod/EarningsTracker.cpp @@ -19,6 +19,7 @@ namespace Boss { namespace Mod { class EarningsTracker::Impl { private: S::Bus& bus; + std::function get_now; Sqlite3::Db db; /* Information of a pending MoveFunds. */ @@ -287,13 +288,13 @@ class EarningsTracker::Impl { Impl(Impl const&) =delete; explicit - Impl(S::Bus& bus_) : bus(bus_) { start(); } + Impl(S::Bus& bus_, std::function get_now_) : bus(bus_), get_now(std::move(get_now_)) { start(); } }; EarningsTracker::EarningsTracker(EarningsTracker&&) =default; EarningsTracker::~EarningsTracker() =default; -EarningsTracker::EarningsTracker(S::Bus& bus) - : pimpl(Util::make_unique(bus)) { } + EarningsTracker::EarningsTracker(S::Bus& bus, std::function get_now_ ) + : pimpl(Util::make_unique(bus, get_now_)) { } }} diff --git a/Boss/Mod/EarningsTracker.hpp b/Boss/Mod/EarningsTracker.hpp index 6c059329b..d8829687c 100644 --- a/Boss/Mod/EarningsTracker.hpp +++ b/Boss/Mod/EarningsTracker.hpp @@ -1,6 +1,8 @@ #ifndef BOSS_MOD_EARNINGSTRACKER_HPP #define BOSS_MOD_EARNINGSTRACKER_HPP +#include"Ev/now.hpp" +#include #include namespace S { class Bus; } @@ -28,7 +30,7 @@ class EarningsTracker { ~EarningsTracker(); explicit - EarningsTracker(S::Bus& bus); + EarningsTracker(S::Bus& bus, std::function get_now_ = &Ev::now); }; }} diff --git a/tests/boss/test_earningstracker.cpp b/tests/boss/test_earningstracker.cpp index 929ae4418..3ace7d9c8 100644 --- a/tests/boss/test_earningstracker.cpp +++ b/tests/boss/test_earningstracker.cpp @@ -27,11 +27,16 @@ auto const B = Ln::NodeId("02000000000000000000000000000000000000000000000000000 auto const C = Ln::NodeId("020000000000000000000000000000000000000000000000000000000000000003"); } +double mock_now = 0.0; +double mock_get_now() { + return mock_now; +} + int main() { auto bus = S::Bus(); /* Module under test */ - Boss::Mod::EarningsTracker mut(bus); + Boss::Mod::EarningsTracker mut(bus, &mock_get_now); auto db = Sqlite3::Db(":memory:"); @@ -51,6 +56,7 @@ int main() { auto code = Ev::lift().then([&]() { return bus.raise(Boss::Msg::DbResource{ db }); }).then([&]() { + mock_now = 1000.0; return bus.raise( Boss::Msg::ForwardFee{ A, // in_id @@ -59,6 +65,7 @@ int main() { 1.0 // resolution_time }); }).then([&]() { + mock_now = 2000.0; return bus.raise(Boss::Msg::SolicitStatus{}); }).then([&]() { // std::cerr << lastStatus.value.output() << std::endl; @@ -89,6 +96,7 @@ int main() { )JSON")); return Ev::lift(); }).then([&]() { + mock_now = 3000.0; return bus.raise( Boss::Msg::ForwardFee{ A, // in_id @@ -127,6 +135,7 @@ int main() { )JSON")); return Ev::lift(); }).then([&]() { + mock_now = 4000.0; return bus.raise( Boss::Msg::RequestMoveFunds{ NULL, // requester (match ResponseMoveFunds) @@ -136,6 +145,7 @@ int main() { Ln::Amount::sat(3) // fee_budget }); }).then([&]() { + mock_now = 5000.0; return bus.raise( Boss::Msg::ResponseMoveFunds{ NULL, // requester (match RequestMoveFunds) From b0691155aac2d8ebffebf704221e6d35e3ece1ba Mon Sep 17 00:00:00 2001 From: Ken Sedgwick Date: Tue, 6 Aug 2024 12:43:14 -0700 Subject: [PATCH 08/18] add EarningsTracker::bucket_time quantizer and unit tests --- Boss/Mod/EarningsTracker.cpp | 10 +++++++++- Boss/Mod/EarningsTracker.hpp | 3 +++ tests/boss/test_earningstracker.cpp | 6 ++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Boss/Mod/EarningsTracker.cpp b/Boss/Mod/EarningsTracker.cpp index a3caea4de..a1fa99d47 100644 --- a/Boss/Mod/EarningsTracker.cpp +++ b/Boss/Mod/EarningsTracker.cpp @@ -12,6 +12,8 @@ #include"S/Bus.hpp" #include"Sqlite3.hpp" #include"Util/make_unique.hpp" + +#include #include namespace Boss { namespace Mod { @@ -294,7 +296,13 @@ class EarningsTracker::Impl { EarningsTracker::EarningsTracker(EarningsTracker&&) =default; EarningsTracker::~EarningsTracker() =default; - EarningsTracker::EarningsTracker(S::Bus& bus, std::function get_now_ ) +EarningsTracker::EarningsTracker(S::Bus& bus, std::function get_now_ ) : pimpl(Util::make_unique(bus, get_now_)) { } +double EarningsTracker::bucket_time(double input_time) { + constexpr double seconds_per_day = 24 * 60 * 60; + double bucket_start = std::floor(input_time / seconds_per_day) * seconds_per_day; + return bucket_start; +} + }} diff --git a/Boss/Mod/EarningsTracker.hpp b/Boss/Mod/EarningsTracker.hpp index d8829687c..9f47e2ea1 100644 --- a/Boss/Mod/EarningsTracker.hpp +++ b/Boss/Mod/EarningsTracker.hpp @@ -31,6 +31,9 @@ class EarningsTracker { explicit EarningsTracker(S::Bus& bus, std::function get_now_ = &Ev::now); + + // exposed for unit testing, otherwise internal + static double bucket_time(double input_time); }; }} diff --git a/tests/boss/test_earningstracker.cpp b/tests/boss/test_earningstracker.cpp index 3ace7d9c8..fe6cb0270 100644 --- a/tests/boss/test_earningstracker.cpp +++ b/tests/boss/test_earningstracker.cpp @@ -33,6 +33,12 @@ double mock_get_now() { } int main() { + // check the UTC quantitization boundaries + assert(Boss::Mod::EarningsTracker::bucket_time(1722902400 - 1) == 1722816000); + assert(Boss::Mod::EarningsTracker::bucket_time(1722902400) == 1722902400); + assert(Boss::Mod::EarningsTracker::bucket_time(1722902400 + 86400 - 1) == 1722902400); + assert(Boss::Mod::EarningsTracker::bucket_time(1722902400 + 86400) == 1722988800); + auto bus = S::Bus(); /* Module under test */ From 84f2faea218fdc691fdf7e32a288c33941b290aa Mon Sep 17 00:00:00 2001 From: Ken Sedgwick Date: Tue, 6 Aug 2024 16:19:04 -0700 Subject: [PATCH 09/18] Upgrade EarningsTracker to time bucket schema, use old semantics This commit modifies the schema of EarningsTracker to allow storing and accessing earning and expenditure data in specific time ranges. All existing strategies and reports still use all data from all time so this PR should not change any balancing behavior. After we've run w/ this for a while we'll have time-based data collected and can evaluate how to improve the strategies. --- Boss/Mod/EarningsTracker.cpp | 98 +++++++++++++++++++++++++++++------- 1 file changed, 81 insertions(+), 17 deletions(-) diff --git a/Boss/Mod/EarningsTracker.cpp b/Boss/Mod/EarningsTracker.cpp index a1fa99d47..3f92f384d 100644 --- a/Boss/Mod/EarningsTracker.cpp +++ b/Boss/Mod/EarningsTracker.cpp @@ -65,28 +65,79 @@ class EarningsTracker::Impl { Ev::Io init() { return db.transact().then([](Sqlite3::Tx tx) { + // If we already have a bucket schema we're done + if (have_bucket_table(tx)) { + tx.commit(); + return Ev::lift(); + } + + // Create the bucket table w/ a temp name tx.query_execute(R"QRY( - CREATE TABLE IF NOT EXISTS "EarningsTracker" - ( node TEXT PRIMARY KEY + CREATE TABLE IF NOT EXISTS EarningsTracker_New + ( node TEXT NOT NULL + , time_bucket REAL NOT NULL , in_earnings INTEGER NOT NULL , in_expenditures INTEGER NOT NULL , out_earnings INTEGER NOT NULL , out_expenditures INTEGER NOT NULL + , PRIMARY KEY (node, time_bucket) ); )QRY"); + // If we have a legacy table migrate the data and lose the old table + if (have_legacy_table(tx)) { + tx.query_execute(R"QRY( + INSERT INTO EarningsTracker_New + ( node, time_bucket, in_earnings, in_expenditures + , out_earnings, out_expenditures) + SELECT node, 0, in_earnings, in_expenditures + , out_earnings, out_expenditures FROM EarningsTracker; + DROP TABLE EarningsTracker; + )QRY"); + } + + // Move the new table to the official name and add the indexes + tx.query_execute(R"QRY( + ALTER TABLE EarningsTracker_New RENAME TO EarningsTracker; + CREATE INDEX IF NOT EXISTS + idx_earnings_tracker_node_time ON EarningsTracker (node, time_bucket); + CREATE INDEX IF NOT EXISTS + idx_earnings_tracker_time_node ON EarningsTracker (time_bucket, node); + )QRY"); + tx.commit(); return Ev::lift(); }); } + + static bool have_bucket_table(Sqlite3::Tx& tx) { + auto fetch = tx.query(R"QRY( + SELECT 1 + FROM pragma_table_info('EarningsTracker') + WHERE name='time_bucket'; + )QRY").execute(); + return fetch.begin() != fetch.end(); + } + + static bool have_legacy_table(Sqlite3::Tx& tx) { + // Do we have a legacy table defined? + auto fetch = tx.query(R"QRY( + SELECT 1 + FROM pragma_table_info('EarningsTracker') + WHERE name='node'; + )QRY").execute(); + return fetch.begin() != fetch.end(); + } + /* Ensure the given node has an entry. */ - void ensure(Sqlite3::Tx& tx, Ln::NodeId const& node) { + void ensure(Sqlite3::Tx& tx, Ln::NodeId const& node, double bucket) { tx.query(R"QRY( INSERT OR IGNORE INTO "EarningsTracker" - VALUES(:node, 0, 0, 0, 0); + VALUES(:node, :bucket, 0, 0, 0, 0); )QRY") .bind(":node", std::string(node)) + .bind(":bucket", bucket) .execute() ; } @@ -96,16 +147,19 @@ class EarningsTracker::Impl { ) { return db.transact().then([this, in, out, fee ](Sqlite3::Tx tx) { - ensure(tx, in); - ensure(tx, out); + auto bucket = bucket_time(get_now()); + ensure(tx, in, bucket); + ensure(tx, out, bucket); tx.query(R"QRY( UPDATE "EarningsTracker" SET in_earnings = in_earnings + :fee WHERE node = :node + AND time_bucket = :bucket ; )QRY") .bind(":node", std::string(in)) + .bind(":bucket", bucket) .bind(":fee", fee.to_msat()) .execute() ; @@ -113,9 +167,11 @@ class EarningsTracker::Impl { UPDATE "EarningsTracker" SET out_earnings = out_earnings + :fee WHERE node = :node + AND time_bucket = :bucket ; )QRY") .bind(":node", std::string(out)) + .bind(":bucket", bucket) .bind(":fee", fee.to_msat()) .execute() ; @@ -143,8 +199,9 @@ class EarningsTracker::Impl { return Ev::lift(); auto& pending = it->second; - ensure(tx, pending.source); - ensure(tx, pending.destination); + auto bucket = bucket_time(get_now()); + ensure(tx, pending.source, bucket); + ensure(tx, pending.destination, bucket); /* Source gets in-expenditures since it gets more * incoming capacity (for more earnings for the @@ -153,9 +210,11 @@ class EarningsTracker::Impl { UPDATE "EarningsTracker" SET in_expenditures = in_expenditures + :fee WHERE node = :node + AND time_bucket = :bucket ; )QRY") .bind(":node", std::string(pending.source)) + .bind(":bucket", bucket) .bind(":fee", fee.to_msat()) .execute() ; @@ -166,11 +225,13 @@ class EarningsTracker::Impl { UPDATE "EarningsTracker" SET out_expenditures = out_expenditures + :fee WHERE node = :node + AND time_bucket = :bucket ; )QRY") .bind( ":node" , std::string(pending.destination) ) + .bind(":bucket", bucket) .bind(":fee", fee.to_msat()) .execute() ; @@ -195,11 +256,12 @@ class EarningsTracker::Impl { auto out_expenditures = Ln::Amount::sat(0); auto fetch = tx.query(R"QRY( - SELECT in_earnings, in_expenditures - , out_earnings, out_expenditures + SELECT SUM(in_earnings), + SUM(in_expenditures), + SUM(out_earnings), + SUM(out_expenditures) FROM "EarningsTracker" - WHERE node = :node - ; + WHERE node = :node; )QRY") .bind(":node", std::string(node)) .execute() @@ -231,11 +293,13 @@ class EarningsTracker::Impl { Ev::Io status() { return db.transact().then([this](Sqlite3::Tx tx) { auto fetch = tx.query(R"QRY( - SELECT node - , in_earnings, in_expenditures - , out_earnings, out_expenditures - FROM "EarningsTracker" - ; + SELECT node, + SUM(in_earnings) AS total_in_earnings, + SUM(in_expenditures) AS total_in_expenditures, + SUM(out_earnings) AS total_out_earnings, + SUM(out_expenditures) AS total_out_expenditures + FROM "EarningsTracker" + GROUP BY node; )QRY").execute(); uint64_t total_in_earnings = 0; From 0ab5ca7360c0a25034f83ba3e76e0dad07c62329 Mon Sep 17 00:00:00 2001 From: Ken Sedgwick Date: Wed, 7 Aug 2024 14:02:46 -0700 Subject: [PATCH 10/18] add Either::operator<< to support debugging --- Util/Either.hpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Util/Either.hpp b/Util/Either.hpp index 5fc5bdd95..0fbd25fd6 100644 --- a/Util/Either.hpp +++ b/Util/Either.hpp @@ -2,6 +2,7 @@ #define UTIL_EITHER_HPP #include +#include #include namespace Util { @@ -225,6 +226,19 @@ bool operator<=(Util::Either const& a, Util::Either const& b) { return !(b < a); } +template +std::ostream& operator<<(std::ostream& os, const Either& either) { + either.cmatch( + [&](const L& l) { + os << "Left(" << l << ")"; + }, + [&](const R& r) { + os << "Right(" << r << ")"; + } + ); + return os; } +} // namespace Util + #endif /* !defined(UTIL_EITHER_HPP) */ From c9c2d3ddd36b2454ea7cbb282c6166aa6bb6dc27 Mon Sep 17 00:00:00 2001 From: Ken Sedgwick Date: Wed, 7 Aug 2024 16:44:14 -0700 Subject: [PATCH 11/18] add clboss-recent-earnings and clboss-earnings-history --- Boss/Mod/EarningsTracker.cpp | 216 +++++++++++++++- Makefile.am | 2 + README.md | 49 ++++ .../test_availablerpccommandsannouncer.cpp | 10 + tests/boss/test_earningshistory.cpp | 232 ++++++++++++++++++ tests/boss/test_recentearnings.cpp | 213 ++++++++++++++++ 6 files changed, 721 insertions(+), 1 deletion(-) create mode 100644 tests/boss/test_earningshistory.cpp create mode 100644 tests/boss/test_recentearnings.cpp diff --git a/Boss/Mod/EarningsTracker.cpp b/Boss/Mod/EarningsTracker.cpp index 3f92f384d..4bb4a751a 100644 --- a/Boss/Mod/EarningsTracker.cpp +++ b/Boss/Mod/EarningsTracker.cpp @@ -1,6 +1,11 @@ #include"Boss/Mod/EarningsTracker.hpp" +#include"Boss/Msg/CommandFail.hpp" +#include"Boss/Msg/CommandRequest.hpp" +#include"Boss/Msg/CommandResponse.hpp" #include"Boss/Msg/DbResource.hpp" #include"Boss/Msg/ForwardFee.hpp" +#include"Boss/Msg/ManifestCommand.hpp" +#include"Boss/Msg/Manifestation.hpp" #include"Boss/Msg/ProvideStatus.hpp" #include"Boss/Msg/RequestEarningsInfo.hpp" #include"Boss/Msg/RequestMoveFunds.hpp" @@ -15,6 +20,7 @@ #include #include +#include namespace Boss { namespace Mod { @@ -54,7 +60,92 @@ class EarningsTracker::Impl { >([this](Msg::RequestEarningsInfo const& req) { return Boss::concurrent(request_earnings_info(req)); }); - + bus.subscribe([this](Msg::Manifestation const& _) { + return bus.raise(Msg::ManifestCommand{ + "clboss-recent-earnings", "[days]", + "Show offchain_earnings_tracker data for the most recent {days} " + "(default 14 days).", + false + }) + bus.raise(Msg::ManifestCommand{ + "clboss-earnings-history", "[nodeid]", + "Show earnings and expenditure history for {nodeid} " + "(default all nodes)", + false + }); + }); + bus.subscribe([this](Msg::CommandRequest const& req) { + auto id = req.id; + auto paramfail = [this, id]() { + return bus.raise(Msg::CommandFail{ + id, -32602, + "Parameter failure", + Json::Out::empty_object() + }); + }; + if (req.command == "clboss-recent-earnings") { + auto days = double(14.0); + auto days_j = Jsmn::Object(); + auto params = req.params; + if (params.is_object()) { + if (params.size() > 1) + return paramfail(); + if (params.size() == 1) { + if (!params.has("days")) + return paramfail(); + days_j = params["days"]; + } + } else if (params.is_array()) { + if (params.size() > 1) + return paramfail(); + if (params.size() == 1) + days_j = params[0]; + } + if (!days_j.is_null()) { + if (!days_j.is_number()) + return paramfail(); + days = (double) days_j; + } + return db.transact().then([this, id, days](Sqlite3::Tx tx) { + auto report = recent_earnings_report(tx, days); + tx.commit(); + return bus.raise(Msg::CommandResponse{ + id, report + }); + }); + } else if (req.command == "clboss-earnings-history") { + auto nodeid = std::string(""); + auto nodeid_j = Jsmn::Object(); + auto params = req.params; + if (params.is_object()) { + if (params.size() > 1) + return paramfail(); + if (params.size() == 1) { + if (!params.has("nodeid")) + return paramfail(); + nodeid_j = params["nodeid"]; + } + } else if (params.is_array()) { + if (params.size() > 1) + return paramfail(); + if (params.size() == 1) + nodeid_j = params[0]; + } + if (!nodeid_j.is_null()) { + if (!nodeid_j.is_string()) + return paramfail(); + nodeid = (std::string) nodeid_j; + } + return db.transact().then([this, id, nodeid](Sqlite3::Tx tx) { + auto report = earnings_history_report(tx, nodeid); + tx.commit(); + return bus.raise(Msg::CommandResponse{ + id, report + }); + }); + } + return Ev::lift(); + }); bus.subscribe([this](Msg::SolicitStatus const& _) { if (!db) @@ -348,6 +439,129 @@ class EarningsTracker::Impl { }); } + Json::Out recent_earnings_report(Sqlite3::Tx& tx, double days) { + auto cutoff = bucket_time(get_now()) - (days * 24 * 60 * 60); + auto fetch = tx.query(R"QRY( + SELECT node, + SUM(in_earnings) AS total_in_earnings, + SUM(in_expenditures) AS total_in_expenditures, + SUM(out_earnings) AS total_out_earnings, + SUM(out_expenditures) AS total_out_expenditures + FROM "EarningsTracker" + WHERE time_bucket >= :cutoff + GROUP BY node; + )QRY") + .bind(":cutoff", cutoff) + .execute() + ; + auto out = Json::Out(); + auto top = out.start_object(); + auto recent = top.start_object("recent"); + uint64_t total_in_earnings = 0; + uint64_t total_in_expenditures = 0; + uint64_t total_out_earnings = 0; + uint64_t total_out_expenditures = 0; + for (auto& r : fetch) { + auto in_earnings = r.get(1); + auto in_expenditures = r.get(2); + auto out_earnings = r.get(3); + auto out_expenditures = r.get(4); + auto sub = recent.start_object(r.get(0)); + sub + .field("in_earnings", in_earnings) + .field("in_expenditures", in_expenditures) + .field("out_earnings", out_earnings) + .field("out_expenditures", out_expenditures) + ; + sub.end_object(); + total_in_earnings += in_earnings; + total_in_expenditures += in_expenditures; + total_out_earnings += out_earnings; + total_out_expenditures += out_expenditures; + } + recent.end_object(); + auto total = top.start_object("total"); + total + .field("in_earnings", total_in_earnings) + .field("in_expenditures", total_in_expenditures) + .field("out_earnings", total_out_earnings) + .field("out_expenditures", total_out_expenditures) + ; + total.end_object(); + top.end_object(); + return out; + } + + Json::Out earnings_history_report(Sqlite3::Tx& tx, std::string nodeid) { + std::string sql; + if (nodeid.empty()) { + sql = R"QRY( + SELECT time_bucket, + SUM(in_earnings) AS total_in_earnings, + SUM(in_expenditures) AS total_in_expenditures, + SUM(out_earnings) AS total_out_earnings, + SUM(out_expenditures) AS total_out_expenditures + FROM "EarningsTracker" + GROUP BY time_bucket + ORDER BY time_bucket; + )QRY"; + } else { + sql = R"QRY( + SELECT time_bucket, + in_earnings, + in_expenditures, + out_earnings, + out_expenditures + FROM "EarningsTracker" + WHERE node = :nodeid + ORDER BY time_bucket; + )QRY"; + } + auto query = tx.query(sql.c_str()); + if (!nodeid.empty()) { + query.bind(":nodeid", nodeid); + } + auto fetch = query.execute(); + + auto out = Json::Out(); + auto top = out.start_object(); + auto history = top.start_array("history"); + uint64_t total_in_earnings = 0; + uint64_t total_in_expenditures = 0; + uint64_t total_out_earnings = 0; + uint64_t total_out_expenditures = 0; + for (auto& r : fetch) { + auto in_earnings = r.get(1); + auto in_expenditures = r.get(2); + auto out_earnings = r.get(3); + auto out_expenditures = r.get(4); + auto sub = history.start_object(); + sub + .field("bucket_time", static_cast(r.get(0))) + .field("in_earnings", in_earnings) + .field("in_expenditures", in_expenditures) + .field("out_earnings", out_earnings) + .field("out_expenditures", out_expenditures) + ; + sub.end_object(); + total_in_earnings += in_earnings; + total_in_expenditures += in_expenditures; + total_out_earnings += out_earnings; + total_out_expenditures += out_expenditures; + } + history.end_array(); + auto total = top.start_object("total"); + total + .field("in_earnings", total_in_earnings) + .field("in_expenditures", total_in_expenditures) + .field("out_earnings", total_out_earnings) + .field("out_expenditures", total_out_expenditures) + ; + total.end_object(); + top.end_object(); + return out; + } + public: Impl() =delete; Impl(Impl&&) =delete; diff --git a/Makefile.am b/Makefile.am index bd9fc9683..1ad6540e5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -600,6 +600,8 @@ TESTS = \ tests/boss/test_needsconnectsolicitor \ tests/boss/test_onchainfeemonitor_samples_init \ tests/boss/test_peerjudge_agetracker \ + tests/boss/test_recentearnings \ + tests/boss/test_earningshistory \ tests/boss/test_peerjudge_algo \ tests/boss/test_peerjudge_datagatherer \ tests/boss/test_peerstatistician \ diff --git a/README.md b/README.md index b0b444547..81272ad1c 100644 --- a/README.md +++ b/README.md @@ -433,3 +433,52 @@ Specify the value in satoshis without adding any unit suffix, e.g. lightningd --clboss-min-channel=1000000 + +### `clboss-recent-earnings`, `clboss-earnings-history` + +As of CLBOSS version [TBD] earnings and expenditures are tracked on a +daily basis. + +The `clboss-recent-earnings` command returns an equivalent data +structure to the `offchain_earnings_tracker` collection in +`clboss-status` but only shows recent earnings and expenditures. The +`clboss-recent-earnings` command takes an optional `days` argument +which defaults to a fortnight (14 days). + +The `clboss-earnings-history` command returns a daily breakdown of +earnings and expenditures. The `clboss-earnings-history` command +takes an optional `nodeid` argument which limits the history to a +particular node. Without the argument the daily history accumulated +across all nodes. In the history output the time bucket with value 0 +holds any legacy earnings and expenditures which were collected by +legacy nodes before daily tracking. + +### `clboss-recent-earnings`, `clboss-earnings-history` Commands + +As of CLBOSS version [TBD], earnings and expenditures are tracked on a daily basis. +The following commands have been added to observe the new data: + +- **`clboss-recent-earnings`**: + - **Purpose**: Returns a data structure equivalent to the + `offchain_earnings_tracker` collection in `clboss-status`, but + only includes recent earnings and expenditures. + - **Arguments**: + - `days` (optional): Specifies the number of days to include in + the report. Defaults to a fortnight (14 days) if not provided. + +- **`clboss-earnings-history`**: + - **Purpose**: Provides a daily breakdown of earnings and expenditures. + - **Arguments**: + - `nodeid` (optional): Limits the history to a particular node if + provided. Without this argument, the history is accumulated + across all nodes. + - **Output**: + - The history consists of an array of records showing the earnings + and expenditures for each day. + - The history includes an initial record with a time value of 0, + which contains any legacy earnings and expenditures collected by + CLBOSS before daily tracking was implemented. + +These commands enhance the tracking of financial metrics, allowing for +detailed and recent analysis of earnings and expenditures on a daily +basis. diff --git a/tests/boss/test_availablerpccommandsannouncer.cpp b/tests/boss/test_availablerpccommandsannouncer.cpp index eb6157dd6..28c454962 100644 --- a/tests/boss/test_availablerpccommandsannouncer.cpp +++ b/tests/boss/test_availablerpccommandsannouncer.cpp @@ -25,6 +25,12 @@ auto const help = std::string(R"JSON( , { "command": "clboss-unmanage nodeid tags" , "category": "plugin" } + , { "command": "clboss-recent-earnings [days]" + , "category": "plugin" + } + , { "command": "clboss-earnings-history [nodeid]" + , "category": "plugin" + } ] } )JSON"); @@ -73,6 +79,10 @@ int main() { assert(commands["clboss-status"].usage == ""); assert(commands["clboss-ignore-onchain"].usage == "[hours]"); assert(commands["clboss-unmanage"].usage == "nodeid tags"); + assert(commands.count("clboss-recent-earnings") != 0); + assert(commands["clboss-recent-earnings"].usage == "[days]"); + assert(commands.count("clboss-earnings-history") != 0); + assert(commands["clboss-earnings-history"].usage == "[nodeid]"); return Ev::lift(); }); diff --git a/tests/boss/test_earningshistory.cpp b/tests/boss/test_earningshistory.cpp new file mode 100644 index 000000000..6e9ac35b7 --- /dev/null +++ b/tests/boss/test_earningshistory.cpp @@ -0,0 +1,232 @@ +#undef NDEBUG + +#include"Boss/Mod/EarningsTracker.hpp" +#include"Boss/Msg/DbResource.hpp" +#include"Boss/Msg/ForwardFee.hpp" +#include"Boss/Msg/RequestMoveFunds.hpp" +#include"Boss/Msg/ResponseMoveFunds.hpp" +#include"Boss/Msg/CommandResponse.hpp" +#include"Boss/Msg/CommandRequest.hpp" +#include"Ev/start.hpp" +#include"Jsmn/Object.hpp" +#include"Jsmn/Parser.hpp" +#include"Json/Out.hpp" +#include"Ln/NodeId.hpp" +#include"S/Bus.hpp" +#include"Sqlite3.hpp" + +#include +#include +#include + +namespace { +auto const A = Ln::NodeId("020000000000000000000000000000000000000000000000000000000000000001"); +auto const B = Ln::NodeId("020000000000000000000000000000000000000000000000000000000000000002"); +auto const C = Ln::NodeId("020000000000000000000000000000000000000000000000000000000000000003"); +} + +double mock_now = 0.0; +double mock_get_now() { + return mock_now; +} + +Ev::Io raiseForwardFeeLoop(S::Bus& bus, int count) { + // Creates a series of forwards, one per hour + if (count == 0) { + return Ev::lift(); + } + return bus.raise(Boss::Msg::ForwardFee{ + A, // in_id + B, // out_id + Ln::Amount::sat(1), // fee + 1.0 // resolution_time + }) + .then([&, count]() { + mock_now += 60 * 60; + return raiseForwardFeeLoop(bus, count - 1); + }); +} + +Ev::Io raiseMoveFundsLoop(S::Bus& bus, int count) { + if (count == 0) { + return Ev::lift(); + } + return bus.raise( + Boss::Msg::RequestMoveFunds{ + nullptr, // requester (match ResponseMoveFunds) + C, // source + A, // destination + Ln::Amount::sat(1000), // amount + Ln::Amount::sat(3) // fee_budget + }) + .then([&bus]() { + return bus.raise( + Boss::Msg::ResponseMoveFunds{ + nullptr, // requester (see RequestMoveFunds) + Ln::Amount::sat(1000), // amount_moved + Ln::Amount::sat(2) // fee_spent + }); + }) + .then([&bus, count]() { + mock_now += 24 * 60 * 60; // advance one day + return raiseMoveFundsLoop(bus, count - 1); + }); +} + +int main() { + auto bus = S::Bus(); + + /* Module under test */ + Boss::Mod::EarningsTracker mut(bus, &mock_get_now); + + auto db = Sqlite3::Db(":memory:"); + + auto req_id = std::uint64_t(); + auto lastRsp = Boss::Msg::CommandResponse{}; + auto rsp = false; + bus.subscribe([&](Boss::Msg::CommandResponse const& m) { + lastRsp = m; + rsp = true; + return Ev::yield(); + }); + + + auto code = Ev::lift().then([&]() { + return bus.raise(Boss::Msg::DbResource{ db }); + }).then([&]() { + // Insert 2 months of one fee per hour + mock_now = 1722902400; // stroke of midnight + return raiseForwardFeeLoop(bus, 24 * 60); + }).then([&]() { + // Rewind 1 week and add 7 days of one rebalance per day + mock_now -= (7 * 24 * 60 * 60); + return raiseMoveFundsLoop(bus, 7); + }).then([&]() { + // Check history for all peers + ++req_id; + return bus.raise(Boss::Msg::CommandRequest{ + "clboss-earnings-history", + Jsmn::Object(), + Ln::CommandId::left(req_id) + }); + }).then([&]() { + // std::cerr << lastRsp.response.output() << std::endl; + assert(rsp); + assert(lastRsp.id == Ln::CommandId::left(req_id)); + auto result = Jsmn::Object::parse_json(lastRsp.response.output().c_str()); + assert(result["history"].size() == 60); + assert(result["history"][0] == Jsmn::Object::parse_json(R"JSON( + { + "bucket_time": 1722902400, + "in_earnings": 24000, + "in_expenditures": 0, + "out_earnings": 24000, + "out_expenditures": 0 + } + )JSON")); + assert(result["history"][59] == Jsmn::Object::parse_json(R"JSON( + { + "bucket_time": 1728000000, + "in_earnings": 24000, + "in_expenditures": 2000, + "out_earnings": 24000, + "out_expenditures": 2000 + } + )JSON")); + assert(result["total"] == Jsmn::Object::parse_json(R"JSON( + { + "in_earnings": 1440000, + "in_expenditures": 14000, + "out_earnings": 1440000, + "out_expenditures": 14000 + } + )JSON")); + return Ev::lift(); + }).then([&]() { + // Check history for peer A + ++req_id; + return bus.raise(Boss::Msg::CommandRequest{ + "clboss-earnings-history", + Jsmn::Object::parse_json(R"JSON(["020000000000000000000000000000000000000000000000000000000000000001"])JSON"), + Ln::CommandId::left(req_id) + }); + }).then([&]() { + // std::cerr << lastRsp.response.output() << std::endl; + assert(rsp); + assert(lastRsp.id == Ln::CommandId::left(req_id)); + auto result = Jsmn::Object::parse_json(lastRsp.response.output().c_str()); + assert(result["history"].size() == 60); + assert(result["history"][0] == Jsmn::Object::parse_json(R"JSON( + { + "bucket_time": 1722902400, + "in_earnings": 24000, + "in_expenditures": 0, + "out_earnings": 0, + "out_expenditures": 0 + } + )JSON")); + assert(result["history"][59] == Jsmn::Object::parse_json(R"JSON( + { + "bucket_time": 1728000000, + "in_earnings": 24000, + "in_expenditures": 0, + "out_earnings": 0, + "out_expenditures": 2000 + } + )JSON")); + assert(result["total"] == Jsmn::Object::parse_json(R"JSON( + { + "in_earnings": 1440000, + "in_expenditures": 0, + "out_earnings": 0, + "out_expenditures": 14000 + } + )JSON")); + return Ev::lift(); + }).then([&]() { + // Check history for peer C (should only be one week) + ++req_id; + return bus.raise(Boss::Msg::CommandRequest{ + "clboss-earnings-history", + Jsmn::Object::parse_json(R"JSON(["020000000000000000000000000000000000000000000000000000000000000003"])JSON"), + Ln::CommandId::left(req_id) + }); + }).then([&]() { + // std::cerr << lastRsp.response.output() << std::endl; + assert(rsp); + assert(lastRsp.id == Ln::CommandId::left(req_id)); + auto result = Jsmn::Object::parse_json(lastRsp.response.output().c_str()); + assert(result["history"].size() == 7); + assert(result["history"][0] == Jsmn::Object::parse_json(R"JSON( + { + "bucket_time": 1727481600, + "in_earnings": 0, + "in_expenditures": 2000, + "out_earnings": 0, + "out_expenditures": 0 + } + )JSON")); + assert(result["history"][6] == Jsmn::Object::parse_json(R"JSON( + { + "bucket_time": 1728000000, + "in_earnings": 0, + "in_expenditures": 2000, + "out_earnings": 0, + "out_expenditures": 0 + } + )JSON")); + assert(result["total"] == Jsmn::Object::parse_json(R"JSON( + { + "in_earnings": 0, + "in_expenditures": 14000, + "out_earnings": 0, + "out_expenditures": 0 + } + )JSON")); + return Ev::lift(); + }).then([&]() { + return Ev::lift(0); + }); + + return Ev::start(std::move(code)); +} diff --git a/tests/boss/test_recentearnings.cpp b/tests/boss/test_recentearnings.cpp new file mode 100644 index 000000000..78045419e --- /dev/null +++ b/tests/boss/test_recentearnings.cpp @@ -0,0 +1,213 @@ +#undef NDEBUG + +#include"Boss/Mod/EarningsTracker.hpp" +#include"Boss/Msg/DbResource.hpp" +#include"Boss/Msg/ForwardFee.hpp" +#include"Boss/Msg/RequestMoveFunds.hpp" +#include"Boss/Msg/ResponseMoveFunds.hpp" +#include"Boss/Msg/CommandResponse.hpp" +#include"Boss/Msg/CommandRequest.hpp" +#include"Ev/start.hpp" +#include"Jsmn/Object.hpp" +#include"Jsmn/Parser.hpp" +#include"Json/Out.hpp" +#include"Ln/NodeId.hpp" +#include"S/Bus.hpp" +#include"Sqlite3.hpp" + +#include +#include +#include + +namespace { +auto const A = Ln::NodeId("020000000000000000000000000000000000000000000000000000000000000001"); +auto const B = Ln::NodeId("020000000000000000000000000000000000000000000000000000000000000002"); +auto const C = Ln::NodeId("020000000000000000000000000000000000000000000000000000000000000003"); +} + +double mock_now = 0.0; +double mock_get_now() { + return mock_now; +} + +Ev::Io raiseForwardFeeLoop(S::Bus& bus, int count) { + // Creates a series of forwards, one per hour + if (count == 0) { + return Ev::lift(); + } + return bus.raise(Boss::Msg::ForwardFee{ + A, // in_id + B, // out_id + Ln::Amount::sat(1), // fee + 1.0 // resolution_time + }) + .then([&, count]() { + mock_now += 60 * 60; + return raiseForwardFeeLoop(bus, count - 1); + }); +} + +Ev::Io raiseMoveFundsLoop(S::Bus& bus, int count) { + if (count == 0) { + return Ev::lift(); + } + return bus.raise( + Boss::Msg::RequestMoveFunds{ + nullptr, // requester (match ResponseMoveFunds) + C, // source + A, // destination + Ln::Amount::sat(1000), // amount + Ln::Amount::sat(3) // fee_budget + }) + .then([&bus]() { + return bus.raise( + Boss::Msg::ResponseMoveFunds{ + nullptr, // requester (see RequestMoveFunds) + Ln::Amount::sat(1000), // amount_moved + Ln::Amount::sat(2) // fee_spent + }); + }) + .then([&bus, count]() { + mock_now += 24 * 60 * 60; // advance one day + return raiseMoveFundsLoop(bus, count - 1); + }); +} + +int main() { + auto bus = S::Bus(); + + /* Module under test */ + Boss::Mod::EarningsTracker mut(bus, &mock_get_now); + + auto db = Sqlite3::Db(":memory:"); + + auto req_id = std::uint64_t(); + auto lastRsp = Boss::Msg::CommandResponse{}; + auto rsp = false; + bus.subscribe([&](Boss::Msg::CommandResponse const& m) { + lastRsp = m; + rsp = true; + return Ev::yield(); + }); + + + auto code = Ev::lift().then([&]() { + return bus.raise(Boss::Msg::DbResource{ db }); + }).then([&]() { + ++req_id; + return bus.raise(Boss::Msg::CommandRequest{ + "clboss-recent-earnings", + Jsmn::Object(), + Ln::CommandId::left(req_id) + }); + }).then([&]() { + // With no fees should see an empty result collection + auto result = Jsmn::Object::parse_json(lastRsp.response.output().c_str()); + assert(result["recent"] == Jsmn::Object::parse_json(R"JSON( + {} + )JSON")); + return Ev::lift(); + }).then([&]() { + // Insert 2 months of one fee per hour + mock_now = 1722902400; // stroke of midnight + return raiseForwardFeeLoop(bus, 24 * 60); + }).then([&]() { + // Rewind 1 week and add 7 days of one rebalance per day + mock_now -= (7 * 24 * 60 * 60); + return raiseMoveFundsLoop(bus, 7); + }).then([&]() { + // Default report should include 14 days + ++req_id; + return bus.raise(Boss::Msg::CommandRequest{ + "clboss-recent-earnings", + Jsmn::Object(), + Ln::CommandId::left(req_id) + }); + }).then([&]() { + // std::cerr << lastRsp.response.output() << std::endl; + assert(rsp); + assert(lastRsp.id == Ln::CommandId::left(req_id)); + auto result = Jsmn::Object::parse_json(lastRsp.response.output().c_str()); + assert(result["recent"] == Jsmn::Object::parse_json(R"JSON( + { + "020000000000000000000000000000000000000000000000000000000000000001": { + "in_earnings": 336000, + "in_expenditures": 0, + "out_earnings": 0, + "out_expenditures": 14000 + }, + "020000000000000000000000000000000000000000000000000000000000000002": { + "in_earnings": 0, + "in_expenditures": 0, + "out_earnings": 336000, + "out_expenditures": 0 + }, + "020000000000000000000000000000000000000000000000000000000000000003": { + "in_earnings": 0, + "in_expenditures": 14000, + "out_earnings": 0, + "out_expenditures": 0 + } + } + )JSON")); + assert(result["total"] == Jsmn::Object::parse_json(R"JSON( + { + "in_earnings": 336000, + "in_expenditures": 14000, + "out_earnings": 336000, + "out_expenditures": 14000 + } + )JSON")); + return Ev::lift(); + }).then([&]() { + // Check a 30 day report + ++req_id; + return bus.raise(Boss::Msg::CommandRequest{ + "clboss-recent-earnings", + Jsmn::Object::parse_json(R"JSON( [30] )JSON"), + Ln::CommandId::left(req_id) + }); + }).then([&]() { + // std::cerr << lastRsp.response.output() << std::endl; + assert(rsp); + assert(lastRsp.id == Ln::CommandId::left(req_id)); + auto result = Jsmn::Object::parse_json(lastRsp.response.output().c_str()); + // The expenditures stay the same because they are entirely in the last + // week but the earnings increase. + assert(result["recent"] == Jsmn::Object::parse_json(R"JSON( + { + "020000000000000000000000000000000000000000000000000000000000000001": { + "in_earnings": 720000, + "in_expenditures": 0, + "out_earnings": 0, + "out_expenditures": 14000 + }, + "020000000000000000000000000000000000000000000000000000000000000002": { + "in_earnings": 0, + "in_expenditures": 0, + "out_earnings": 720000, + "out_expenditures": 0 + }, + "020000000000000000000000000000000000000000000000000000000000000003": { + "in_earnings": 0, + "in_expenditures": 14000, + "out_earnings": 0, + "out_expenditures": 0 + } + } + )JSON")); + assert(result["total"] == Jsmn::Object::parse_json(R"JSON( + { + "in_earnings": 720000, + "in_expenditures": 14000, + "out_earnings": 720000, + "out_expenditures": 14000 + } + )JSON")); + return Ev::lift(); + }).then([&]() { + return Ev::lift(0); + }); + + return Ev::start(std::move(code)); +} From 35cbf0abc9ee1e93ca811454c16ebed2f71d48bf Mon Sep 17 00:00:00 2001 From: Ken Sedgwick Date: Sun, 11 Aug 2024 14:04:34 -0700 Subject: [PATCH 12/18] f: sort clboss-recent-earnings by earnings sum descending --- Boss/Mod/EarningsTracker.cpp | 4 +++- tests/boss/test_recentearnings.cpp | 10 +++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Boss/Mod/EarningsTracker.cpp b/Boss/Mod/EarningsTracker.cpp index 4bb4a751a..a3e329eb9 100644 --- a/Boss/Mod/EarningsTracker.cpp +++ b/Boss/Mod/EarningsTracker.cpp @@ -449,7 +449,9 @@ class EarningsTracker::Impl { SUM(out_expenditures) AS total_out_expenditures FROM "EarningsTracker" WHERE time_bucket >= :cutoff - GROUP BY node; + GROUP BY node + ORDER BY (total_in_earnings - total_in_expenditures + + total_out_earnings - total_out_expenditures) DESC; )QRY") .bind(":cutoff", cutoff) .execute() diff --git a/tests/boss/test_recentearnings.cpp b/tests/boss/test_recentearnings.cpp index 78045419e..3079a9c40 100644 --- a/tests/boss/test_recentearnings.cpp +++ b/tests/boss/test_recentearnings.cpp @@ -55,7 +55,7 @@ Ev::Io raiseMoveFundsLoop(S::Bus& bus, int count) { Boss::Msg::RequestMoveFunds{ nullptr, // requester (match ResponseMoveFunds) C, // source - A, // destination + B, // destination Ln::Amount::sat(1000), // amount Ln::Amount::sat(3) // fee_budget }) @@ -134,13 +134,13 @@ int main() { "in_earnings": 336000, "in_expenditures": 0, "out_earnings": 0, - "out_expenditures": 14000 + "out_expenditures": 0 }, "020000000000000000000000000000000000000000000000000000000000000002": { "in_earnings": 0, "in_expenditures": 0, "out_earnings": 336000, - "out_expenditures": 0 + "out_expenditures": 14000 }, "020000000000000000000000000000000000000000000000000000000000000003": { "in_earnings": 0, @@ -180,13 +180,13 @@ int main() { "in_earnings": 720000, "in_expenditures": 0, "out_earnings": 0, - "out_expenditures": 14000 + "out_expenditures": 0 }, "020000000000000000000000000000000000000000000000000000000000000002": { "in_earnings": 0, "in_expenditures": 0, "out_earnings": 720000, - "out_expenditures": 0 + "out_expenditures": 14000 }, "020000000000000000000000000000000000000000000000000000000000000003": { "in_earnings": 0, From dc0f13a57173c9a89296bcd1acf8086fba8df0d0 Mon Sep 17 00:00:00 2001 From: Ken Sedgwick Date: Thu, 15 Aug 2024 13:37:29 -0700 Subject: [PATCH 13/18] Add associated primary volume (forwarded and rebalanced) to EarningsTracker Addresses ([#229]) This allows effective feerates (PPM) to be computed for earnings and expenses. This PR updates the schema automatically. Downgrading to previous will require manual DB migration (but is possible). --- Boss/Mod/EarningsTracker.cpp | 295 ++++++++++++++++---------- Boss/Mod/ForwardFeeMonitor.cpp | 11 +- Boss/Mod/ForwardFeeMonitor.hpp | 1 + Boss/Msg/ForwardFee.hpp | 2 + Boss/Msg/ResponseEarningsInfo.hpp | 5 + tests/boss/test_earningshistory.cpp | 89 +++++--- tests/boss/test_earningstracker.cpp | 66 ++++-- tests/boss/test_forwardfeemonitor.cpp | 2 +- tests/boss/test_recentearnings.cpp | 141 +++++++----- 9 files changed, 401 insertions(+), 211 deletions(-) diff --git a/Boss/Mod/EarningsTracker.cpp b/Boss/Mod/EarningsTracker.cpp index a3e329eb9..d12b28576 100644 --- a/Boss/Mod/EarningsTracker.cpp +++ b/Boss/Mod/EarningsTracker.cpp @@ -14,6 +14,7 @@ #include"Boss/Msg/SolicitStatus.hpp" #include"Boss/concurrent.hpp" #include"Ev/Io.hpp" +#include"Json/Out.hpp" #include"S/Bus.hpp" #include"Sqlite3.hpp" #include"Util/make_unique.hpp" @@ -21,6 +22,62 @@ #include #include #include +#include + +namespace { + +struct EarningsData { + uint64_t in_earnings = 0; + uint64_t in_expenditures = 0; + uint64_t out_earnings = 0; + uint64_t out_expenditures = 0; + uint64_t in_forwarded = 0; + uint64_t in_rebalanced = 0; + uint64_t out_forwarded = 0; + uint64_t out_rebalanced = 0; + + // Unmarshal from database fetch + static EarningsData from_row(Sqlite3::Row& r, size_t& ndx) { + return { + r.get(ndx++), + r.get(ndx++), + r.get(ndx++), + r.get(ndx++), + r.get(ndx++), + r.get(ndx++), + r.get(ndx++), + r.get(ndx++) + }; + } + + // Operator+= for accumulation + EarningsData& operator+=(const EarningsData& other) { + in_earnings += other.in_earnings; + in_expenditures += other.in_expenditures; + out_earnings += other.out_earnings; + out_expenditures += other.out_expenditures; + in_forwarded += other.in_forwarded; + in_rebalanced += other.in_rebalanced; + out_forwarded += other.out_forwarded; + out_rebalanced += other.out_rebalanced; + return *this; + } + + template + void to_json(Json::Detail::Object& obj) const { + obj + .field("in_earnings", in_earnings) + .field("in_expenditures", in_expenditures) + .field("out_earnings", out_earnings) + .field("out_expenditures", out_expenditures) + .field("in_forwarded", in_forwarded) + .field("in_rebalanced", in_rebalanced) + .field("out_forwarded", out_forwarded) + .field("out_rebalanced", out_rebalanced); + } +}; + +} namespace Boss { namespace Mod { @@ -46,7 +103,7 @@ class EarningsTracker::Impl { }); bus.subscribe([this](Msg::ForwardFee const& f) { - return forward_fee(f.in_id, f.out_id, f.fee); + return forward_fee(f.in_id, f.out_id, f.fee, f.amount); }); bus.subscribe([this](Msg::RequestMoveFunds const& req) { @@ -158,6 +215,7 @@ class EarningsTracker::Impl { return db.transact().then([](Sqlite3::Tx tx) { // If we already have a bucket schema we're done if (have_bucket_table(tx)) { + add_missing_columns(tx); tx.commit(); return Ev::lift(); } @@ -171,6 +229,10 @@ class EarningsTracker::Impl { , in_expenditures INTEGER NOT NULL , out_earnings INTEGER NOT NULL , out_expenditures INTEGER NOT NULL + , in_forwarded INTEGER NOT NULL + , in_rebalanced INTEGER NOT NULL + , out_forwarded INTEGER NOT NULL + , out_rebalanced INTEGER NOT NULL , PRIMARY KEY (node, time_bucket) ); )QRY"); @@ -220,12 +282,40 @@ class EarningsTracker::Impl { return fetch.begin() != fetch.end(); } + static void add_missing_columns(Sqlite3::Tx& tx) { + std::unordered_set existing_columns; + auto columns_query = tx.query("PRAGMA table_info(EarningsTracker);").execute(); + for (auto& row : columns_query) { + existing_columns.insert(row.get(1)); + } + + // These columns are new. + std::vector> columns_to_check = { + {"in_forwarded", "INTEGER NOT NULL DEFAULT 0"}, + {"in_rebalanced", "INTEGER NOT NULL DEFAULT 0"}, + {"out_forwarded", "INTEGER NOT NULL DEFAULT 0"}, + {"out_rebalanced", "INTEGER NOT NULL DEFAULT 0"}, + }; + + // Add missing columns + for (const auto& [col_name, col_def] : columns_to_check) { + if (existing_columns.find(col_name) == existing_columns.end()) { + std::string add_column_query = + "ALTER TABLE EarningsTracker ADD COLUMN " + + col_name + " " + col_def + ";"; + tx.query_execute(add_column_query); + } + } + } + /* Ensure the given node has an entry. */ void ensure(Sqlite3::Tx& tx, Ln::NodeId const& node, double bucket) { tx.query(R"QRY( INSERT OR IGNORE INTO "EarningsTracker" - VALUES(:node, :bucket, 0, 0, 0, 0); + VALUES(:node, :bucket, + 0, 0, 0, 0, + 0, 0, 0, 0); )QRY") .bind(":node", std::string(node)) .bind(":bucket", bucket) @@ -235,8 +325,9 @@ class EarningsTracker::Impl { Ev::Io forward_fee( Ln::NodeId const& in , Ln::NodeId const& out , Ln::Amount fee + , Ln::Amount amount ) { - return db.transact().then([this, in, out, fee + return db.transact().then([this, in, out, fee, amount ](Sqlite3::Tx tx) { auto bucket = bucket_time(get_now()); ensure(tx, in, bucket); @@ -244,26 +335,30 @@ class EarningsTracker::Impl { tx.query(R"QRY( UPDATE "EarningsTracker" - SET in_earnings = in_earnings + :fee + SET in_earnings = in_earnings + :fee, + in_forwarded = in_forwarded + :amount WHERE node = :node AND time_bucket = :bucket ; )QRY") + .bind(":fee", fee.to_msat()) + .bind(":amount", amount.to_msat()) .bind(":node", std::string(in)) .bind(":bucket", bucket) - .bind(":fee", fee.to_msat()) .execute() ; tx.query(R"QRY( UPDATE "EarningsTracker" - SET out_earnings = out_earnings + :fee + SET out_earnings = out_earnings + :fee, + out_forwarded = out_forwarded + :amount WHERE node = :node AND time_bucket = :bucket ; )QRY") + .bind(":fee", fee.to_msat()) + .bind(":amount", amount.to_msat()) .bind(":node", std::string(out)) .bind(":bucket", bucket) - .bind(":fee", fee.to_msat()) .execute() ; @@ -282,7 +377,8 @@ class EarningsTracker::Impl { response_move_funds(Boss::Msg::ResponseMoveFunds const& rsp) { auto requester = rsp.requester; auto fee = rsp.fee_spent; - return db.transact().then([this, requester, fee + auto amount = rsp.amount_moved; + return db.transact().then([this, requester, fee, amount ](Sqlite3::Tx tx) { auto it = pendings.find(requester); if (it == pendings.end()) @@ -299,7 +395,8 @@ class EarningsTracker::Impl { * incoming direction). */ tx.query(R"QRY( UPDATE "EarningsTracker" - SET in_expenditures = in_expenditures + :fee + SET in_expenditures = in_expenditures + :fee, + in_rebalanced = in_rebalanced + :amount WHERE node = :node AND time_bucket = :bucket ; @@ -307,6 +404,7 @@ class EarningsTracker::Impl { .bind(":node", std::string(pending.source)) .bind(":bucket", bucket) .bind(":fee", fee.to_msat()) + .bind(":amount", amount.to_msat()) .execute() ; /* Destination gets out-expenditures for same @@ -314,7 +412,8 @@ class EarningsTracker::Impl { */ tx.query(R"QRY( UPDATE "EarningsTracker" - SET out_expenditures = out_expenditures + :fee + SET out_expenditures = out_expenditures + :fee, + out_rebalanced = out_rebalanced + :amount WHERE node = :node AND time_bucket = :bucket ; @@ -324,6 +423,7 @@ class EarningsTracker::Impl { ) .bind(":bucket", bucket) .bind(":fee", fee.to_msat()) + .bind(":amount", amount.to_msat()) .execute() ; @@ -341,42 +441,39 @@ class EarningsTracker::Impl { auto node = req.node; return db.transact().then([this, requester, node ](Sqlite3::Tx tx) { - auto in_earnings = Ln::Amount::sat(0); - auto in_expenditures = Ln::Amount::sat(0); - auto out_earnings = Ln::Amount::sat(0); - auto out_expenditures = Ln::Amount::sat(0); - auto fetch = tx.query(R"QRY( SELECT SUM(in_earnings), SUM(in_expenditures), SUM(out_earnings), - SUM(out_expenditures) + SUM(out_expenditures), + SUM(in_forwarded), + SUM(in_rebalanced), + SUM(out_forwarded), + SUM(out_rebalanced) FROM "EarningsTracker" WHERE node = :node; )QRY") .bind(":node", std::string(node)) .execute() ; - for (auto& r : fetch) { - in_earnings = Ln::Amount::msat( - r.get(0) - ); - in_expenditures = Ln::Amount::msat( - r.get(1) - ); - out_earnings = Ln::Amount::msat( - r.get(2) - ); - out_expenditures = Ln::Amount::msat( - r.get(3) - ); + + EarningsData earnings; + if (fetch.begin() != fetch.end()) { + size_t ndx = 0; + earnings = EarningsData::from_row(*fetch.begin(), ndx); } tx.commit(); return bus.raise(Msg::ResponseEarningsInfo{ requester, node, - in_earnings, in_expenditures, - out_earnings, out_expenditures + Ln::Amount::msat(earnings.in_earnings), + Ln::Amount::msat(earnings.in_expenditures), + Ln::Amount::msat(earnings.out_earnings), + Ln::Amount::msat(earnings.out_expenditures), + Ln::Amount::msat(earnings.in_forwarded), + Ln::Amount::msat(earnings.in_rebalanced), + Ln::Amount::msat(earnings.out_forwarded), + Ln::Amount::msat(earnings.out_rebalanced), }); }); } @@ -388,46 +485,30 @@ class EarningsTracker::Impl { SUM(in_earnings) AS total_in_earnings, SUM(in_expenditures) AS total_in_expenditures, SUM(out_earnings) AS total_out_earnings, - SUM(out_expenditures) AS total_out_expenditures + SUM(out_expenditures) AS total_out_expenditures, + SUM(in_forwarded) AS total_in_forwarded, + SUM(in_rebalanced) AS total_in_rebalanced, + SUM(out_forwarded) AS total_out_forwarded, + SUM(out_rebalanced) AS total_out_rebalanced FROM "EarningsTracker" - GROUP BY node; + GROUP BY node; )QRY").execute(); - uint64_t total_in_earnings = 0; - uint64_t total_in_expenditures = 0; - uint64_t total_out_earnings = 0; - uint64_t total_out_expenditures = 0; - + EarningsData total_earnings; auto out = Json::Out(); auto obj = out.start_object(); for (auto& r : fetch) { - auto in_earnings = r.get(1); - auto in_expenditures = r.get(2); - auto out_earnings = r.get(3); - auto out_expenditures = r.get(4); - auto sub = obj.start_object(r.get(0)); - sub - .field("in_earnings", in_earnings) - .field("in_expenditures", in_expenditures) - .field("out_earnings", out_earnings) - .field("out_expenditures", out_expenditures) - ; + size_t ndx = 0; + auto node = r.get(ndx++); + auto earnings = EarningsData::from_row(r, ndx); + auto sub = obj.start_object(node); + earnings.to_json(sub); sub.end_object(); - total_in_earnings += in_earnings; - total_in_expenditures += in_expenditures; - total_out_earnings += out_earnings; - total_out_expenditures += out_expenditures; + total_earnings += earnings; } - - auto sub = obj.start_object("total"); - sub - .field("in_earnings", total_in_earnings) - .field("in_expenditures", total_in_expenditures) - .field("out_earnings", total_out_earnings) - .field("out_expenditures", total_out_expenditures) - ; - sub.end_object(); - + auto total = obj.start_object("total"); + total_earnings.to_json(total); + total.end_object(); obj.end_object(); tx.commit(); @@ -446,7 +527,11 @@ class EarningsTracker::Impl { SUM(in_earnings) AS total_in_earnings, SUM(in_expenditures) AS total_in_expenditures, SUM(out_earnings) AS total_out_earnings, - SUM(out_expenditures) AS total_out_expenditures + SUM(out_expenditures) AS total_out_expenditures, + SUM(in_forwarded) AS total_in_forwarded, + SUM(in_rebalanced) AS total_in_rebalanced, + SUM(out_forwarded) AS total_out_forwarded, + SUM(out_rebalanced) AS total_out_rebalanced FROM "EarningsTracker" WHERE time_bucket >= :cutoff GROUP BY node @@ -456,40 +541,26 @@ class EarningsTracker::Impl { .bind(":cutoff", cutoff) .execute() ; + + EarningsData total_earnings; auto out = Json::Out(); auto top = out.start_object(); auto recent = top.start_object("recent"); - uint64_t total_in_earnings = 0; - uint64_t total_in_expenditures = 0; - uint64_t total_out_earnings = 0; - uint64_t total_out_expenditures = 0; for (auto& r : fetch) { - auto in_earnings = r.get(1); - auto in_expenditures = r.get(2); - auto out_earnings = r.get(3); - auto out_expenditures = r.get(4); - auto sub = recent.start_object(r.get(0)); - sub - .field("in_earnings", in_earnings) - .field("in_expenditures", in_expenditures) - .field("out_earnings", out_earnings) - .field("out_expenditures", out_expenditures) - ; + size_t ndx = 0; + auto node = r.get(ndx++); + auto earnings = EarningsData::from_row(r, ndx); + auto sub = recent.start_object(node); + earnings.to_json(sub); sub.end_object(); - total_in_earnings += in_earnings; - total_in_expenditures += in_expenditures; - total_out_earnings += out_earnings; - total_out_expenditures += out_expenditures; + total_earnings += earnings; } recent.end_object(); + auto total = top.start_object("total"); - total - .field("in_earnings", total_in_earnings) - .field("in_expenditures", total_in_expenditures) - .field("out_earnings", total_out_earnings) - .field("out_expenditures", total_out_expenditures) - ; + total_earnings.to_json(total); total.end_object(); + top.end_object(); return out; } @@ -502,7 +573,11 @@ class EarningsTracker::Impl { SUM(in_earnings) AS total_in_earnings, SUM(in_expenditures) AS total_in_expenditures, SUM(out_earnings) AS total_out_earnings, - SUM(out_expenditures) AS total_out_expenditures + SUM(out_expenditures) AS total_out_expenditures, + SUM(in_forwarded) AS total_in_forwarded, + SUM(in_rebalanced) AS total_in_rebalanced, + SUM(out_forwarded) AS total_out_forwarded, + SUM(out_rebalanced) AS total_out_rebalanced FROM "EarningsTracker" GROUP BY time_bucket ORDER BY time_bucket; @@ -513,7 +588,11 @@ class EarningsTracker::Impl { in_earnings, in_expenditures, out_earnings, - out_expenditures + out_expenditures, + in_forwarded, + in_rebalanced, + out_forwarded, + out_rebalanced FROM "EarningsTracker" WHERE node = :nodeid ORDER BY time_bucket; @@ -527,39 +606,23 @@ class EarningsTracker::Impl { auto out = Json::Out(); auto top = out.start_object(); + EarningsData total_earnings; auto history = top.start_array("history"); - uint64_t total_in_earnings = 0; - uint64_t total_in_expenditures = 0; - uint64_t total_out_earnings = 0; - uint64_t total_out_expenditures = 0; for (auto& r : fetch) { - auto in_earnings = r.get(1); - auto in_expenditures = r.get(2); - auto out_earnings = r.get(3); - auto out_expenditures = r.get(4); + size_t ndx = 0; + auto bucket_time = r.get(ndx++); + auto earnings = EarningsData::from_row(r, ndx); auto sub = history.start_object(); - sub - .field("bucket_time", static_cast(r.get(0))) - .field("in_earnings", in_earnings) - .field("in_expenditures", in_expenditures) - .field("out_earnings", out_earnings) - .field("out_expenditures", out_expenditures) - ; + sub.field("bucket_time", static_cast(bucket_time)); + earnings.to_json(sub); sub.end_object(); - total_in_earnings += in_earnings; - total_in_expenditures += in_expenditures; - total_out_earnings += out_earnings; - total_out_expenditures += out_expenditures; + total_earnings += earnings; } history.end_array(); auto total = top.start_object("total"); - total - .field("in_earnings", total_in_earnings) - .field("in_expenditures", total_in_expenditures) - .field("out_earnings", total_out_earnings) - .field("out_expenditures", total_out_expenditures) - ; + total_earnings.to_json(total); total.end_object(); + top.end_object(); return out; } diff --git a/Boss/Mod/ForwardFeeMonitor.cpp b/Boss/Mod/ForwardFeeMonitor.cpp index d3cb0b6cc..979bcd13d 100644 --- a/Boss/Mod/ForwardFeeMonitor.cpp +++ b/Boss/Mod/ForwardFeeMonitor.cpp @@ -25,6 +25,7 @@ void ForwardFeeMonitor::start() { auto in_scid = Ln::Scid(); auto out_scid = Ln::Scid(); auto fee = Ln::Amount(); + auto amount = Ln::Amount(); auto resolution_time = double(); try { auto payload = n.params["forward_event"]; @@ -46,10 +47,12 @@ void ForwardFeeMonitor::start() { fee = Ln::Amount::object( payload["fee_msat"] ); + amount = Ln::Amount::object( + payload["out_msat"] + ); resolution_time = double(payload["resolved_time"]) - double(payload["received_time"]) ; - } catch (std::runtime_error const& err) { return Boss::log( bus, Error , "ForwardFeeMonitor: Unexpected " @@ -71,11 +74,13 @@ void ForwardFeeMonitor::start() { ).then([ this , fee , resolution_time + , amount ](std::vector nids) { return cont( std::move(nids[0]) , std::move(nids[1]) , fee , resolution_time + , amount ); }); }); @@ -85,6 +90,7 @@ Ev::Io ForwardFeeMonitor::cont( Ln::NodeId in_id , Ln::NodeId out_id , Ln::Amount fee , double resolution_time + , Ln::Amount amount ) { if (!in_id || !out_id) return Ev::lift(); @@ -102,7 +108,8 @@ Ev::Io ForwardFeeMonitor::cont( Ln::NodeId in_id std::move(in_id), std::move(out_id), fee, - resolution_time + resolution_time, + amount })); return act; } diff --git a/Boss/Mod/ForwardFeeMonitor.hpp b/Boss/Mod/ForwardFeeMonitor.hpp index 3c8f47b40..1a9352cb8 100644 --- a/Boss/Mod/ForwardFeeMonitor.hpp +++ b/Boss/Mod/ForwardFeeMonitor.hpp @@ -29,6 +29,7 @@ class ForwardFeeMonitor { , Ln::NodeId out_id , Ln::Amount fee , double resolution_time + , Ln::Amount amount ); public: diff --git a/Boss/Msg/ForwardFee.hpp b/Boss/Msg/ForwardFee.hpp index eb49220f6..3544a9754 100644 --- a/Boss/Msg/ForwardFee.hpp +++ b/Boss/Msg/ForwardFee.hpp @@ -21,6 +21,8 @@ struct ForwardFee { /* The time, in seconds, it took from us receiving the incoming HTLC * to us receiving the preimage from the outgoing HTLC. */ double resolution_time; + /* The amount forwarded. */ + Ln::Amount amount; }; }} diff --git a/Boss/Msg/ResponseEarningsInfo.hpp b/Boss/Msg/ResponseEarningsInfo.hpp index 532ceb2e1..755b3f7d4 100644 --- a/Boss/Msg/ResponseEarningsInfo.hpp +++ b/Boss/Msg/ResponseEarningsInfo.hpp @@ -30,6 +30,11 @@ struct ResponseEarningsInfo { Ln::Amount out_earnings; /* How much we have spent on rebalancing *to* this node. */ Ln::Amount out_expenditures; + + Ln::Amount in_forwarded; // Amount forwarded inward to earn fees + Ln::Amount in_rebalanced; // Amount inward rebalanced for expenditure + Ln::Amount out_forwarded; // Amount forwarded outward to earn fees + Ln::Amount out_rebalanced; // Amount outward rebalanced for expenditure }; }} diff --git a/tests/boss/test_earningshistory.cpp b/tests/boss/test_earningshistory.cpp index 6e9ac35b7..816086a4a 100644 --- a/tests/boss/test_earningshistory.cpp +++ b/tests/boss/test_earningshistory.cpp @@ -38,8 +38,9 @@ Ev::Io raiseForwardFeeLoop(S::Bus& bus, int count) { return bus.raise(Boss::Msg::ForwardFee{ A, // in_id B, // out_id - Ln::Amount::sat(1), // fee - 1.0 // resolution_time + Ln::Amount::sat(2), // fee + 1.0, // resolution_time + Ln::Amount::sat(1000), // fee }) .then([&, count]() { mock_now += 60 * 60; @@ -64,7 +65,7 @@ Ev::Io raiseMoveFundsLoop(S::Bus& bus, int count) { Boss::Msg::ResponseMoveFunds{ nullptr, // requester (see RequestMoveFunds) Ln::Amount::sat(1000), // amount_moved - Ln::Amount::sat(2) // fee_spent + Ln::Amount::sat(1) // fee_spent }); }) .then([&bus, count]() { @@ -118,27 +119,39 @@ int main() { assert(result["history"][0] == Jsmn::Object::parse_json(R"JSON( { "bucket_time": 1722902400, - "in_earnings": 24000, + "in_earnings": 48000, "in_expenditures": 0, - "out_earnings": 24000, - "out_expenditures": 0 + "out_earnings": 48000, + "out_expenditures": 0, + "in_forwarded": 24000000, + "in_rebalanced": 0, + "out_forwarded": 24000000, + "out_rebalanced": 0 } )JSON")); assert(result["history"][59] == Jsmn::Object::parse_json(R"JSON( { "bucket_time": 1728000000, - "in_earnings": 24000, - "in_expenditures": 2000, - "out_earnings": 24000, - "out_expenditures": 2000 + "in_earnings": 48000, + "in_expenditures": 1000, + "out_earnings": 48000, + "out_expenditures": 1000, + "in_forwarded": 24000000, + "in_rebalanced": 1000000, + "out_forwarded": 24000000, + "out_rebalanced": 1000000 } )JSON")); assert(result["total"] == Jsmn::Object::parse_json(R"JSON( { - "in_earnings": 1440000, - "in_expenditures": 14000, - "out_earnings": 1440000, - "out_expenditures": 14000 + "in_earnings": 2880000, + "in_expenditures": 7000, + "out_earnings": 2880000, + "out_expenditures": 7000, + "in_forwarded": 1440000000, + "in_rebalanced": 7000000, + "out_forwarded": 1440000000, + "out_rebalanced": 7000000 } )JSON")); return Ev::lift(); @@ -159,27 +172,39 @@ int main() { assert(result["history"][0] == Jsmn::Object::parse_json(R"JSON( { "bucket_time": 1722902400, - "in_earnings": 24000, + "in_earnings": 48000, "in_expenditures": 0, "out_earnings": 0, - "out_expenditures": 0 + "out_expenditures": 0, + "in_forwarded": 24000000, + "in_rebalanced": 0, + "out_forwarded": 0, + "out_rebalanced": 0 } )JSON")); assert(result["history"][59] == Jsmn::Object::parse_json(R"JSON( { "bucket_time": 1728000000, - "in_earnings": 24000, + "in_earnings": 48000, "in_expenditures": 0, "out_earnings": 0, - "out_expenditures": 2000 + "out_expenditures": 1000, + "in_forwarded": 24000000, + "in_rebalanced": 0, + "out_forwarded": 0, + "out_rebalanced": 1000000 } )JSON")); assert(result["total"] == Jsmn::Object::parse_json(R"JSON( { - "in_earnings": 1440000, + "in_earnings": 2880000, "in_expenditures": 0, "out_earnings": 0, - "out_expenditures": 14000 + "out_expenditures": 7000, + "in_forwarded": 1440000000, + "in_rebalanced": 0, + "out_forwarded": 0, + "out_rebalanced": 7000000 } )JSON")); return Ev::lift(); @@ -201,26 +226,38 @@ int main() { { "bucket_time": 1727481600, "in_earnings": 0, - "in_expenditures": 2000, + "in_expenditures": 1000, "out_earnings": 0, - "out_expenditures": 0 + "out_expenditures": 0, + "in_forwarded": 0, + "in_rebalanced": 1000000, + "out_forwarded": 0, + "out_rebalanced": 0 } )JSON")); assert(result["history"][6] == Jsmn::Object::parse_json(R"JSON( { "bucket_time": 1728000000, "in_earnings": 0, - "in_expenditures": 2000, + "in_expenditures": 1000, "out_earnings": 0, - "out_expenditures": 0 + "out_expenditures": 0, + "in_forwarded": 0, + "in_rebalanced": 1000000, + "out_forwarded": 0, + "out_rebalanced": 0 } )JSON")); assert(result["total"] == Jsmn::Object::parse_json(R"JSON( { "in_earnings": 0, - "in_expenditures": 14000, + "in_expenditures": 7000, "out_earnings": 0, - "out_expenditures": 0 + "out_expenditures": 0, + "in_forwarded": 0, + "in_rebalanced": 7000000, + "out_forwarded": 0, + "out_rebalanced": 0 } )JSON")); return Ev::lift(); diff --git a/tests/boss/test_earningstracker.cpp b/tests/boss/test_earningstracker.cpp index fe6cb0270..d0f29c074 100644 --- a/tests/boss/test_earningstracker.cpp +++ b/tests/boss/test_earningstracker.cpp @@ -68,7 +68,8 @@ int main() { A, // in_id B, // out_id Ln::Amount::sat(1), // fee - 1.0 // resolution_time + 1.0, // resolution_time + Ln::Amount::sat(1000), // amount forwarded }); }).then([&]() { mock_now = 2000.0; @@ -84,19 +85,31 @@ int main() { "in_earnings": 1000, "in_expenditures": 0, "out_earnings": 0, - "out_expenditures": 0 + "out_expenditures": 0, + "in_forwarded": 1000000, + "in_rebalanced": 0, + "out_forwarded": 0, + "out_rebalanced": 0 }, "020000000000000000000000000000000000000000000000000000000000000002": { "in_earnings": 0, "in_expenditures": 0, "out_earnings": 1000, - "out_expenditures": 0 + "out_expenditures": 0, + "in_forwarded": 0, + "in_rebalanced": 0, + "out_forwarded": 1000000, + "out_rebalanced": 0 }, "total": { "in_earnings": 1000, "in_expenditures": 0, "out_earnings": 1000, - "out_expenditures": 0 + "out_expenditures": 0, + "in_forwarded": 1000000, + "in_rebalanced": 0, + "out_forwarded": 1000000, + "out_rebalanced": 0 } } )JSON")); @@ -108,7 +121,8 @@ int main() { A, // in_id B, // out_id Ln::Amount::sat(1), // fee - 1.0 // resolution_time + 1.0, // resolution_time + Ln::Amount::sat(2000), // amount forwarded }); }).then([&]() { return bus.raise(Boss::Msg::SolicitStatus{}); @@ -123,19 +137,31 @@ int main() { "in_earnings": 2000, "in_expenditures": 0, "out_earnings": 0, - "out_expenditures": 0 + "out_expenditures": 0, + "in_forwarded": 3000000, + "in_rebalanced": 0, + "out_forwarded": 0, + "out_rebalanced": 0 }, "020000000000000000000000000000000000000000000000000000000000000002": { "in_earnings": 0, "in_expenditures": 0, "out_earnings": 2000, - "out_expenditures": 0 + "out_expenditures": 0, + "in_forwarded": 0, + "in_rebalanced": 0, + "out_forwarded": 3000000, + "out_rebalanced": 0 }, "total": { "in_earnings": 2000, "in_expenditures": 0, "out_earnings": 2000, - "out_expenditures": 0 + "out_expenditures": 0, + "in_forwarded": 3000000, + "in_rebalanced": 0, + "out_forwarded": 3000000, + "out_rebalanced": 0 } } )JSON")); @@ -171,25 +197,41 @@ int main() { "in_earnings": 2000, "in_expenditures": 0, "out_earnings": 0, - "out_expenditures": 2000 + "out_expenditures": 2000, + "in_forwarded": 3000000, + "in_rebalanced": 0, + "out_forwarded": 0, + "out_rebalanced": 1000000 }, "020000000000000000000000000000000000000000000000000000000000000002": { "in_earnings": 0, "in_expenditures": 0, "out_earnings": 2000, - "out_expenditures": 0 + "out_expenditures": 0, + "in_forwarded": 0, + "in_rebalanced": 0, + "out_forwarded": 3000000, + "out_rebalanced": 0 }, "020000000000000000000000000000000000000000000000000000000000000003": { "in_earnings": 0, "in_expenditures": 2000, "out_earnings": 0, - "out_expenditures": 0 + "out_expenditures": 0, + "in_forwarded": 0, + "in_rebalanced": 1000000, + "out_forwarded": 0, + "out_rebalanced": 0 }, "total": { "in_earnings": 2000, "in_expenditures": 2000, "out_earnings": 2000, - "out_expenditures": 2000 + "out_expenditures": 2000, + "in_forwarded": 3000000, + "in_rebalanced": 1000000, + "out_forwarded": 3000000, + "out_rebalanced": 1000000 } } )JSON")); diff --git a/tests/boss/test_forwardfeemonitor.cpp b/tests/boss/test_forwardfeemonitor.cpp index 094e365ca..65d7d1a24 100644 --- a/tests/boss/test_forwardfeemonitor.cpp +++ b/tests/boss/test_forwardfeemonitor.cpp @@ -140,7 +140,7 @@ int main() { assert(forwardfee->out_id == Ln::NodeId("020000000000000000000000000000000000000000000000000000000000000000")); assert(forwardfee->fee == Ln::Amount::msat(1001)); assert(forwardfee->resolution_time == (1560696342.556 - 1560696342.368)); - + assert(forwardfee->amount == Ln::Amount::msat(100000000)); return Ev::lift(0); }); diff --git a/tests/boss/test_recentearnings.cpp b/tests/boss/test_recentearnings.cpp index 3079a9c40..e2be9cf55 100644 --- a/tests/boss/test_recentearnings.cpp +++ b/tests/boss/test_recentearnings.cpp @@ -38,8 +38,9 @@ Ev::Io raiseForwardFeeLoop(S::Bus& bus, int count) { return bus.raise(Boss::Msg::ForwardFee{ A, // in_id B, // out_id - Ln::Amount::sat(1), // fee - 1.0 // resolution_time + Ln::Amount::sat(2), // fee + 1.0, // resolution_time + Ln::Amount::sat(1000), // amount forwarded }) .then([&, count]() { mock_now += 60 * 60; @@ -57,14 +58,14 @@ Ev::Io raiseMoveFundsLoop(S::Bus& bus, int count) { C, // source B, // destination Ln::Amount::sat(1000), // amount - Ln::Amount::sat(3) // fee_budget + Ln::Amount::sat(2) // fee_budget }) .then([&bus]() { return bus.raise( Boss::Msg::ResponseMoveFunds{ nullptr, // requester (see RequestMoveFunds) Ln::Amount::sat(1000), // amount_moved - Ln::Amount::sat(2) // fee_spent + Ln::Amount::sat(1) // fee_spent }); }) .then([&bus, count]() { @@ -128,36 +129,52 @@ int main() { assert(rsp); assert(lastRsp.id == Ln::CommandId::left(req_id)); auto result = Jsmn::Object::parse_json(lastRsp.response.output().c_str()); - assert(result["recent"] == Jsmn::Object::parse_json(R"JSON( + assert(result == Jsmn::Object::parse_json(R"JSON( { - "020000000000000000000000000000000000000000000000000000000000000001": { - "in_earnings": 336000, - "in_expenditures": 0, - "out_earnings": 0, - "out_expenditures": 0 - }, - "020000000000000000000000000000000000000000000000000000000000000002": { - "in_earnings": 0, - "in_expenditures": 0, - "out_earnings": 336000, - "out_expenditures": 14000 + "recent": { + "020000000000000000000000000000000000000000000000000000000000000001": { + "in_earnings": 672000, + "in_expenditures": 0, + "out_earnings": 0, + "out_expenditures": 0, + "in_forwarded": 336000000, + "in_rebalanced": 0, + "out_forwarded": 0, + "out_rebalanced": 0 + }, + "020000000000000000000000000000000000000000000000000000000000000002": { + "in_earnings": 0, + "in_expenditures": 0, + "out_earnings": 672000, + "out_expenditures": 7000, + "in_forwarded": 0, + "in_rebalanced": 0, + "out_forwarded": 336000000, + "out_rebalanced": 7000000 + }, + "020000000000000000000000000000000000000000000000000000000000000003": { + "in_earnings": 0, + "in_expenditures": 7000, + "out_earnings": 0, + "out_expenditures": 0, + "in_forwarded": 0, + "in_rebalanced": 7000000, + "out_forwarded": 0, + "out_rebalanced": 0 + } }, - "020000000000000000000000000000000000000000000000000000000000000003": { - "in_earnings": 0, - "in_expenditures": 14000, - "out_earnings": 0, - "out_expenditures": 0 + "total": { + "in_earnings": 672000, + "in_expenditures": 7000, + "out_earnings": 672000, + "out_expenditures": 7000, + "in_forwarded": 336000000, + "in_rebalanced": 7000000, + "out_forwarded": 336000000, + "out_rebalanced": 7000000 } } )JSON")); - assert(result["total"] == Jsmn::Object::parse_json(R"JSON( - { - "in_earnings": 336000, - "in_expenditures": 14000, - "out_earnings": 336000, - "out_expenditures": 14000 - } - )JSON")); return Ev::lift(); }).then([&]() { // Check a 30 day report @@ -174,36 +191,52 @@ int main() { auto result = Jsmn::Object::parse_json(lastRsp.response.output().c_str()); // The expenditures stay the same because they are entirely in the last // week but the earnings increase. - assert(result["recent"] == Jsmn::Object::parse_json(R"JSON( + assert(result == Jsmn::Object::parse_json(R"JSON( { - "020000000000000000000000000000000000000000000000000000000000000001": { - "in_earnings": 720000, - "in_expenditures": 0, - "out_earnings": 0, - "out_expenditures": 0 - }, - "020000000000000000000000000000000000000000000000000000000000000002": { - "in_earnings": 0, - "in_expenditures": 0, - "out_earnings": 720000, - "out_expenditures": 14000 + "recent": { + "020000000000000000000000000000000000000000000000000000000000000001": { + "in_earnings": 1440000, + "in_expenditures": 0, + "out_earnings": 0, + "out_expenditures": 0, + "in_forwarded": 720000000, + "in_rebalanced": 0, + "out_forwarded": 0, + "out_rebalanced": 0 + }, + "020000000000000000000000000000000000000000000000000000000000000002": { + "in_earnings": 0, + "in_expenditures": 0, + "out_earnings": 1440000, + "out_expenditures": 7000, + "in_forwarded": 0, + "in_rebalanced": 0, + "out_forwarded": 720000000, + "out_rebalanced": 7000000 + }, + "020000000000000000000000000000000000000000000000000000000000000003": { + "in_earnings": 0, + "in_expenditures": 7000, + "out_earnings": 0, + "out_expenditures": 0, + "in_forwarded": 0, + "in_rebalanced": 7000000, + "out_forwarded": 0, + "out_rebalanced": 0 + } }, - "020000000000000000000000000000000000000000000000000000000000000003": { - "in_earnings": 0, - "in_expenditures": 14000, - "out_earnings": 0, - "out_expenditures": 0 + "total": { + "in_earnings": 1440000, + "in_expenditures": 7000, + "out_earnings": 1440000, + "out_expenditures": 7000, + "in_forwarded": 720000000, + "in_rebalanced": 7000000, + "out_forwarded": 720000000, + "out_rebalanced": 7000000 } } )JSON")); - assert(result["total"] == Jsmn::Object::parse_json(R"JSON( - { - "in_earnings": 720000, - "in_expenditures": 14000, - "out_earnings": 720000, - "out_expenditures": 14000 - } - )JSON")); return Ev::lift(); }).then([&]() { return Ev::lift(0); From 2124690b0feee3c13b26fdc01cb0a0384b2fb4f2 Mon Sep 17 00:00:00 2001 From: Ken Sedgwick Date: Thu, 15 Aug 2024 18:07:56 -0700 Subject: [PATCH 14/18] f: fix migration from pre-timebucket to latest, add rollback --- Boss/Mod/EarningsTracker.cpp | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/Boss/Mod/EarningsTracker.cpp b/Boss/Mod/EarningsTracker.cpp index d12b28576..53edea9a2 100644 --- a/Boss/Mod/EarningsTracker.cpp +++ b/Boss/Mod/EarningsTracker.cpp @@ -213,6 +213,9 @@ class EarningsTracker::Impl { Ev::Io init() { return db.transact().then([](Sqlite3::Tx tx) { + // NOTE - we can't just alter table here because we + // are changing the primary key. + // If we already have a bucket schema we're done if (have_bucket_table(tx)) { add_missing_columns(tx); @@ -242,9 +245,11 @@ class EarningsTracker::Impl { tx.query_execute(R"QRY( INSERT INTO EarningsTracker_New ( node, time_bucket, in_earnings, in_expenditures - , out_earnings, out_expenditures) + , out_earnings, out_expenditures, in_forwarded + , in_rebalanced, out_forwarded, out_rebalanced) SELECT node, 0, in_earnings, in_expenditures - , out_earnings, out_expenditures FROM EarningsTracker; + , out_earnings, out_expenditures, 0, 0, 0, 0 + FROM EarningsTracker; DROP TABLE EarningsTracker; )QRY"); } @@ -261,6 +266,30 @@ class EarningsTracker::Impl { tx.commit(); return Ev::lift(); }); + + // These statements revert the schema to before time buckets: + /* + CREATE TABLE IF NOT EXISTS EarningsTracker_Old + ( node TEXT PRIMARY KEY + , in_earnings INTEGER NOT NULL + , in_expenditures INTEGER NOT NULL + , out_earnings INTEGER NOT NULL + , out_expenditures INTEGER NOT NULL + ); + + INSERT INTO EarningsTracker_Old + ( node, in_earnings, in_expenditures + , out_earnings, out_expenditures) + SELECT node + , SUM(in_earnings), SUM(in_expenditures) + , SUM(out_earnings), SUM(out_expenditures) + FROM EarningsTracker + GROUP BY node; + + DROP TABLE EarningsTracker; + + ALTER TABLE EarningsTracker_Old RENAME TO EarningsTracker; + */ } static bool have_bucket_table(Sqlite3::Tx& tx) { From 09762ddd796187d4c253798ba0db7499e1a0e4f5 Mon Sep 17 00:00:00 2001 From: Ken Sedgwick Date: Thu, 15 Aug 2024 22:08:56 -0700 Subject: [PATCH 15/18] f: reorder fields to put relevant adjacent --- Boss/Mod/EarningsTracker.cpp | 54 ++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/Boss/Mod/EarningsTracker.cpp b/Boss/Mod/EarningsTracker.cpp index 53edea9a2..4d873e67c 100644 --- a/Boss/Mod/EarningsTracker.cpp +++ b/Boss/Mod/EarningsTracker.cpp @@ -28,12 +28,12 @@ namespace { struct EarningsData { uint64_t in_earnings = 0; - uint64_t in_expenditures = 0; - uint64_t out_earnings = 0; - uint64_t out_expenditures = 0; uint64_t in_forwarded = 0; + uint64_t in_expenditures = 0; uint64_t in_rebalanced = 0; + uint64_t out_earnings = 0; uint64_t out_forwarded = 0; + uint64_t out_expenditures = 0; uint64_t out_rebalanced = 0; // Unmarshal from database fetch @@ -53,12 +53,12 @@ struct EarningsData { // Operator+= for accumulation EarningsData& operator+=(const EarningsData& other) { in_earnings += other.in_earnings; - in_expenditures += other.in_expenditures; - out_earnings += other.out_earnings; - out_expenditures += other.out_expenditures; in_forwarded += other.in_forwarded; + in_expenditures += other.in_expenditures; in_rebalanced += other.in_rebalanced; + out_earnings += other.out_earnings; out_forwarded += other.out_forwarded; + out_expenditures += other.out_expenditures; out_rebalanced += other.out_rebalanced; return *this; } @@ -67,12 +67,12 @@ struct EarningsData { void to_json(Json::Detail::Object& obj) const { obj .field("in_earnings", in_earnings) - .field("in_expenditures", in_expenditures) - .field("out_earnings", out_earnings) - .field("out_expenditures", out_expenditures) .field("in_forwarded", in_forwarded) + .field("in_expenditures", in_expenditures) .field("in_rebalanced", in_rebalanced) + .field("out_earnings", out_earnings) .field("out_forwarded", out_forwarded) + .field("out_expenditures", out_expenditures) .field("out_rebalanced", out_rebalanced); } }; @@ -229,12 +229,12 @@ class EarningsTracker::Impl { ( node TEXT NOT NULL , time_bucket REAL NOT NULL , in_earnings INTEGER NOT NULL - , in_expenditures INTEGER NOT NULL - , out_earnings INTEGER NOT NULL - , out_expenditures INTEGER NOT NULL , in_forwarded INTEGER NOT NULL + , in_expenditures INTEGER NOT NULL , in_rebalanced INTEGER NOT NULL + , out_earnings INTEGER NOT NULL , out_forwarded INTEGER NOT NULL + , out_expenditures INTEGER NOT NULL , out_rebalanced INTEGER NOT NULL , PRIMARY KEY (node, time_bucket) ); @@ -472,12 +472,12 @@ class EarningsTracker::Impl { ](Sqlite3::Tx tx) { auto fetch = tx.query(R"QRY( SELECT SUM(in_earnings), - SUM(in_expenditures), - SUM(out_earnings), - SUM(out_expenditures), SUM(in_forwarded), + SUM(in_expenditures), SUM(in_rebalanced), + SUM(out_earnings), SUM(out_forwarded), + SUM(out_expenditures), SUM(out_rebalanced) FROM "EarningsTracker" WHERE node = :node; @@ -512,12 +512,12 @@ class EarningsTracker::Impl { auto fetch = tx.query(R"QRY( SELECT node, SUM(in_earnings) AS total_in_earnings, - SUM(in_expenditures) AS total_in_expenditures, - SUM(out_earnings) AS total_out_earnings, - SUM(out_expenditures) AS total_out_expenditures, SUM(in_forwarded) AS total_in_forwarded, + SUM(in_expenditures) AS total_in_expenditures, SUM(in_rebalanced) AS total_in_rebalanced, + SUM(out_earnings) AS total_out_earnings, SUM(out_forwarded) AS total_out_forwarded, + SUM(out_expenditures) AS total_out_expenditures, SUM(out_rebalanced) AS total_out_rebalanced FROM "EarningsTracker" GROUP BY node; @@ -554,12 +554,12 @@ class EarningsTracker::Impl { auto fetch = tx.query(R"QRY( SELECT node, SUM(in_earnings) AS total_in_earnings, - SUM(in_expenditures) AS total_in_expenditures, - SUM(out_earnings) AS total_out_earnings, - SUM(out_expenditures) AS total_out_expenditures, SUM(in_forwarded) AS total_in_forwarded, + SUM(in_expenditures) AS total_in_expenditures, SUM(in_rebalanced) AS total_in_rebalanced, + SUM(out_earnings) AS total_out_earnings, SUM(out_forwarded) AS total_out_forwarded, + SUM(out_expenditures) AS total_out_expenditures, SUM(out_rebalanced) AS total_out_rebalanced FROM "EarningsTracker" WHERE time_bucket >= :cutoff @@ -600,12 +600,12 @@ class EarningsTracker::Impl { sql = R"QRY( SELECT time_bucket, SUM(in_earnings) AS total_in_earnings, - SUM(in_expenditures) AS total_in_expenditures, - SUM(out_earnings) AS total_out_earnings, - SUM(out_expenditures) AS total_out_expenditures, SUM(in_forwarded) AS total_in_forwarded, + SUM(in_expenditures) AS total_in_expenditures, SUM(in_rebalanced) AS total_in_rebalanced, + SUM(out_earnings) AS total_out_earnings, SUM(out_forwarded) AS total_out_forwarded, + SUM(out_expenditures) AS total_out_expenditures, SUM(out_rebalanced) AS total_out_rebalanced FROM "EarningsTracker" GROUP BY time_bucket @@ -615,12 +615,12 @@ class EarningsTracker::Impl { sql = R"QRY( SELECT time_bucket, in_earnings, - in_expenditures, - out_earnings, - out_expenditures, in_forwarded, + in_expenditures, in_rebalanced, + out_earnings, out_forwarded, + out_expenditures, out_rebalanced FROM "EarningsTracker" WHERE node = :nodeid From 01f3eb59365df69b2a40c31f3fd0fdf27bcb4089 Mon Sep 17 00:00:00 2001 From: Ken Sedgwick Date: Thu, 15 Aug 2024 23:23:20 -0700 Subject: [PATCH 16/18] tests: Increase jsmn/test_performance timeout --- tests/jsmn/test_performance.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/jsmn/test_performance.cpp b/tests/jsmn/test_performance.cpp index b05865b0b..c74fee448 100644 --- a/tests/jsmn/test_performance.cpp +++ b/tests/jsmn/test_performance.cpp @@ -36,7 +36,7 @@ int main() { Jsmn::Parser parser; return parser.feed(sample_text); }); - return waiter.timed(5.0, std::move(act)); + return waiter.timed(10.0, std::move(act)); }).then([&](std::vector result) { assert(result.size() == 1); assert(result[0].is_array()); From 99b9f07e1c642074e1417fa27c09e038f77fa652 Mon Sep 17 00:00:00 2001 From: Ken Sedgwick Date: Thu, 15 Aug 2024 23:53:10 -0700 Subject: [PATCH 17/18] Improve commit_hash.h dependencies and generation The previous setup was not regenerating commit_hash.h correctly when the dev tree was modified by git operations. --- Makefile.am | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Makefile.am b/Makefile.am index 1ad6540e5..6d4543204 100644 --- a/Makefile.am +++ b/Makefile.am @@ -676,10 +676,20 @@ create-tarball: create-tarball.in Makefile sed -e 's/[@]PACKAGE_VERSION[@]/$(PACKAGE_VERSION)/' < $(srcdir)/create-tarball.in > create-tarball chmod +x create-tarball +.PHONY: commit_hash.h + BUILT_SOURCES = commit_hash.h -CLEANFILES += commit_hash.h commit_hash.h: - ./generate_commit_hash.sh + @if test -e $(srcdir)/.git/logs/HEAD; then \ + if [ ! -f $@ ] || [ $(srcdir)/.git/logs/HEAD -nt $@ ]; then \ + echo "Regenerating commit_hash.h..."; \ + $(SHELL) $(srcdir)/generate_commit_hash.sh; \ + else \ + echo "commit_hash.h is up-to-date."; \ + fi \ + else \ + echo "Using existing commit_hash.h"; \ + fi clboss libclboss.la: $(BUILT_SOURCES) From b91d4f50a5f4589039cfe1a177a7567e95d9fcbd Mon Sep 17 00:00:00 2001 From: Ken Sedgwick Date: Sat, 17 Aug 2024 10:34:32 -0700 Subject: [PATCH 18/18] contrib: add clboss-earnings-history and clboss-recent-earnings scripts --- contrib/clboss-earnings-history | 115 +++++++++++++++++++++ contrib/clboss-recent-earnings | 173 ++++++++++++++++++++++++++++++++ 2 files changed, 288 insertions(+) create mode 100755 contrib/clboss-earnings-history create mode 100755 contrib/clboss-recent-earnings diff --git a/contrib/clboss-earnings-history b/contrib/clboss-earnings-history new file mode 100755 index 000000000..ff382abe9 --- /dev/null +++ b/contrib/clboss-earnings-history @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 + +import subprocess +import argparse +import json +from datetime import datetime +from tabulate import tabulate + +def run_lightning_cli_command(network_option, command, *args): + try: + result = subprocess.run(['lightning-cli', network_option, command, *args], capture_output=True, text=True, check=True) + return json.loads(result.stdout) + except subprocess.CalledProcessError as e: + print(f"Command '{command}' failed with error: {e}") + except json.JSONDecodeError as e: + print(f"Failed to parse JSON from command '{command}': {e}") + return None + +def format_bucket_time(bucket_time): + if bucket_time == 0: + return "Legacy" + else: + return datetime.utcfromtimestamp(bucket_time).strftime('%Y-%m-%d') + +def main(): + parser = argparse.ArgumentParser(description="Run lightning-cli with specified network") + parser.add_argument('--mainnet', action='store_true', help='Run on mainnet') + parser.add_argument('--testnet', action='store_true', help='Run on testnet') + + args = parser.parse_args() + + if args.testnet: + network_option = '--testnet' + else: + network_option = '--mainnet' # Default to mainnet if no option is specified + + # Replace the API call + earnings_data = run_lightning_cli_command(network_option, 'clboss-earnings-history') + + # Initialize totals + total_forwarded = 0 + total_earnings = 0 + total_rebalanced = 0 + total_expense = 0 + total_net_earnings = 0 + + # Process and format data + rows = [] + for entry in earnings_data['history']: + bucket_time = format_bucket_time(entry['bucket_time']) + earnings = entry['in_earnings'] + forwarded = entry['in_forwarded'] + expense = entry['in_expenditures'] + rebalanced = entry['in_rebalanced'] + + # Calculate rates with checks for division by zero + forwarded_rate = (earnings / forwarded) * 1_000_000 if forwarded != 0 else 0 + rebalance_rate = (expense / rebalanced) * 1_000_000 if rebalanced != 0 else 0 + net_earnings = earnings - expense + + # Update totals + total_forwarded += forwarded + total_earnings += earnings + total_rebalanced += rebalanced + total_expense += expense + total_net_earnings += net_earnings + + rows.append([ + bucket_time, + f"{forwarded:,}".replace(',', '_'), + f"{forwarded_rate:,.0f}", + f"{earnings:,}".replace(',', '_'), + f"{rebalanced:,}".replace(',', '_'), + f"{rebalance_rate:,.0f}", + f"{expense:,}".replace(',', '_'), + f"{int(net_earnings):,}".replace(',', '_') + ]) + + # Calculate total rates + total_forwarded_rate = (total_earnings / total_forwarded) * 1_000_000 if total_forwarded != 0 else 0 + total_rebalance_rate = (total_expense / total_rebalanced) * 1_000_000 if total_rebalanced != 0 else 0 + + # Add a separator row + separator_row = ["-" * len(header) for header in ["Date", "Forwarded", "Rate", "Earnings", "Rebalanced", "Rate", "Expense", "Net Earnings"]] + rows.append(separator_row) + + # Append the total row + rows.append([ + "TOTAL", + f"{total_forwarded:,}".replace(',', '_'), + # misleading because legacy: f"{total_forwarded_rate:,.0f}", + f"", + f"{total_earnings:,}".replace(',', '_'), + f"{total_rebalanced:,}".replace(',', '_'), + # misleading because legacy: f"{total_rebalance_rate:,.0f}", + f"", + f"{total_expense:,}".replace(',', '_'), + f"{int(total_net_earnings):,}".replace(',', '_') + ]) + + headers = [ + "Date", + "Forwarded", + "Rate", + "Earnings", + "Rebalanced", + "Rate", + "Expense", + "Net Earnings" + ] + + print(tabulate(rows, headers=headers, tablefmt="pretty", stralign="right", numalign="right")) + +if __name__ == "__main__": + main() diff --git a/contrib/clboss-recent-earnings b/contrib/clboss-recent-earnings new file mode 100755 index 000000000..76c3babd0 --- /dev/null +++ b/contrib/clboss-recent-earnings @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 + +import subprocess +import argparse +import json +from tabulate import tabulate + +def run_lightning_cli_command(network_option, command, *args): + try: + result = subprocess.run(['lightning-cli', network_option, command, *args], capture_output=True, text=True, check=True) + return json.loads(result.stdout) + except subprocess.CalledProcessError as e: + print(f"Command '{command}' failed with error: {e}") + except json.JSONDecodeError as e: + print(f"Failed to parse JSON from command '{command}': {e}") + return None + +def lookup_alias(network_option, peer_id): + listnodes_data = run_lightning_cli_command(network_option, 'listnodes', peer_id) + if listnodes_data: + nodes = listnodes_data.get("nodes", []) + for node in nodes: + alias = node.get("alias") + if alias: + return alias + return peer_id # Fallback to peer_id if alias is not found + +def calculate_net_earnings(data, network_option): + rows = [] + + # Initialize totals + total_net_earnings = 0 + total_in_earnings = 0 + total_in_forwarded = 0 + total_in_expenditures = 0 + total_in_rebalanced = 0 + total_out_earnings = 0 + total_out_forwarded = 0 + total_out_expenditures = 0 + total_out_rebalanced = 0 + + for node, stats in data['recent'].items(): + in_earnings = stats['in_earnings'] + in_forwarded = stats['in_forwarded'] + in_expenditures = stats['in_expenditures'] + in_rebalanced = stats['in_rebalanced'] + + out_earnings = stats['out_earnings'] + out_forwarded = stats['out_forwarded'] + out_expenditures = stats['out_expenditures'] + out_rebalanced = stats['out_rebalanced'] + + # Skip rows where all values are zero + if ( + in_earnings == 0 and in_forwarded == 0 and in_expenditures == 0 and in_rebalanced == 0 and + out_earnings == 0 and out_forwarded == 0 and out_expenditures == 0 and out_rebalanced == 0 + ): + continue + + alias = lookup_alias(network_option, node) + in_rate = (in_earnings / in_forwarded) * 1_000_000 if in_forwarded != 0 else 0 + in_rebalance_rate = (in_expenditures / in_rebalanced) * 1_000_000 if in_rebalanced != 0 else 0 + out_rate = (out_earnings / out_forwarded) * 1_000_000 if out_forwarded != 0 else 0 + out_rebalance_rate = (out_expenditures / out_rebalanced) * 1_000_000 if out_rebalanced != 0 else 0 + + net_earnings = in_earnings - in_expenditures + out_earnings - out_expenditures + + # Update totals + total_net_earnings += net_earnings + total_in_earnings += in_earnings + total_in_forwarded += in_forwarded + total_in_expenditures += in_expenditures + total_in_rebalanced += in_rebalanced + total_out_earnings += out_earnings + total_out_forwarded += out_forwarded + total_out_expenditures += out_expenditures + total_out_rebalanced += out_rebalanced + + avg_in_earnings_rate = (total_in_earnings / total_in_forwarded) * 1_000_000 if total_in_forwarded != 0 else 0 + avg_out_earnings_rate = (total_out_earnings / total_out_forwarded) * 1_000_000 if total_out_forwarded != 0 else 0 + avg_in_expenditures_rate = (total_in_expenditures / total_in_rebalanced) * 1_000_000 if total_in_rebalanced != 0 else 0 + avg_out_expenditures_rate = (total_out_expenditures / total_out_rebalanced) * 1_000_000 if total_out_rebalanced != 0 else 0 + + rows.append([ + alias, + f"{in_forwarded:,}".replace(',', '_'), + f"{in_rate:,.0f}", + f"{in_earnings:,}".replace(',', '_'), + f"{out_forwarded:,}".replace(',', '_'), + f"{out_rate:,.0f}", + f"{out_earnings:,}".replace(',', '_'), + f"{in_rebalanced:,}".replace(',', '_'), + f"{in_rebalance_rate:,.0f}", + f"{in_expenditures:,}".replace(',', '_'), + f"{out_rebalanced:,}".replace(',', '_'), + f"{out_rebalance_rate:,.0f}", + f"{out_expenditures:,}".replace(',', '_'), + f"{net_earnings:,}".replace(',', '_'), + ]) + + # Divide the net earnings total by 2 + total_net_earnings /= 2 + + # Add a separator row + separator_row = ["-" * len(header) for header in [ + "Alias", + "In Forwarded", + "Rate", + "In Earn", + "Out Forwarded", + "Rate", + "Out Earn", + "In Rebal", + "Rate", + "In Exp", + "Out Rebal", + "Rate", + "Out Exp", + "Net Earn", + ]] + rows.append(separator_row) + + # Append the total row + rows.append([ + "TOTAL", + f"{total_in_forwarded:,}".replace(',', '_'), + f"{avg_in_earnings_rate:,.0f}", + f"{total_in_earnings:,}".replace(',', '_'), + f"{total_out_forwarded:,}".replace(',', '_'), + f"{avg_out_earnings_rate:,.0f}", + f"{total_out_earnings:,}".replace(',', '_'), + f"{total_in_rebalanced:,}".replace(',', '_'), + f"{avg_in_expenditures_rate:,.0f}", + f"{total_in_expenditures:,}".replace(',', '_'), + f"{total_out_rebalanced:,}".replace(',', '_'), + f"{avg_out_expenditures_rate:,.0f}", + f"{total_out_expenditures:,}".replace(',', '_'), + f"{int(total_net_earnings):,}".replace(',', '_'), + ]) + + return rows + +def main(): + parser = argparse.ArgumentParser(description="Run lightning-cli with specified network") + parser.add_argument('recent_earnings_arg', type=int, help='Argument to pass to clboss-recent-earnings') + + args = parser.parse_args() + + network_option = '--mainnet' # Default to mainnet + + earnings_data = run_lightning_cli_command(network_option, 'clboss-recent-earnings', str(args.recent_earnings_arg)) + + if earnings_data: + rows = calculate_net_earnings(earnings_data, network_option) + print(tabulate(rows, headers=[ + "Alias", + "In Forwarded", + "Rate", + "In Earn", + "Out Forwarded", + "Rate", + "Out Earn", + "In Rebal", + "Rate", + "In Exp", + "Out Rebal", + "Rate", + "Out Exp", + "Net Earn", + ],tablefmt="pretty", stralign="right", numalign="right")) + +if __name__ == "__main__": + main()