feat: external mode of lnd

images/utils/launcher/check_wallets.py

@@ -3,7 +3,7 @@
 import os
 import docker
 import time
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor, wait
 from docker.models.containers import Container
 from datetime import datetime
 import re
@@ -276,7 +276,10 @@ def ensure_layer2_ready(self) -> None:
         nodes = self.config.nodes
         if self.node_manager.newly_installed:
             if self.network == "simnet":
-                self.lnd_cfheaders["bitcoin"] = CFHeaderState()
+                if self.config.nodes["lndbtc"]["mode"] == "native":
+                    self.lnd_cfheaders["bitcoin"] = CFHeaderState()
+                if self.config.nodes["lndltc"]["mode"] == "native":
+                    self.lnd_cfheaders["litecoin"] = CFHeaderState()
                 self.lnd_cfheaders["litecoin"] = CFHeaderState()
             if "bitcoind" in nodes and nodes["bitcoind"]["mode"] in ["neutrino", "light"]:
                 self.lnd_cfheaders["bitcoin"] = CFHeaderState()
@@ -287,19 +290,18 @@ def ensure_layer2_ready(self) -> None:
                 print("Syncing light clients:")
                 self._print_lnd_cfheaders(erase_last_line=False)
 
-        with ThreadPoolExecutor(max_workers=2, thread_name_prefix="LndReady") as executor:
-            f1 = executor.submit(self.ensure_lnd_ready, "bitcoin")
-            f2 = executor.submit(self.ensure_lnd_ready, "litecoin")
+        with ThreadPoolExecutor(max_workers=len(self.lnd_cfheaders), thread_name_prefix="LndReady") as executor:
+            futs = {}
+            for chain in self.lnd_cfheaders:
+                futs[executor.submit(self.ensure_lnd_ready, chain)] = chain
 
-            try:
-                f1.result()
-            except Exception as e:
-                raise FatalError("Failed to wait for lndbtc to be ready") from e
+            done, not_done = wait(futs)
 
-            try:
-                f2.result()
-            except Exception as e:
-                raise FatalError("Failed to wait for lndltc to be ready") from e
+            if len(not_done) > 0:
+                for f in not_done:
+                    f.cancel()
+                lnds = ", ".join([futs[f] for f in not_done])
+                raise FatalError("Failed to wait for {} to be ready".format(lnds))
 
         if self.node_manager.newly_installed:
             print()

images/utils/launcher/config/config.py

@@ -265,6 +265,33 @@ def parse_command_line_arguments(self):
             action="store_true",
             help="Preserve lndbtc lnd.conf file during updates"
         )
+        group.add_argument(
+            "--lndbtc.mode",
+            metavar="<mode>",
+            choices=["native", "external"],
+            help="Lndbtc service mode"
+        )
+        group.add_argument(
+            "--lndbtc.rpc-host",
+            metavar="<hostname>",
+            help="External lndbtc RPC hostname"
+        )
+        group.add_argument(
+            "--lndbtc.rpc-port",
+            type=int,
+            metavar="<port>",
+            help="External lndbtc RPC port"
+        )
+        group.add_argument(
+            "--lndbtc.certpath",
+            metavar="<hostname>",
+            help="External lndbtc TLS certificate file"
+        )
+        group.add_argument(
+            "--lndbtc.macaroonpath",
+            metavar="<hostname>",
+            help="External lndbtc admin.macaroon path"
+        )
 
         group = parser.add_argument_group("lndltc")
         group.add_argument(
@@ -277,6 +304,33 @@ def parse_command_line_arguments(self):
             action="store_true",
             help="Preserve lndltc lnd.conf file during updates"
         )
+        group.add_argument(
+            "--lndltc.mode",
+            metavar="<mode>",
+            choices=["native", "external"],
+            help="Lndltc service mode"
+        )
+        group.add_argument(
+            "--lndltc.rpc-host",
+            metavar="<hostname>",
+            help="External lndltc RPC hostname"
+        )
+        group.add_argument(
+            "--lndltc.rpc-port",
+            type=int,
+            metavar="<port>",
+            help="External lndltc RPC port"
+        )
+        group.add_argument(
+            "--lndltc.certpath",
+            metavar="<hostname>",
+            help="External lndltc TLS certificate file"
+        )
+        group.add_argument(
+            "--lndltc.macaroonpath",
+            metavar="<hostname>",
+            help="External lndltc admin.macaroon path"
+        )
 
         group = parser.add_argument_group("connext")
         group.add_argument(
@@ -665,20 +719,72 @@ def update_geth(self, parsed):
             value = getattr(self.args, opt)
             node["cache"] = value
 
+    def _get_value(self, key, node, parsed, validator=None, converter=None, default=None):
+        result = None
+
+        node_name = node["name"]
+
+        def process(value, hint):
+            if converter:
+                try:
+                    value = converter(value)
+                except Exception as e:
+                    raise ValueError("({}) Invalid value: {}".format(hint, value)) from e
+
+            if validator and not validator(value):
+                raise ValueError(value)
+
+            return value
+
+        if key in parsed:
+            value = parsed[key]
+            value = process(value, "{}.conf > {} > {}".format(self.network, node_name, key))
+            result = value
+
+        opt = "{}.{}".format(node_name, key.replace("-", "_"))
+
+        if hasattr(self.args, opt):
+            value = getattr(self.args, opt)
+            value = process(value, "--{}".format(opt.replace("_", "-")))
+            result = value
+
+        if not result:
+            if default:
+                return default
+            msg = "configuration \"{}.{}\" missing (please specify command-line option \"--{}\" or add \"{}\" in your {}.conf \"{}\" section)".format(
+                node_name, key, opt.replace("_", "-"), key, self.network, node_name)
+            raise ValueError(msg)
+
+        return result
+
     def update_lndbtc(self, parsed):
         """Update lndbtc related configurations from parsed TOML lndbtc section
         :param parsed: Parsed lndbtc TOML section
         """
         node = self.nodes["lndbtc"]
         self.update_ports(node, parsed)
 
+        node["mode"] = self._get_value("mode", node, parsed, validator=lambda v: v in ["native", "external"], default="native")
+        if node["mode"] == "external":
+            node["rpc_host"] = self._get_value("rpc_host", node, parsed)
+            node["rpc_port"] = self._get_value("rpc_port", node, parsed, converter=lambda v: int(v))
+            node["certpath"] = self._get_value("certpath", node, parsed)
+            node["macaroonpath"] = self._get_value("macaroonpath", node, parsed)
+
     def update_lndltc(self, parsed):
         """Update lndltc related configurations from parsed TOML lndltc section
         :param parsed: Parsed lndltc TOML section
         """
         node = self.nodes["lndltc"]
         self.update_ports(node, parsed)
 
+        node["mode"] = self._get_value("mode", node, parsed, validator=lambda v: v in ["native", "external"], default="native")
+        if node["mode"] == "external":
+            node["rpc_host"] = self._get_value("rpc_host", node, parsed)
+            node["rpc_port"] = self._get_value("rpc_port", node, parsed, converter=lambda v: int(v))
+            node["certpath"] = self._get_value("certpath", node, parsed)
+            node["macaroonpath"] = self._get_value("macaroonpath", node, parsed)
+
     def update_connext(self, parsed):
         """Update Connext related configurations from parsed TOML connext section
         :param parsed: Parsed connext TOML section
@@ -975,6 +1081,7 @@ def dump_attr(attr: str) -> None:
                 if isinstance(value, bool):
                     value = str(value).lower()
                 print("{}=\"{}\"".format(key, value), file=f)
+
             dump_attr("branch")
             dump_attr("disable_update")
             dump_attr("external_ip")
@@ -1024,16 +1131,21 @@ def dump_node_attr(node: str, attr: str) -> None:
                 if node in ["bitcoind", "litecoind"]:
                     dump_node_attr(node, "external_rpc_host")
                     dump_node_attr(node, "external_rpc_port")
-                    #dump_node_attr(node, "external_rpc_user")
-                    #dump_node_attr(node, "external_rpc_password")
+                    # dump_node_attr(node, "external_rpc_user")
+                    # dump_node_attr(node, "external_rpc_password")
                     dump_node_attr(node, "external_zmqpubrawblock")
                     dump_node_attr(node, "external_zmqpubrawtx")
                 elif node == "geth":
                     dump_node_attr(node, "external_rpc_host")
                     dump_node_attr(node, "external_rpc_port")
-                    #dump_node_attr(node, "infura_project_id")
-                    #dump_node_attr(node, "infura_project_secret")
+                    # dump_node_attr(node, "infura_project_id")
+                    # dump_node_attr(node, "infura_project_secret")
                     dump_node_attr(node, "cache")
+                elif node in ["lndbtc", "lndltc"]:
+                    dump_node_attr(node, "rpc_host")
+                    dump_node_attr(node, "rpc_port")
+                    dump_node_attr(node, "certpath")
+                    dump_node_attr(node, "macaroonpath")
                 elif node == "arby":
                     dump_node_attr(node, "test_centralized_baseasset_balance")
                     dump_node_attr(node, "test_centralized_quoteasset_balance")
@@ -1044,6 +1156,6 @@ def dump_node_attr(node: str, attr: str) -> None:
                     dump_node_attr(node, "live_cex")
                     dump_node_attr(node, "test_mode")
                     dump_node_attr(node, "cex")
-                    #dump_node_attr(node, "cex_api_key")
-                    #dump_node_attr(node, "cex_api_secret")
+                    # dump_node_attr(node, "cex_api_key")
+                    # dump_node_attr(node, "cex_api_secret")
                     dump_node_attr(node, "margin")

images/utils/launcher/config/mainnet.conf

@@ -104,12 +104,24 @@
 # 10010 - REST port
 #expose-ports = ["9735", "10009", "10010"]
 
+#mode = "external"
+#rpc-host = "127.0.0.1"
+#rpc-port = 10009
+#certpath = "$HOME/.lnd/tls.cert"
+#macaroonpath = "$HOME/.lnd/data/chain/bitcoin/mainnet/admin.macaroon"
+
 [lndltc]
 # 10735 - P2P port
 # 11009 - gRPC port
 # 11010 - REST port
 #expose-ports = ["10735", "11009:10009", "11010:10010"]
 
+#mode = "external"
+#rpc-host = "127.0.0.1"
+#rpc-port = 10009
+#certpath = "$HOME/.lnd/tls.cert"
+#macaroonpath = "$HOME/.lnd/data/chain/litecoin/mainnet/admin.macaroon"
+
 [connext]
 # 5040 - connext API port
 #expose-ports = ["5040"]

images/utils/launcher/config/simnet.conf

@@ -17,12 +17,24 @@
 # 30010 - REST port
 #expose-ports = ["29735", "30009:10009", "30010:10010"]
 
+#mode = "external"
+#rpc-host = "127.0.0.1"
+#rpc-port = 10009
+#certpath = "$HOME/.lnd/tls.cert"
+#macaroonpath = "$HOME/.lnd/data/chain/bitcoin/simnet/admin.macaroon"
+
 [lndltc]
 # 30735 - P2P port
 # 31009 - gRPC port
 # 31010 - REST port
 #expose-ports = ["30735", "31009:10009", "31010:10010"]
 
+#mode = "external"
+#rpc-host = "127.0.0.1"
+#rpc-port = 10009
+#certpath = "$HOME/.lnd/tls.cert"
+#macaroonpath = "$HOME/.lnd/data/chain/litecoin/simnet/admin.macaroon"
+
 [connext]
 # 25040 - connext API port
 #expose-ports = ["25040:5040"]

images/utils/launcher/config/template.py

@@ -87,6 +87,10 @@ def __str__(self):
             "mode": "native",
             "preserve_config": False,
             "use_local_image": False,
+            "rpc_host": None,
+            "rpc_port": None,
+            "certpath": None,
+            "macaroonpath": None,
         },
         "lndltc": {
             "name": "lndltc",
@@ -101,6 +105,10 @@ def __str__(self):
             "mode": "native",
             "preserve_config": False,
             "use_local_image": False,
+            "rpc_host": None,
+            "rpc_port": None,
+            "certpath": None,
+            "macaroonpath": None,
         },
         "connext": {
             "name": "connext",
@@ -282,6 +290,10 @@ def __str__(self):
             "mode": "native",
             "preserve_config": False,
             "use_local_image": False,
+            "rpc_host": None,
+            "rpc_port": None,
+            "certpath": None,
+            "macaroonpath": None,
         },
         "lndltc": {
             "name": "lndltc",
@@ -296,6 +308,10 @@ def __str__(self):
             "mode": "native",
             "preserve_config": False,
             "use_local_image": False,
+            "rpc_host": None,
+            "rpc_port": None,
+            "certpath": None,
+            "macaroonpath": None,
         },
         "connext": {
             "name": "connext",
@@ -500,6 +516,10 @@ def __str__(self):
             "mode": "native",
             "preserve_config": False,
             "use_local_image": False,
+            "rpc_host": None,
+            "rpc_port": None,
+            "certpath": None,
+            "macaroonpath": None,
         },
         "lndltc": {
             "name": "lndltc",
@@ -514,6 +534,10 @@ def __str__(self):
             "mode": "native",
             "preserve_config": False,
             "use_local_image": False,
+            "rpc_host": None,
+            "rpc_port": None,
+            "certpath": None,
+            "macaroonpath": None,
         },
         "connext": {
             "name": "connext",

images/utils/launcher/config/testnet.conf

@@ -104,12 +104,24 @@
 # 20010 - REST port
 #expose-ports = ["19735", "20009:10009", "20010:10010"]
 
+#mode = "external"
+#rpc-host = "127.0.0.1"
+#rpc-port = 10009
+#certpath = "$HOME/.lnd/tls.cert"
+#macaroonpath = "$HOME/.lnd/data/chain/bitcoin/testnet/admin.macaroon"
+
 [lndltc]
 # 20735 - P2P port
 # 21009 - gRPC port
 # 21010 - REST port
 #expose-ports = ["20735", "21009:10009", "21010:10010"]
 
+#mode = "external"
+#rpc-host = "127.0.0.1"
+#rpc-port = 10009
+#certpath = "$HOME/.lnd/tls.cert"
+#macaroonpath = "$HOME/.lnd/data/chain/litecoin/testnet/admin.macaroon"
+
 [connext]
 # 15040 - connext API port
 #expose-ports = ["15040:5040"]

images/utils/launcher/node/base.py

@@ -367,10 +367,7 @@ def ignored(item):
         if old_set != new_set:
             old.diff = old_set - new_set
             new.diff = new_set - old_set
-            if len(new.diff) == 0:
-                return CompareResult(True, "", old, new)
-            else:
-                return CompareResult(False, "Environments are different", old, new)
+            return CompareResult(False, "Environments are different", old, new)
 
         return CompareResult(True, "", old, new)