Skip to content

Commit

Permalink
End to end test; change handshake
Browse files Browse the repository at this point in the history
The handshake is changed to include nick
and connection information.
Then we remove the getpeers request from non-dir
nodes, and instead the dir-nodes forward connection
info to clients whenever they get a privmsg from
another client. This avoids too much messaging spam
coming from the directory.
Also, a new test construction is found in
test/e2e-coinjoin-test.py; here, we automate not only the
setup and start of yieldgenerators in regtest, but
also we start up a JMWalletDaemon instance for the
Taker, who then runs a coinjoin with the docoinjoin
request.
  • Loading branch information
AdamISZ committed Dec 24, 2021
1 parent 7ffbd6d commit 2c395d0
Show file tree
Hide file tree
Showing 10 changed files with 677 additions and 186 deletions.
12 changes: 10 additions & 2 deletions jmclient/jmclient/client_protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -434,7 +434,7 @@ def clientStart(self):
"blockchain_source")
#needed only for channel naming convention
network = jm_single().config.get("BLOCKCHAIN", "network")
irc_configs = get_mchannels()
irc_configs = self.factory.get_mchannels()
#only here because Init message uses this field; not used by makers TODO
minmakers = jm_single().config.getint("POLICY", "minimum_makers")
maker_timeout_sec = jm_single().maker_timeout_sec
Expand Down Expand Up @@ -600,7 +600,7 @@ def clientStart(self):
"blockchain_source")
#needed only for channel naming convention
network = jm_single().config.get("BLOCKCHAIN", "network")
irc_configs = get_mchannels()
irc_configs = self.factory.get_mchannels()
minmakers = jm_single().config.getint("POLICY", "minimum_makers")
maker_timeout_sec = jm_single().maker_timeout_sec

Expand Down Expand Up @@ -793,6 +793,14 @@ def getClient(self):
def buildProtocol(self, addr):
return self.protocol(self, self.client)

def get_mchannels(self):
""" A transparent wrapper that allows override,
so that a script can return a customised set of
message channel configs; currently used for testing
multiple bots on regtest.
"""
return get_mchannels()

def start_reactor(host, port, factory=None, snickerfactory=None,
bip78=False, jm_coinjoin=True, ish=True,
daemon=False, rs=True, gui=False): #pragma: no cover
Expand Down
114 changes: 91 additions & 23 deletions jmclient/jmclient/configure.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
import sys
import subprocess
import atexit
import copy
import shutil
from signal import SIGINT

from configparser import ConfigParser, NoOptionError
Expand Down Expand Up @@ -170,13 +172,49 @@ def jm_single():
# This is a comma separated list (comma can be omitted if only one item).
# Each item has format pubkey@host:port ; all are required. Host can be
# a *.onion address (tor v3 only).
directory-nodes = 0344bc51c0b0cf58ebfee222bdd8ff4855ac3becd7a9c8bff8ff08100771179047@mvm5ffyipzf4tis2g753w7q25evcqmvj6qnnwgr3dkpyukomykzvwuad.onion:9736
directory-nodes = 033e65b76c4a3c0b907fddbc57822dbff1cf7ce48d341b8cfaf11bd324ea2d433d@45ojrjlrl2wh6yrnihlb2kl6k752sonptt2rpuv4shbob7u bxkdcmdqd.onion:9736
# The port via which we receive data from the c-lightning plugin; if not bundled,
# make sure it matches what you set in the jmcl.py plugin:
passthrough-port = 49100
# this is the port serving lightning on your onion service, bundled or otherwise:
lightning-port = 9735
# other IRC servers are commented out by default:
## SERVER 2/4) hackint IRC (Tor, IP)
################################################################################
#[MESSAGING:server2]
#channel = joinmarket-pit
## for traditional IP (default):
#host = irc.hackint.org
#port = 6697
#usessl = true
#socks5 = false
## for Tor (recommended as clearnet alternative):
##host = ncwkrwxpq2ikcngxq3dy2xctuheniggtqeibvgofixpzvrwpa77tozqd.onion
##port = 6667
##usessl = false
##socks5 = true
##socks5_host = localhost
##socks5_port = 9050
## SERVER 3/4) Anarplex IRC (Tor, IP)
################################################################################
#[MESSAGING:server3]
#channel = joinmarket-pit
## for traditional IP (default):
#host = agora.anarplex.net
#port = 14716
#usessl = true
#socks5 = false
## for Tor (recommended as clearnet alternative):
##host = vxecvd6lc4giwtasjhgbrr3eop6pzq6i5rveracktioneunalgqlwfad.onion
##port = 6667
##usessl = false
##socks5 = true
##socks5_host = localhost
##socks5_port = 9050
## SERVER 4/4) ILITA IRC (Tor - disabled by default)
################################################################################
#[MESSAGING:server4]
Expand Down Expand Up @@ -499,9 +537,11 @@ def get_mchannels():
lightning_fields = [("type", str), ("directory-nodes", str),
("passthrough-port", int), ("lightning-rpc", str),
("lightning-hostname", str), ("lightning-port", int),
("clightning-location", str)]
("clightning-location", str), ("regtest-count", str)]

configs = []

# processing the IRC sections:
for section in sections:
if jm_single().config.has_option(section, "type"):
# legacy IRC configs do not have "type" but just
Expand All @@ -520,15 +560,17 @@ def get_mchannels():
server_data["socks5_host"] = jm_single().config.get(section, "socks5_host")
server_data["socks5_port"] = jm_single().config.get(section, "socks5_port")

for option, otype in req_fields:
for option, otype in irc_fields:
val = jm_single().config.get(section, option)
server_data[option] = otype(val)
server_data['btcnet'] = get_network()
configs.append(server_data)

# processing the lightning sections:
for section in sections:
if not jm_single().config.has_option(section, "type") or \
not jm_single().config.get(section, "type").lower() == "ln-onion":
break
continue
ln_data = {}
for option, otype in lightning_fields:
try:
Expand All @@ -540,8 +582,8 @@ def get_mchannels():
# Just to allow a dynamic set of var:
ln_data["section-name"] = section
configs.append(ln_data)
return configs

return configs

def _get_irc_mchannels_old():
fields = [("host", str), ("port", int), ("channel", str), ("usessl", str),
Expand Down Expand Up @@ -736,7 +778,7 @@ def load_program_config(config_path="", bs=None, plugin_services=[],
# Check if a ln-onion message channel was configured:
chans = get_mchannels()
lnchans = [x for x in chans if "type" in x and x["type"] == "ln-onion"]
# only 1; multiple directories will be in the config:
# only 1; multiple directories will be in this config section:
assert len(lnchans) < 2
if lnchans and ln_backend_needed:
lnchanconfig = lnchans[0]
Expand All @@ -745,29 +787,55 @@ def load_program_config(config_path="", bs=None, plugin_services=[],
# are considered not-needed if you are not running the bundled
# copy of c-lightning; you should have read the docs and set it up
# yourself otherwise (see the config comments).
jm_ln_dir = os.path.join(global_singleton.datadir, "lightning")
if not os.path.exists(jm_ln_dir):
os.makedirs(jm_ln_dir)
start_ln(lnchanconfig, jm_ln_dir)
# First, we dynamically update this LN message chan
# config section to include the location on which its
# RPC socket will exist:
if global_singleton.config.get("BLOCKCHAIN", "blockchain_source") == "regtest":
brpc_net = "regtest"
else:
brpc_net = get_network()
if brpc_net == "regtest":
# We're going to create a separate data sub-directory for each lightning
# instance we setup.
start_num_instances, end_num_instances = [int(x) for x in lnchanconfig[
"regtest-count"].split(",")]
# special case for manual regtest testing: if we
jmprint("Running {} regtest instances".format(
end_num_instances - start_num_instances + 1))
# set the non-conflicting data location, then the ports/rpc:
for i in range(start_num_instances, end_num_instances + 1):
jm_ln_dir = os.path.join(global_singleton.datadir,
"lightning-regtest" + str(i))
jm_ln_dir = os.path.abspath(jm_ln_dir)
if not os.path.exists(jm_ln_dir):
os.makedirs(jm_ln_dir)
# we cannot restart Lightning instances with old chain state,
# because c-lightning will complain about time warp:
path_to_lightning_regtest_data = os.path.join(jm_ln_dir, "regtest")
if os.path.exists(path_to_lightning_regtest_data):
shutil.rmtree(path_to_lightning_regtest_data)
temp_config = copy.deepcopy(lnchanconfig)
temp_config["passthrough-port"] = 49100 + i
temp_config["lightning-port"] = 9735 + i
start_ln(temp_config, jm_ln_dir, brpc_net, last_digit=i)
else:
jm_ln_dir = os.path.join(global_singleton.datadir, "lightning")
jm_ln_dir = os.path.abspath(jm_ln_dir)
if not os.path.exists(jm_ln_dir):
os.makedirs(jm_ln_dir)
start_ln(lnchanconfig, jm_ln_dir, brpc_net)
else:
# we still need to set the location of the RPC to instantiate
# our LightningRpc object:
global_singleton.config.set(lnchanconfig["section-name"],
"lightning-rpc", lnchanconfig["clightning-location"])

def start_ln(chaninfo, jm_ln_dir):
def start_ln(chaninfo, jm_ln_dir, brpc_net, last_digit=-1):
""" If a LN message channel is configured with the
# "bundled" setting, we start the packaged/customised
# lightning daemon with the required ports
# in the background at the startup of each script.
"""
# First, we dynamically update this LN message chan
# config section to include the location on which its
# RPC socket will exist:
if global_singleton.config.get("BLOCKCHAIN", "blockchain_source") == "regtest":
brpc_net = "regtest"
else:
brpc_net = get_network()
global_singleton.config.set(chaninfo["section-name"], "lightning-rpc",
os.path.join(jm_ln_dir, brpc_net, "lightning-rpc"))

Expand All @@ -783,10 +851,7 @@ def start_ln(chaninfo, jm_ln_dir):
"--jmport="+passthrough_port, "--lightning-dir=" + jm_ln_dir]
# testing needs static values:
if brpc_net == "regtest":
# just to save an extra var in tests, use the last digit of the
# passthrough port:
last_digit = str(chaninfo["passthrough-port"] % 10)
fixed_key_str = "121212121212121212121212121212121212121212121212121212121212121" + last_digit
fixed_key_str = "121212121212121212121212121212121212121212121212121212121212121" + str(last_digit)
command.append("--dev-force-privkey="+fixed_key_str)
# we need to create c-lightning's own config file, in lightningdir/config.
# This requires the bitcoin rpc config also:
Expand All @@ -803,7 +868,9 @@ def start_ln(chaninfo, jm_ln_dir):
if brpc_net == "regtest":
lnconfiglines += ["addr=127.0.0.1:" + str(ln_serving_port),
"log-level=debug",
"log-file=" + jm_ln_dir + "/log"]
"log-file=" + jm_ln_dir + "/log",
"dev-fast-gossip",
"dev-bitcoind-poll=2"]
else:
lnconfiglines += ["proxy=127.0.0.1:9050",
"bind-addr=127.0.0.1:" + str(ln_serving_port),
Expand All @@ -813,6 +880,7 @@ def start_ln(chaninfo, jm_ln_dir):
f.write("\n".join(lnconfiglines))

FNULL = open(os.devnull, 'w')
print("starting ln with command: ", command)
ln_subprocess = subprocess.Popen(command, stdout=FNULL,
stderr=subprocess.STDOUT, close_fds=True)
def gracefully_kill_subprocess(p):
Expand Down
10 changes: 7 additions & 3 deletions jmclient/jmclient/wallet_rpc.py
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,9 @@ def __init__(self, port, wss_port, tls=True):
# ensure shut down does not leave dangling services:
atexit.register(self.stopService)

def get_client_factory(self):
return JMClientProtocolFactory(self.taker)

def activate_coinjoin_state(self, state):
""" To be set when a maker or taker
operation is initialized; they cannot
Expand Down Expand Up @@ -371,7 +374,8 @@ def dummy_restart_callback(msg):
walletname=self.wallet_name,
token=encoded_token)

def taker_finished(self, res, fromtx=False, waittime=0.0, txdetails=None):
def taker_finished(self, res, fromtx=False,
waittime=0.0, txdetails=None):
# This is a slimmed down version compared with what is seen in
# the CLI code, since that code encompasses schedules with multiple
# entries; for now, the RPC only supports single joins.
Expand Down Expand Up @@ -815,13 +819,13 @@ def dummy_user_callback(rel, abs):
self.taker = Taker(self.wallet_service, schedule,
max_cj_fee = max_cj_fee,
callbacks=(self.filter_orders_callback,
None, self.taker_finished))
None, self.taker_finished))
# TODO ; this makes use of a pre-existing hack to allow
# selectively disabling the stallMonitor function that checks
# if transactions went through or not; here we want to cleanly
# destroy the Taker after an attempt is made, successful or not.
self.taker.testflag = True
self.clientfactory = JMClientProtocolFactory(self.taker)
self.clientfactory = self.get_client_factory()

dhost, dport = self.check_daemon_ready()

Expand Down
16 changes: 13 additions & 3 deletions jmdaemon/jmdaemon/jmcl.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ def parse_host_port(path: str) -> HostPortInfo:
if eidx == -1:
raise ValueError('Port number missing.')
host = path[0:eidx]
if re.match('\d+\.\d+\.\d+\.\d+$', host): # matches IPv4 address format
if re.match(r'\d+\.\d+\.\d+\.\d+$', host): # matches IPv4 address format
addrtype = AddrType.IPv4
else:
addrtype = AddrType.NAME
Expand Down Expand Up @@ -137,14 +137,24 @@ def connect(self):
logging.info('Connecting to {}:{} (addrtype {}, proxytype {}, proxytarget {})...'.format(
self.url.target.host, self.url.target.port, self.url.target.addrtype,
self.url.proxytype, self.url.proxytarget))
self.sock.connect((self.url.target.host, self.url.target.port))
try:
self.sock.connect((self.url.target.host, self.url.target.port))
except Exception as e:
plugin.log("JMCL failed to connect to backend at host, port: {}, {} "
"with exception: {}".format(self.url.target.host,
self.url.target.port, repr(e)))
return
plugin.log('Connected to JM backend at: {}'.format(self.destination))
plugin.is_connected_to_backend = True

def sendLine(self, msg: bytes) -> None:
# TODO no length check here; should be accepted
# by backend if len(msg) < basic.LineReceiver.MAX_LENGTH
self.sock.sendall(msg + self.delimiter)
try:
self.sock.sendall(msg + self.delimiter)
except Exception as e:
plugin.log("JMCL failed to send message, exception: {}".format(
repr(e)))

backend_line_sender = SocketToBackend()

Expand Down
Loading

0 comments on commit 2c395d0

Please sign in to comment.