diff --git a/CHANGELOG.md b/CHANGELOG.md index f25ae98..14e1717 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,15 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/). -## [Unreleased](https://github.com/trailofbits/manticore/compare/0.2.2...HEAD) +## [Unreleased](https://github.com/trailofbits/etheno/compare/v0.2.0...HEAD) + +### 0.2.1 — 2019-02-07 + +Bugfix release. + +- Manticore is now an optional requirement +- Improvements and bugfixes to the logger integration with Manticore +- Added a workaround to the examples for a bug in Truffle ## 0.2.0 — 2018-11-02 diff --git a/Dockerfile b/Dockerfile index 019768a..a311bab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,7 +53,7 @@ COPY etheno/*.py /home/etheno/etheno/etheno/ RUN mkdir -p /home/etheno/examples COPY examples /home/etheno/examples/ -RUN cd etheno && pip3 install --user . +RUN cd etheno && pip3 install --user '.[manticore]' USER root diff --git a/etheno/__main__.py b/etheno/__main__.py index f26ee10..ea34e21 100644 --- a/etheno/__main__.py +++ b/etheno/__main__.py @@ -8,7 +8,7 @@ from .client import RpcProxyClient from .differentials import DifferentialTester from .echidna import echidna_exists, EchidnaPlugin, install_echidna -from .etheno import app, EthenoView, GETH_DEFAULT_RPC_PORT, ManticoreClient, ETHENO, VERSION_NAME +from .etheno import app, EthenoView, GETH_DEFAULT_RPC_PORT, ETHENO, VERSION_NAME from .genesis import Account, make_accounts, make_genesis from .synchronization import AddressSynchronizingClient, RawTransactionClient from .utils import clear_directory, decode_value, find_open_port, format_hex_address, ynprompt @@ -16,10 +16,16 @@ from . import ganache from . import geth from . import logger -from . import manticoreutils from . import parity from . import truffle +try: + from .manticoreclient import ManticoreClient + from . import manticoreutils + MANTICORE_INSTALLED = True +except ModuleNotFoundError: + MANTICORE_INSTALLED = False + def main(argv = None): parser = argparse.ArgumentParser(description='An Ethereum JSON RPC multiplexer and Manticore wrapper') parser.add_argument('--debug', action='store_true', default=False, help='Enable debugging from within the web server') @@ -238,12 +244,19 @@ def main(argv = None): manticore_client = None if args.manticore: + if not MANTICORE_INSTALLED: + ETHENO.logger.error('Manticore is not installed! Running Etheno with Manticore requires Manticore version 0.2.2 or newer. Reinstall Etheno with Manticore support by running `pip3 install --user \'etheno[manticore]\'`, or install Manticore separately with `pip3 install --user \'manticore\'`') + sys.exit(1) + new_enough = manticoreutils.manticore_is_new_enough() + if new_enough is None: + ETHENO.logger.warning(f"Unknown Manticore version {manticoreutils.manticore_version()}; it may not be new enough to have Etheno support!") + elif not new_enough: + ETHENO.logger.error(f"The version of Manticore installed is {manticoreutils.manticore_version()}, but the minimum required version with Etheno support is 0.2.2. We will try to proceed, but things might not work correctly! Please upgrade Manticore.") manticore_client = ManticoreClient() ETHENO.add_client(manticore_client) if args.manticore_max_depth is not None: manticore_client.manticore.register_detector(manticoreutils.StopAtDepth(args.manticore_max_depth)) manticore_client.manticore.verbosity(getattr(logger, args.log_level)) - manticore_client.reassign_manticore_loggers() if args.truffle: truffle_controller = truffle.Truffle(parent_logger=ETHENO.logger) diff --git a/etheno/etheno.py b/etheno/etheno.py index 041d8f4..4fc0f61 100644 --- a/etheno/etheno.py +++ b/etheno/etheno.py @@ -4,19 +4,14 @@ VERSION_ID=67 import logging -import sha3 from threading import Thread -import time from flask import Flask, g, jsonify, request, abort from flask.views import MethodView -from manticore.ethereum import ManticoreEVM -import manticore - from . import logger from . import threadwrapper -from .client import EthenoClient, JSONRPCError, RpcProxyClient, SelfPostingClient, DATA, QUANTITY, transaction_receipt_succeeded, jsonrpc +from .client import JSONRPCError, SelfPostingClient from .utils import format_hex_address app = Flask(__name__) @@ -30,17 +25,6 @@ def to_account_address(raw_address): addr = "%x" % raw_address return "0x%s%s" % ('0'*(40 - len(addr)), addr) -def encode_hex(data): - if data is None: - return None - elif isinstance(data, int) or isinstance(data, long): - encoded = hex(data) - if encoded[-1] == 'L': - encoded = encoded[:-1] - return encoded - else: - return "0x%s" % data.encode('hex') - _CONTROLLER = threadwrapper.MainThreadController() @app.route('/shutdown') @@ -53,124 +37,6 @@ def _etheno_shutdown(): shutdown() return '' -class ManticoreClient(EthenoClient): - def __init__(self, manticore=None): - self._assigned_manticore = manticore - self._manticore = None - self.contracts = [] - self.short_name = 'Manticore' - self._accounts_to_create = [] - - @property - def manticore(self): - if self._manticore is None: - if self._assigned_manticore is None: - # we do lazy evaluation of ManticoreClient.manticore so self.log_directory will be assigned already - if self.log_directory is None: - workspace = None - else: - workspace = self.log_directory - self._assigned_manticore = ManticoreEVM(workspace_url=workspace) - self._manticore = threadwrapper.MainThreadWrapper(self._assigned_manticore, _CONTROLLER) - self._finalize_manticore() - return self._manticore - - def _finalize_manticore(self): - if not self._manticore: - return - for balance, address in self._accounts_to_create: - self._manticore.create_account(balance=balance, address=address) - self._accounts_to_create = [] - self.logger.cleanup_empty = True - - def create_account(self, balance, address): - self._accounts_to_create.append((balance, address)) - self._finalize_manticore() - - def reassign_manticore_loggers(self): - # Manticore uses a global to track its loggers: - for name in manticore.utils.log.all_loggers: - manticore_logger = logging.getLogger(name) - for handler in list(manticore_logger.handlers): - manticore_logger.removeHandler(handler) - logger.EthenoLogger(name, parent=self.logger, cleanup_empty=True) - - @jsonrpc(from_addr = QUANTITY, to = QUANTITY, gas = QUANTITY, gasPrice = QUANTITY, value = QUANTITY, data = DATA, nonce = QUANTITY, RETURN = DATA) - def eth_sendTransaction(self, from_addr, to = None, gas = 90000, gasPrice = None, value = 0, data = None, nonce = None, rpc_client_result = None): - if to is None or to == 0: - # we are creating a new contract - if rpc_client_result is not None: - tx_hash = rpc_client_result['result'] - while True: - receipt = self.etheno.master_client.post({ - 'id' : "%s_receipt" % rpc_client_result['id'], - 'method' : 'eth_getTransactionReceipt', - 'params' : [tx_hash] - }) - if 'result' in receipt and receipt['result']: - address = int(receipt['result']['contractAddress'], 16) - break - # The transaction is still pending - time.sleep(1.0) - else: - address = None - contract_address = self.manticore.create_contract(owner = from_addr, balance = value, init=data) - self.contracts.append(contract_address) - self.logger.info(f"Manticore contract created: {encode_hex(contract_address.address)}") - #self.logger.info("Block number: %s" % self.manticore.world.block_number()) - else: - self.manticore.transaction(address = to, data = data, caller=from_addr, value = value) - # Just mimic the result from the master client - # We need to return something valid to appease the differential tester - return rpc_client_result - - @jsonrpc(TX_HASH = QUANTITY) - def eth_getTransactionReceipt(self, tx_hash, rpc_client_result = None): - # Mimic the result from the master client - # to appease the differential tester - return rpc_client_result - - def multi_tx_analysis(self, contract_address = None, tx_limit=None, tx_use_coverage=True, args=None): - if contract_address is None: - for contract_address in self.contracts: - self.multi_tx_analysis(contract_address = contract_address, tx_limit = tx_limit, tx_use_coverage = tx_use_coverage, args = args) - return - - tx_account = self.etheno.accounts - - prev_coverage = 0 - current_coverage = 0 - tx_no = 0 - while (current_coverage < 100 or not tx_use_coverage) and not self.manticore.is_shutdown(): - try: - self.logger.info("Starting symbolic transaction: %d" % tx_no) - - # run_symbolic_tx - symbolic_data = self.manticore.make_symbolic_buffer(320) - symbolic_value = self.manticore.make_symbolic_value() - self.manticore.transaction(caller=tx_account[min(tx_no, len(tx_account) - 1)], - address=contract_address, - data=symbolic_data, - value=symbolic_value) - self.logger.info("%d alive states, %d terminated states" % (self.manticore.count_running_states(), self.manticore.count_terminated_states())) - except NoAliveStates: - break - - # Check if the maximun number of tx was reached - if tx_limit is not None and tx_no + 1 >= tx_limit: - break - - # Check if coverage has improved or not - if tx_use_coverage: - prev_coverage = current_coverage - current_coverage = self.manticore.global_coverage(contract_address) - found_new_coverage = prev_coverage < current_coverage - - if not found_new_coverage: - break - - tx_no += 1 - class EthenoPlugin(object): _etheno = None logger = None @@ -299,6 +165,8 @@ def estimate_gas(self, transaction): return None def post(self, data): + self.logger.debug(f"Handling JSON RPC request {data}") + for plugin in self.plugins: plugin.before_post(data) @@ -332,6 +200,7 @@ def post(self, data): ret = e self.rpc_client_result = ret + self.logger.debug(f"Result from the master client ({self.master_client}): {ret}") results = [] @@ -357,6 +226,7 @@ def post(self, data): except JSONRPCError as e: self.logger.error(e) results.append(e) + self.logger.debug(f"Result from client {client}: {results[-1]}") if ret is None: return None @@ -487,6 +357,8 @@ def post(self): ret = ETHENO.post(data) + ETHENO.logger.debug(f"Returning {ret}") + if ret is None: return None diff --git a/etheno/ganache.py b/etheno/ganache.py index 15ef083..4ca1432 100644 --- a/etheno/ganache.py +++ b/etheno/ganache.py @@ -13,9 +13,8 @@ def __init__(self, args=None, port=8546): super().__init__("http://127.0.0.1:%d/" % port) self.port = port if args is None: - self.args = [] - else: - self.args = ['/usr/bin/env', 'ganache-cli', '-d', '-p', str(port)] + args + args = [] + self.args = ['/usr/bin/env', 'ganache-cli', '-d', '-p', str(port)] + args self.ganache = None self._client = None def start(self): diff --git a/etheno/logger.py b/etheno/logger.py index 6d842cd..59d5f63 100644 --- a/etheno/logger.py +++ b/etheno/logger.py @@ -83,21 +83,47 @@ def format(self, *args, **kwargs): else: return self._parent_formatter.format(*args, **kwargs) +ETHENO_LOGGERS = {} + +_LOGGING_GETLOGGER = logging.getLogger +def getLogger(name): + if name in ETHENO_LOGGERS: + # TODO: Only enable this if Etheno was run as a standalone application + ret = ETHENO_LOGGERS[name] + else: + ret = _LOGGING_GETLOGGER(name) + # ####BEGIN#### + # Horrible hack to workaround Manticore's global logging system. + # This can be removed after https://github.com/trailofbits/manticore/issues/1369 + # is resolved. + if name.startswith('manticore'): + ret.propagate = False + # ####END#### + return ret +logging.getLogger = getLogger + class EthenoLogger(object): DEFAULT_FORMAT='$RESET$LEVELCOLOR$BOLD%(levelname)-8s $BLUE[$RESET$WHITE%(asctime)14s$BLUE$BOLD]$NAME$RESET %(message)s' - def __init__(self, name, log_level=None, parent=None, cleanup_empty=False): + def __init__(self, name, log_level=None, parent=None, cleanup_empty=False, displayname=None): + if name in ETHENO_LOGGERS: + raise Exception(f'An EthenoLogger instance for name {name} already exists: {ETHENO_LOGGERS[name]}') + ETHENO_LOGGERS[name] = self self._directory = None self.parent = parent self.cleanup_empty = cleanup_empty self.children = [] self._descendant_handlers = [] + if displayname is None: + self.displayname = name + else: + self.displayname = displayname if log_level is None: if parent is None: raise ValueError('A logger must be provided a parent if `log_level` is None') log_level = parent.log_level self._log_level = log_level - self._logger = logging.getLogger(name) + self._logger = _LOGGING_GETLOGGER(name) self._handlers = [logging.StreamHandler()] if log_level is not None: self.log_level = log_level @@ -136,7 +162,7 @@ def directory(self): return self._directory def _add_child(self, child): - if child in self.children: + if child in self.children or any(c for c in self.children if c.name == child.name): raise ValueError("Cannot double-add child logger %s to logger %s" % (child.name, self.name)) self.children.append(child) if self.directory is not None: @@ -155,7 +181,7 @@ def _name_format(self): ret = self.parent._name_format() else: ret = '' - return ret + "[$RESET$WHITE%s$BLUE$BOLD]" % self._logger.name + return ret + "[$RESET$WHITE%s$BLUE$BOLD]" % self.displayname def addHandler(self, handler, include_descendants=True, set_log_level=True): if set_log_level: @@ -249,6 +275,10 @@ def log_level(self, level): def __getattr__(self, name): return getattr(self._logger, name) + def __repr__(self): + return f'{type(self).__name__}(name={self.name!r}, log_level={self.log_level!r}, parent={self.parent!r}, cleanup_empty={self.cleanup_empty!r}, displayname={self.displayname!r})' + + class StreamLogger(threading.Thread): def __init__(self, logger, *streams, newline_char=b'\n'): super().__init__(daemon=True) diff --git a/etheno/manticoreclient.py b/etheno/manticoreclient.py new file mode 100644 index 0000000..cea87e8 --- /dev/null +++ b/etheno/manticoreclient.py @@ -0,0 +1,178 @@ +import logging +import time + +import builtins +import sys + +# ####BEGIN#### +# Horrible hack to workaround Manticore's global logging system. +# This can be removed after https://github.com/trailofbits/manticore/issues/1369 +# is resolved. +from . import manticorelogger + +oldimport = builtins.__import__ +def manticoreimport(name, *args, **kwargs): + if name == 'manticore.utils.log': + manticorelogger.__name__ = 'manticore.utils.log' + sys.modules[name] = manticorelogger + return manticorelogger + else: + return oldimport(name, *args, **kwargs) + +builtins.__import__ = manticoreimport +try: + import manticore.utils.log + import manticore.utils +finally: + builtins.__import__ = oldimport + +manticore.utils.log = manticorelogger +# ####END#### + +from manticore.ethereum import ManticoreEVM +import manticore + +from . import logger +from . import threadwrapper +from .client import EthenoClient, jsonrpc, DATA, QUANTITY +from .etheno import _CONTROLLER + +def encode_hex(data): + if data is None: + return None + elif isinstance(data, int) or isinstance(data, long): + encoded = hex(data) + if encoded[-1] == 'L': + encoded = encoded[:-1] + return encoded + else: + return "0x%s" % data.encode('hex') + +class ManticoreClient(EthenoClient): + def __init__(self, manticore=None): + self._assigned_manticore = manticore + self._manticore = None + self.contracts = [] + self.short_name = 'Manticore' + self._accounts_to_create = [] + + @property + def manticore(self): + if self._manticore is None: + if self._assigned_manticore is None: + # we do lazy evaluation of ManticoreClient.manticore so self.log_directory will be assigned already + if self.log_directory is None: + workspace = None + else: + workspace = self.log_directory + self._assigned_manticore = ManticoreEVM(workspace_url=workspace) + self._manticore = threadwrapper.MainThreadWrapper(self._assigned_manticore, _CONTROLLER) + self._finalize_manticore() + return self._manticore + + def _finalize_manticore(self): + if not self._manticore: + return + for balance, address in self._accounts_to_create: + self._manticore.create_account(balance=balance, address=address) + self._accounts_to_create = [] + self.reassign_manticore_loggers() + self.logger.cleanup_empty = True + + def create_account(self, balance, address): + self._accounts_to_create.append((balance, address)) + self._finalize_manticore() + + def reassign_manticore_loggers(self): + # Manticore uses a global to track its loggers: + manticore.utils.log.ETHENO_LOGGER = self.logger + manticore_loggers = (name for name in logging.root.manager.loggerDict if name.startswith('manticore')) + logger_parents = {} + for name in sorted(manticore_loggers): + sep = name.rfind('.') + if sep > 0: + path = name[:sep] + parent = logger_parents[path] + displayname = name[len(path)+1:] + else: + parent = self.logger + displayname = name + m_logger = logger.EthenoLogger(name, parent=parent, cleanup_empty=True, displayname=displayname) + m_logger.propagate = False + logger_parents[name] = m_logger + + @jsonrpc(from_addr = QUANTITY, to = QUANTITY, gas = QUANTITY, gasPrice = QUANTITY, value = QUANTITY, data = DATA, nonce = QUANTITY, RETURN = DATA) + def eth_sendTransaction(self, from_addr, to = None, gas = 90000, gasPrice = None, value = 0, data = None, nonce = None, rpc_client_result = None): + if to is None or to == 0: + # we are creating a new contract + if rpc_client_result is not None: + tx_hash = rpc_client_result['result'] + while True: + receipt = self.etheno.master_client.post({ + 'id' : "%s_receipt" % rpc_client_result['id'], + 'method' : 'eth_getTransactionReceipt', + 'params' : [tx_hash] + }) + if 'result' in receipt and receipt['result']: + address = int(receipt['result']['contractAddress'], 16) + break + # The transaction is still pending + time.sleep(1.0) + else: + address = None + contract_address = self.manticore.create_contract(owner = from_addr, balance = value, init=data) + self.contracts.append(contract_address) + self.logger.info(f"Manticore contract created: {encode_hex(contract_address.address)}") + #self.logger.info("Block number: %s" % self.manticore.world.block_number()) + else: + self.manticore.transaction(address = to, data = data, caller=from_addr, value = value) + # Just mimic the result from the master client + # We need to return something valid to appease the differential tester + return rpc_client_result + + @jsonrpc(TX_HASH = QUANTITY) + def eth_getTransactionReceipt(self, tx_hash, rpc_client_result = None): + # Mimic the result from the master client + # to appease the differential tester + return rpc_client_result + + def multi_tx_analysis(self, contract_address = None, tx_limit=None, tx_use_coverage=True, args=None): + if contract_address is None: + for contract_address in self.contracts: + self.multi_tx_analysis(contract_address = contract_address, tx_limit = tx_limit, tx_use_coverage = tx_use_coverage, args = args) + return + + tx_account = self.etheno.accounts + + prev_coverage = 0 + current_coverage = 0 + tx_no = 0 + while (current_coverage < 100 or not tx_use_coverage) and not self.manticore.is_shutdown(): + try: + self.logger.info("Starting symbolic transaction: %d" % tx_no) + + # run_symbolic_tx + symbolic_data = self.manticore.make_symbolic_buffer(320) + symbolic_value = self.manticore.make_symbolic_value() + self.manticore.transaction(caller=tx_account[min(tx_no, len(tx_account) - 1)], + address=contract_address, + data=symbolic_data, + value=symbolic_value) + self.logger.info("%d alive states, %d terminated states" % (self.manticore.count_running_states(), self.manticore.count_terminated_states())) + except NoAliveStates: + break + + # Check if the maximun number of tx was reached + if tx_limit is not None and tx_no + 1 >= tx_limit: + break + + # Check if coverage has improved or not + if tx_use_coverage: + prev_coverage = current_coverage + current_coverage = self.manticore.global_coverage(contract_address) + found_new_coverage = prev_coverage < current_coverage + + if not found_new_coverage: + break + + tx_no += 1 diff --git a/etheno/manticorelogger.py b/etheno/manticorelogger.py new file mode 100644 index 0000000..2245223 --- /dev/null +++ b/etheno/manticorelogger.py @@ -0,0 +1,26 @@ +# This is a horrible hack that is used to replace manticore.utils.log +# Remove this once https://github.com/trailofbits/manticore/issues/1369 +# is resolved. + +ETHENO_LOGGER = None + +@property +def manticore_verbosity(): + return ETHENO_LOGGER.log_level + +@property +def DEFAULT_LOG_LEVEL(): + return ETHENO_LOGGER.log_level + +def set_verbosity(setting): + pass + #global manticore_verbosity + #manticore_verbosity = min(max(setting, 0), len(get_levels()) - 1) + #for logger_name in all_loggers: + # logger = logging.getLogger(logger_name) + # # min because more verbosity == lower numbers + # # This means if you explicitly call setLevel somewhere else in the source, and it's *more* + # # verbose, it'll stay that way even if manticore_verbosity is 0. + # logger.setLevel(min(get_verbosity(logger_name), logger.getEffectiveLevel())) + +all_loggers = set() diff --git a/etheno/manticoreutils.py b/etheno/manticoreutils.py index 006bb91..7354bad 100644 --- a/etheno/manticoreutils.py +++ b/etheno/manticoreutils.py @@ -1,9 +1,31 @@ import inspect +import itertools +import pkg_resources + +# Import manticoreclient before we load any actual Manticore classes. +# We don't need it here, but we do rely on it to hook in the Manticore loggers: +from . import manticoreclient +del manticoreclient from manticore.core.smtlib.operators import AND from manticore.ethereum import ManticoreEVM, Detector import manticore.ethereum.detectors +def manticore_version(): + return pkg_resources.get_distribution('manticore').version + +def manticore_is_new_enough(): + '''Checks if Manticore is newer than version 0.2.2. Returns True or False if known, or None if uncertain.''' + try: + version = manticore_version() + version = map(int, version.split('.')) + for v, required in itertools.zip_longest(version, (0, 2, 2), fillvalue=0): + if v < required: + return False + except Exception as e: + return None + return True + def get_detectors(): for name, obj in inspect.getmembers(manticore.ethereum.detectors): if inspect.isclass(obj) and issubclass(obj, manticore.ethereum.detectors.Detector) and obj != manticore.ethereum.detectors.Detector: diff --git a/setup.py b/setup.py index 7b8dc1b..67d745d 100644 --- a/setup.py +++ b/setup.py @@ -6,11 +6,10 @@ description='Etheno is a JSON RPC multiplexer, Manticore wrapper, differential fuzzer, and test framework integration tool.', url='https://github.com/trailofbits/etheno', author='Trail of Bits', - version='0.2.0', + version='0.2.1', packages=find_packages(), python_requires='>=3.6', install_requires=[ - 'manticore>=0.2.2', 'ptyprocess', 'pysha3>=1.0.2', 'flask>=1.0.2', @@ -19,7 +18,11 @@ # but they should already be satisfied by the `web3` requirement 'cytoolz>=0.9.0,<1.0.0', 'pycryptodome>=3.4.7,<4.0.0', + 'setuptools' ], + extras_require={ + 'manticore': ['manticore>=0.2.2'] + }, entry_points={ 'console_scripts': [ 'etheno = etheno.__main__:main'