From b047a9756de8bc4998802a783badc5b1e76a4a1f Mon Sep 17 00:00:00 2001 From: harzo Date: Thu, 26 Nov 2020 10:33:01 +0100 Subject: [PATCH 01/11] Update Dockerfile to use master --- docker-compose.yml | 1 - docker/bitcoind.Dockerfile | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8f4c58fa8..21f555983 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,6 @@ version: '3.5' services: bitcoind: - image: nexus-01.minebest.local:5001/bitcoind build: context: ./docker dockerfile: bitcoind.Dockerfile diff --git a/docker/bitcoind.Dockerfile b/docker/bitcoind.Dockerfile index e2954b20c..e186de075 100644 --- a/docker/bitcoind.Dockerfile +++ b/docker/bitcoind.Dockerfile @@ -20,10 +20,10 @@ RUN add-apt-repository ppa:bitcoin/bitcoin RUN apt-get update && apt-get -y install libdb4.8-dev libdb4.8++-dev WORKDIR /source -RUN curl -L -k -f https://github.com/bitcoinvault/bitcoinvault/archive/dev.zip -o dev.zip -RUN unzip dev.zip -RUN rm dev.zip -WORKDIR /source/bitcoinvault-dev +RUN curl -L -k -f https://github.com/bitcoinvault/bitcoinvault/archive/master.zip -o bitcoinvault.zip +RUN unzip bitcoinvault.zip +RUN rm bitcoinvault.zip +WORKDIR /source/bitcoinvault-master RUN env CFLAGS=-O2 CXXFLAGS=-O2 \ ./autogen.sh RUN ./configure --disable-bench --disable-tests --disable-wallet --with-gui=no From d5f4e4600d45fed9f84651cae224e1780f37ea45 Mon Sep 17 00:00:00 2001 From: harzo Date: Fri, 13 Nov 2020 12:02:29 +0100 Subject: [PATCH 02/11] Update docs to be up to date with Bitcoin Vault --- README.md | 51 + README.rst | 18 - docs/{HOWTO.rst => HOWTO.md} | 325 ++-- docs/{architecture.rst => architecture.md} | 3 - docs/{authors.rst => authors.md} | 9 +- docs/{changelog.rst => changelog.md} | 80 +- docs/conf.py | 2 +- docs/{environment.rst => environment.md} | 239 ++- docs/{features.rst => features.md} | 2 +- docs/index.md | 16 + docs/index.rst | 66 - .../{peer_discovery.rst => peer_discovery.md} | 31 +- ...protocol-basics.rst => protocol-basics.md} | 129 +- docs/protocol-changes.md | 171 +++ docs/protocol-changes.rst | 184 --- docs/protocol-ideas.rst | 283 ---- docs/protocol-methods.md | 1338 +++++++++++++++++ docs/protocol-methods.rst | 1272 ---------------- docs/protocol-removed.md | 419 ++++++ docs/protocol-removed.rst | 382 ----- docs/protocol.rst | 14 - docs/{rpc-interface.rst => rpc-interface.md} | 0 22 files changed, 2381 insertions(+), 2653 deletions(-) create mode 100644 README.md delete mode 100644 README.rst rename docs/{HOWTO.rst => HOWTO.md} (65%) rename docs/{architecture.rst => architecture.md} (90%) rename docs/{authors.rst => authors.md} (64%) rename docs/{changelog.rst => changelog.md} (73%) rename docs/{environment.rst => environment.md} (67%) rename docs/{features.rst => features.md} (98%) create mode 100644 docs/index.md delete mode 100644 docs/index.rst rename docs/{peer_discovery.rst => peer_discovery.md} (83%) rename docs/{protocol-basics.rst => protocol-basics.md} (67%) create mode 100644 docs/protocol-changes.md delete mode 100644 docs/protocol-changes.rst delete mode 100644 docs/protocol-ideas.rst create mode 100644 docs/protocol-methods.md delete mode 100644 docs/protocol-methods.rst create mode 100644 docs/protocol-removed.md delete mode 100644 docs/protocol-removed.rst delete mode 100644 docs/protocol.rst rename docs/{rpc-interface.rst => rpc-interface.md} (100%) diff --git a/README.md b/README.md new file mode 100644 index 000000000..6813b6d02 --- /dev/null +++ b/README.md @@ -0,0 +1,51 @@ +ElectrumX +========= + +A reimplementation of Electrum-Server for a future with bigger blocks. + +The current version is 2.0. + +Source Code +=========== + +The project is hosted on [GitHub](https://github.com/bitcoinvault/electrumx). + +Please submit an issue on the [bug tracker](https://github.com/bitcoinvault/electrumx/issues) +if you have found a bug or have a suggestion to improve the server. + +Authors and License +=================== + +Neil Booth wrote the vast majority of the code; see [Authors](docs/authors.md). + +Bitcoin Vault development team adjusted the code to BTCV coin. + +Python version at least 3.6 is required. + +The code is released under the [MIT Licence](https://github.com/bitcoinvault/electrumx/LICENCE) + +Getting Started +=============== + +See [HOWTO](docs/HOWTO.md). + +There is also an `installer` available that simplifies the +installation on various Linux-based distributions, and a `Dockerfile` +available . + +Documentation +============= + + + [Features](docs/features.md) + + [Changelog](docs/changelog.md) + + [HOWTO](docs/HOWTO.md) + + [Environment](docs/environment.md) + + Protocol + + [Protocol Basics](docs/protocol-basics.md) + + [Protocol Changes](docs/protocol-changes.md) + + [Protocol Methods](docs/protocol-methods.md) + + [Protocol Removals](docs/protocol-removed.md) + + [Peer Discovery](docs/peer_discovery.md) + + [RPC Interface](docs/rpc-interface.md) + + [Architecture](docs/architecture.md) + + [Authors](docs/authors.md) diff --git a/README.rst b/README.rst deleted file mode 100644 index 47b2cbe90..000000000 --- a/README.rst +++ /dev/null @@ -1,18 +0,0 @@ -.. image:: https://travis-ci.org/kyuupichan/electrumx.svg?branch=master - :target: https://travis-ci.org/kyuupichan/electrumx -.. image:: https://coveralls.io/repos/github/kyuupichan/electrumx/badge.svg - :target: https://coveralls.io/github/kyuupichan/electrumx - -=============================================== -ElectrumX - Reimplementation of electrum-server -=============================================== - -For a future network with bigger blocks. - - :Licence: MIT - :Language: Python (>= 3.6) - -Documentation -============= - -See `readthedocs `_. diff --git a/docs/HOWTO.rst b/docs/HOWTO.md similarity index 65% rename from docs/HOWTO.rst rename to docs/HOWTO.md index 7fe2694bc..fc2de65ff 100644 --- a/docs/HOWTO.rst +++ b/docs/HOWTO.md @@ -1,6 +1,3 @@ -.. _HOWTO: - -===== HOWTO ===== @@ -11,37 +8,39 @@ Prerequisites successfully on MacOS and DragonFlyBSD. It won't run out-of-the-box on Windows, but the changes required to make it do so should be small - pull requests are welcome. - -================ ======================== +``` +================ ================================================== Package Notes -================ ======================== +================ ================================================== Python3 ElectrumX uses asyncio. Python version >= 3.6 is - **required**. -`aiohttp`_ Python library for asynchronous HTTP. Version >= + required. +aiohttp Python library for asynchronous HTTP. Version >= 2.0 required. -`pylru`_ Python LRU cache package. +pylru Python LRU cache package. + DB Engine A database engine package is required; two are - supported (see `Database Engine`_ below). -================ ======================== + supported (see `Database Engine` below). +================ ================================================== +``` Some coins need an additional package, typically for their block hash -functions. For example, `x11_hash`_ is required for DASH. +functions. For example, `x11_hash` is required for DASH. -You **must** to be running a non-pruning bitcoin daemon with:: +You **must** to be running a non-pruning bitcoin daemon with: - txindex=1 +`txindex=1` set in its configuration file. If you have an existing installation of bitcoind and have not previously set this you will need to reindex -the blockchain with:: +the blockchain with: - bitcoind -reindex +`bitcoind -reindex` which can take some time. While not a requirement for running ElectrumX, it is intended to be run with supervisor software such as Daniel Bernstein's -`daemontools`_, Gerrit Pape's `runit`_ package or :command:`systemd`. +`daemontools`, Gerrit Pape's `runit` package or :command:`systemd`. These make administration of secure unix servers very easy, and I strongly recommend you install one of these and familiarise yourself with them. The instructions below and sample run scripts assume @@ -67,21 +66,21 @@ was much worse. You will need to install one of: -+ `plyvel `_ for LevelDB -+ `python-rocksdb `_ for RocksDB (`pip3 install python-rocksdb`) -+ `pyrocksdb `_ for an unmaintained version that doesn't work with recent releases of RocksDB ++ `plyvel ` for LevelDB ++ `python-rocksdb ` for RocksDB (`pip3 install python-rocksdb`) ++ `pyrocksdb ` for an unmaintained version that doesn't work with recent releases of RocksDB Running ======= Install the prerequisites above. -Check out the code from Github:: - - git clone https://github.com/kyuupichan/electrumx.git - cd electrumx - -You can install with :file:`setup.py` or run the code from the source +Check out the code from Github: +``` +git clone https://github.com/bitcoinvault/electrumx.git +cd electrumx +``` +You can install with [setup.py](../setup.py) or run the code from the source tree or a copy of it. You should create a standard user account to run the server under; @@ -92,11 +91,11 @@ under one account which I have called ``electrumx``. Next create a directory where the database will be stored and make it writeable by the ``electrumx`` account. I recommend this directory -live on an SSD:: - - mkdir /path/to/db_directory - chown electrumx /path/to/db_directory - +live on an SSD: +``` +mkdir /path/to/db_directory +chown electrumx /path/to/db_directory +``` Process limits -------------- @@ -110,102 +109,103 @@ to at least 2,500. Note that setting the limit in your shell does *NOT* affect ElectrumX unless you are invoking ElectrumX directly from your shell. If you -are using :command:`systemd`, you need to set it in the -:file:`.service` file (see `contrib/systemd/electrumx.service`_). +are using `systemd`, you need to set it in the +`.service` file (see `contrib/systemd/electrumx.service`). Using daemontools ----------------- Next create a daemontools service directory; this only holds symlinks -(see daemontools documentation). The :command:`svscan` program will +(see daemontools documentation). The `svscan` program will ensure the servers in the directory are running by launching a -:command:`supervise` supervisor for the server and another for its -logging process. You can run :command:`svscan` under the *electrumx* +`supervise` supervisor for the server and another for its +logging process. You can run `svscan` under the *electrumx* account if that is the only one involved (server and logger) otherwise it will need to run as root so that the user can be switched to electrumx. -Assuming this directory is called :file:`service`, you would do one -of:: - - mkdir /service # If running svscan as root - mkdir ~/service # As electrumx if running svscan as that a/c - +Assuming this directory is called `service`, you would do one +of: +``` +mkdir /service # If running svscan as root +mkdir ~/service # As electrumx if running svscan as that a/c +``` Next create a directory to hold the scripts that the -:command:`supervise` process spawned by :command:`svscan` will run - -this directory must be readable by the :command:`svscan` process. -Suppose this directory is called :file:`scripts`, you might do:: - - mkdir -p ~/scripts/electrumx - -Then copy the all sample scripts from the ElectrumX source tree there:: - - cp -R /path/to/repo/electrumx/contrib/daemontools ~/scripts/electrumx - -This copies 3 things: the top level server run script, a :file:`log/` -directory with the logger :command:`run` script, an :file:`env/` +`supervise` process spawned by `svscan` will run - +this directory must be readable by the `svscan` process. +Suppose this directory is called `scripts`, you might do: +``` +mkdir -p ~/scripts/electrumx +``` + +Then copy the all sample scripts from the ElectrumX source tree there: +``` +cp -R /path/to/repo/electrumx/contrib/daemontools ~/scripts/electrumx +``` +This copies 3 things: the top level server run script, a `log/` +directory with the logger `run` script, an `env/` directory. -You need to configure the :ref:`environment variables ` -under :file:`env/` to your setup. ElectrumX server currently takes no +You need to configure the `environment variables ` +under `env/` to your setup. ElectrumX server currently takes no command line arguments; all of its configuration is taken from its -environment which is set up according to :file:`env/` directory (see -:manpage:`envdir` man page). Finally you need to change the -:command:`log/run` script to use the directory where you want the logs +environment which is set up according to `env/` directory (see +`envdir` man page). Finally you need to change the +`log/run` script to use the directory where you want the logs to be written by multilog. The directory need not exist as -:command:`multilog` will create it, but its parent directory must +`multilog` will create it, but its parent directory must exist. -Now start the :command:`svscan` process. This will not do much as the -service directory is still empty:: - - svscan ~/service & disown - -svscan is now waiting for services to be added to the directory:: - - cd ~/service - ln -s ~/scripts/electrumx electrumx - +Now start the `svscan` process. This will not do much as the +service directory is still empty: +``` +svscan ~/service & disown +``` +svscan is now waiting for services to be added to the directory: +``` +cd ~/service +ln -s ~/scripts/electrumx electrumx +``` Creating the symlink will kick off the server process almost immediately. -You can see its logs with:: - - tail -F /path/to/log/dir/current | tai64nlocal - +You can see its logs with: +``` +tail -F /path/to/log/dir/current | tai64nlocal +``` Using systemd ------------- This repository contains a sample systemd unit file that you can use to setup ElectrumX with systemd. Simply copy it to -:file:`/etc/systemd/system`:: - - cp contrib/systemd/electrumx.service /etc/systemd/system/ - +`/etc/systemd/system`: +``` +cp contrib/systemd/electrumx.service /etc/systemd/system/ +``` The sample unit file assumes that the repository is located at -:file:`/home/electrumx/electrumx`. If that differs on your system, you +`/home/electrumx/electrumx`. If that differs on your system, you need to change the unit file accordingly. -You need to set a few :ref:`environment variables ` in -:file:`/etc/electrumx.conf`. - -Now you can start ElectrumX using :command:`systemctl`:: - - systemctl start electrumx - -You can use :command:`journalctl` to check the log output:: - - journalctl -u electrumx -f - -Once configured you may want to start ElectrumX at boot:: - - systemctl enable electrumx - -.. Warning:: systemd is aggressive in forcibly shutting down - processes. Depending on your hardware, ElectrumX can need several - minutes to flush cached data to disk during initial sync. You - should set TimeoutStopSec to *at least* 10 mins in your - :file:`.service` file. +You need to set a few environment variables in `/etc/electrumx.conf`. + +Now you can start ElectrumX using `systemctl`: +``` +systemctl start electrumx +``` +You can use `journalctl` to check the log output: +``` +journalctl -u electrumx -f +``` +Once configured you may want to start ElectrumX at boot: +``` +systemctl enable electrumx +``` +**Warning** +> systemd is aggressive in forcibly shutting down +> processes. Depending on your hardware, ElectrumX can need several +> minutes to flush cached data to disk during initial sync. You +> should set TimeoutStopSec to *at least* 10 mins in your +> `.service` file. Installing Python 3.6 under Ubuntu @@ -213,7 +213,7 @@ Installing Python 3.6 under Ubuntu Many Ubuntu distributions have an incompatible Python version baked in. Because of this, it is easier to install Python 3.6. See -`contrib/python3.6/python-3.6.sh`_. +`contrib/python3.6/python-3.6.sh`. Installing on Raspberry Pi 3 @@ -221,9 +221,9 @@ Installing on Raspberry Pi 3 To install on the Raspberry Pi 3 you will need to update to the ``stretch`` distribution. See the full procedure in -`contrib/raspberrypi3/install_electrumx.sh`_. +`contrib/raspberrypi3/install_electrumx.sh`. -See also `contrib/raspberrypi3/run_electrumx.sh`_ for an easy way to +See also `contrib/raspberrypi3/run_electrumx.sh` for an easy way to configure and launch electrumx. @@ -241,13 +241,13 @@ if the daemon is on the same host. It may even be beneficial to have the daemon on a *separate* machine so the machine doing the indexing has its caches and disk I/O tuned to that task only. -The :envvar:`CACHE_MB` environment variable controls the total cache -size ElectrumX uses; see :ref:`here ` for caveats. +The `CACHE_MB` environment variable controls the total cache +size ElectrumX uses; see `here ` for caveats. Here is my experience with the codebase of early 2017 (the current codebase is faster), to given heights and rough wall-time. The period -from heights 363,000 to 378,000 is the most sluggish:: - +from heights 363,000 to 378,000 is the most sluggish: +``` Machine A Machine B 181,000 25m 00s 5m 30s 283,500 1h 00m @@ -256,38 +256,39 @@ from heights 363,000 to 378,000 is the most sluggish:: 386,000 21h 56m 4h 25m 414,200 1d 12h 29m 6h 30m 447,168 2d 13h 20m 9h 47m - +``` *Machine A*: a low-spec 2011 1.6GHz AMD E-350 dual-core fanless CPU, 8GB RAM and a DragonFlyBSD UFS fileystem on an SSD. It requests -blocks over the LAN from a bitcoind on machine B. :envvar:`DB_CACHE` +blocks over the LAN from a bitcoind on machine B. `DB_CACHE` the default of 1,200. LevelDB. *Machine B*: a late 2012 iMac running Sierra 10.12.2, 2.9GHz quad-core Intel i5 CPU with an HDD and 24GB RAM. Running bitcoind on the same -machine. :envvar:`DB_CACHE` set to 1,800. LevelDB. +machine. `DB_CACHE` set to 1,800. LevelDB. For chains other than bitcoin-mainnet sychronization should be much faster. -.. note:: ElectrumX will not serve normal client connections until it - has fully synchronized and caught up with your daemon. - However LocalRPC connections are served at all times. +**Note** +> ElectrumX will not serve normal client connections until it +> has fully synchronized and caught up with your daemon. +> However LocalRPC connections are served at all times. Terminating ElectrumX ===================== The preferred way to terminate the server process is to send it the -``stop`` RPC command:: - - electrumx_rpc stop - +``stop`` RPC command: +``` +electrumx_rpc stop +``` or alternatively on Unix the ``INT`` or ``TERM`` signals. For a daemontools supervised process this can be done by bringing it down -like so:: - - svc -d ~/service/electrumx - +like so: +``` +svc -d ~/service/electrumx +``` ElectrumX will note receipt of the signals in the logs, and ensure the block chain index is flushed to disk before terminating. You should be patient as flushing data to disk can take many minutes. @@ -300,15 +301,15 @@ if the ElectrumX process if forcibly killed or there is loss of power. The worst case should be having to restart indexing from the most recent UTXO flush. -Once the process has terminated, you can start it up again with:: - - svc -u ~/service/electrumx - -You can see the status of a running service with:: - - svstat ~/service/electrumx - -:command:`svscan` can of course handle multiple services +Once the process has terminated, you can start it up again with: +``` +svc -u ~/service/electrumx +``` +You can see the status of a running service with: +``` +svstat ~/service/electrumx +``` +`svscan` can of course handle multiple services simultaneously from the same service directory, such as a testnet or altcoin server. See the man pages of these various commands for more information. @@ -317,19 +318,18 @@ information. Understanding the Logs ====================== -You can see the logs usefully like so:: - - tail -F /path/to/log/dir/current | tai64nlocal - -Here is typical log output on startup:: - - INFO:BlockProcessor:switching current directory to /crucial/server-good +You can see the logs usefully like so: +``` +tail -F /path/to/log/dir/current | tai64nlocal +``` +Here is typical log output on startup: +``` INFO:BlockProcessor:using leveldb for DB backend INFO:BlockProcessor:created new database INFO:BlockProcessor:creating metadata diretcory - INFO:BlockProcessor:software version: ElectrumX 0.10.2 - INFO:BlockProcessor:DB version: 5 - INFO:BlockProcessor:coin: Bitcoin + INFO:BlockProcessor:software version: ElectrumX 2.0 + INFO:BlockProcessor:DB version: 6 + INFO:BlockProcessor:coin: BitcoinVault INFO:BlockProcessor:network: mainnet INFO:BlockProcessor:height: -1 INFO:BlockProcessor:tip: 0000000000000000000000000000000000000000000000000000000000000000 @@ -353,10 +353,10 @@ Here is typical log output on startup:: INFO:BlockProcessor:flush #1 took 18.7s. Height 134,692 txs: 941,963 INFO:BlockProcessor:tx/sec since genesis: 2,399, since last flush: 2,400 INFO:BlockProcessor:sync time: 0d 00h 06m 32s ETA: 1d 13h 03m 42s - +``` Under normal operation these cache stats repeat once or twice a -minute. UTXO flushes can take several minutes and look like this:: - +minute. UTXO flushes can take several minutes and look like this: +``` INFO:BlockProcessor:our height: 378,745 daemon: 447,332 UTXOs 1,013MB hist 184MB INFO:BlockProcessor:our height: 378,787 daemon: 447,332 UTXOs 1,014MB hist 194MB INFO:BlockProcessor:flushed to FS in 0.3s @@ -366,12 +366,11 @@ minute. UTXO flushes can take several minutes and look like this:: INFO:BlockProcessor:tx/sec since genesis: 1,280, since last flush: 359 INFO:BlockProcessor:sync t ime: 0d 19h 01m 06s ETA: 3d 21h 17m 52s INFO:BlockProcessor:our height: 378,812 daemon: 447,334 UTXOs 10MB hist 10MB - +``` The ETA shown is just a rough guide and in the short term can be quite volatile. It tends to be a little optimistic at first; once you get to height 280,000 is should be fairly accurate. -.. _SSL certificates: Creating a self-signed SSL certificate ====================================== @@ -380,15 +379,15 @@ These instructions are based on those of the ``electrum-server`` documentation. To run an SSL server you need to generate a self-signed certificate -using openssl. Alternatively you could not set :envvar:`SSL_PORT` in +using openssl. Alternatively you could not set `SSL_PORT` in the environment and not serve over SSL, but this is not recommended. Use the sample code below to create a self-signed cert with a recommended validity of 5 years. You may supply any information for your sign request to identify your server. They are not currently checked by the client except for the validity date. When asked for a -challenge password just leave it empty and press enter:: - +challenge password just leave it empty and press enter: +``` $ openssl genrsa -out server.key 2048 $ openssl req -new -key server.key -out server.csr ... @@ -399,9 +398,9 @@ challenge password just leave it empty and press enter:: A challenge password []: ... $ openssl x509 -req -days 1825 -in server.csr -signkey server.key -out server.crt - -The :file:`server.crt` file goes in :envvar:`SSL_CERTFILE` and -:file:`server.key` in :envvar:`SSL_KEYFILE` in the server process's +``` +The `server.crt` file goes in `SSL_CERTFILE` and +`server.key` in `SSL_KEYFILE` in the server process's environment. Starting with Electrum 1.9, the client will learn and locally cache @@ -420,21 +419,11 @@ You may choose to run electrumx on a different port than 50001 / 50002. If you choose a privileged port ( < 1024 ) it makes sense to make use of a iptables NAT rule. -An example, which will forward Port 110 to the internal port 50002 follows:: - - iptables -t nat -A PREROUTING -p tcp --dport 110 -j DNAT --to-destination 127.0.0.1:50002 - -You can then set the port as follows and advertise the service externally on the privileged port:: - - REPORT_SSL_PORT=110 - - -.. _`contrib/systemd/electrumx.service`: https://github.com/kyuupichan/electrumx/blob/master/contrib/systemd/electrumx.service -.. _`daemontools`: http://cr.yp.to/daemontools.html -.. _`runit`: http://smarden.org/runit/index.html -.. _`aiohttp`: https://pypi.python.org/pypi/aiohttp -.. _`pylru`: https://pypi.python.org/pypi/pylru -.. _`x11_hash`: https://pypi.python.org/pypi/x11_hash -.. _`contrib/python3.6/python-3.6.sh`: https://github.com/kyuupichan/electrumx/blob/master/contrib/python3.6/python-3.6.sh -.. _`contrib/raspberrypi3/install_electrumx.sh`: https://github.com/kyuupichan/electrumx/blob/master/contrib/raspberrypi3/install_electrumx.sh -.. _`contrib/raspberrypi3/run_electrumx.sh`: https://github.com/kyuupichan/electrumx/blob/master/contrib/raspberrypi3/run_electrumx.sh +An example, which will forward Port 110 to the internal port 50002 follows: +``` +iptables -t nat -A PREROUTING -p tcp --dport 110 -j DNAT --to-destination 127.0.0.1:50002 +``` +You can then set the port as follows and advertise the service externally on the privileged port: +``` +REPORT_SSL_PORT=110 +``` diff --git a/docs/architecture.rst b/docs/architecture.md similarity index 90% rename from docs/architecture.rst rename to docs/architecture.md index 6ac192d91..81846c300 100644 --- a/docs/architecture.rst +++ b/docs/architecture.md @@ -1,9 +1,6 @@ Architecture ============ -.. image:: https://docs.google.com/drawings/d/1Su_DR2c8__-4phm12hAzV65fL2tNm_1IhKr4XivkW6Q/pub?w=720&h=540 - :target: https://docs.google.com/drawings/d/1Su_DR2c8__-4phm12hAzV65fL2tNm_1IhKr4XivkW6Q/pub?w=960&h=720 - Env --- diff --git a/docs/authors.rst b/docs/authors.md similarity index 64% rename from docs/authors.rst rename to docs/authors.md index ee20e392f..d63ec7488 100644 --- a/docs/authors.rst +++ b/docs/authors.md @@ -1,7 +1,10 @@ -.. _Authors: +BTCV Authors +========== -Authors -======= +* BTCV Team + +Original Authors +================ * Neil Booth diff --git a/docs/changelog.rst b/docs/changelog.md similarity index 73% rename from docs/changelog.rst rename to docs/changelog.md index 0e7ccfcce..8a1d1cd99 100644 --- a/docs/changelog.rst +++ b/docs/changelog.md @@ -1,11 +1,11 @@ -=========== ChangeLog =========== -.. note:: It is strongly recommended you upgrade to Python 3.7, which - fixes bugs in asyncio that caused an ever-growing open file count - and memory consumption whilst serving clients. Those problems - should not occur with Python 3.7. +**Note** +> It is strongly recommended you upgrade to Python 3.7, which +> fixes bugs in asyncio that caused an ever-growing open file count +> and memory consumption whilst serving clients. Those problems +> should not occur with Python 3.7. Version 2.0 (18 Aug 2020) ============================ @@ -31,7 +31,7 @@ Version 1.12.0 (13 May 2019) ============================ * require aiorpcX 0.18.1. This introduces websocket support. The environment variables - changed accordingly; see :envvar:`SERVICES` and :envvar:`REPORT_SERVICES`. + changed accordingly; see `SERVICES` and `REPORT_SERVICES`. * work around bug in recent versions of uvloop * aiorpcX upgrade fixes from Shane M * coin additions / updates: BitcoinSV, Bolivarcoin (Jose Luis Estevez), BTC Testnet (ghost43), @@ -42,21 +42,21 @@ Version 1.11.0 (18 Apr 2019) * require aiorpcX 0.15.x * require aiohttp 3.3 or higher; earlier versions had a problematic bug -* add :envvar:`REQUEST_TIMEOUT` and :envvar:`LOG_LEVEL` environment variables +* add `REQUEST_TIMEOUT` and `LOG_LEVEL` environment variables * mark 4 old environment variables obsolete. ElectrumX won't start until they are removed * getinfo local RPC cleaned up and shows more stats * miscellaneous fixes and improvements * more efficient handling of some RPC methods, particularly - :func:`blockchain.transaction.get_merkle` + `blockchain.transaction.get_merkle` * coin additions / updates: BitcoinSV scaling testnet (Roger Taylor), Dash (zebra lucky), -* issues resolved: `#566`_, `#731`_, `#795`_ +* issues resolved: `#566`, `#731`, `#795` Version 1.10.1 (13 Apr 2019) ============================ * introduce per-request costing. See environment variables documentation for new - variables :envvar:`COST_SOFT_LIMIT`, :envvar:`COST_HARD_LIMIT`, :envvar:`REQUEST_SLEEP`, - :envvar:`INITIAL_CONCURRENT`, :envvar:`BANDWIDTH_UNIT_COST`. Sessions are placed in groups + variables `COST_SOFT_LIMIT`, `COST_HARD_LIMIT`, `REQUEST_SLEEP`, + `INITIAL_CONCURRENT`, `BANDWIDTH_UNIT_COST`. Sessions are placed in groups with which they share some of their costs. Prior cost is remembered across reconnects. * require aiorpcX 0.13.5 for better concurrency handling * require clients use protocol 1.4 or higher @@ -79,7 +79,7 @@ Version 1.10.0 (15 Mar 2019) Bitsend (David), Ravencoin (standard-error), Onixcoin (Jose Estevez), SnowGem * coin removals: Gobyte, Moneci (cronos) * minor tweaks by d42 -* issues fixed `#660`_ - unclean shutdowns during initial sync +* issues fixed `#660` - unclean shutdowns during initial sync Version 1.9.5 (08 Feb 2019) =========================== @@ -87,13 +87,13 @@ Version 1.9.5 (08 Feb 2019) * server blacklist logic (ecdsa) * require aiorpcX 0.10.4 * remove dead wallet code -* fix `#727`_ - not listing same peer twice +* fix `#727` - not listing same peer twice Version 1.9.4 (07 Feb 2019) =========================== * require aiorpcX 0.10.3 -* fix `#713`_ +* fix `#713` Version 1.9.3 (05 Feb 2019) =========================== @@ -116,7 +116,7 @@ Version 1.9.2 (03 Feb 2019) Version 1.9.1 (11 Jan 2019) =========================== -* fix `#684`_ +* fix `#684` Version 1.9.0 (10 Jan 2019) =========================== @@ -126,7 +126,7 @@ Version 1.9.0 (10 Jan 2019) BitcoinABC (mblunderburg), Dash (zebra-lucky), BitcoinABCRegtest (ezegom), AXE (slowdive), NOR (flo071), BitcoinPlus (bushsolo), Myriadcoin (cryptapus), Trezarcoin (ChekaZ), Bitcoin Diamond (John Shine), -* close `#554`_, `#653`_, `#655`_ +* close `#554`, `#653`, `#655` * other minor tweaks (Michael Schmoock, Michael Taborsky) @@ -144,13 +144,13 @@ Version 1.8.10 (05 Nov 2018) ============================ * require aiorpcX 0.10.0 -* fix `#632`_ +* fix `#632` * coin additions / updates: ZelCash (TheTrunk) Version 1.8.9 (02 Nov 2018) =========================== -* fix `#630`_ +* fix `#630` Version 1.8.8 (01 Nov 2018) =========================== @@ -158,7 +158,7 @@ Version 1.8.8 (01 Nov 2018) * require aiorpcX 0.9.0 * coin additions / updates: decred (dajohi, bolapara), zcash (erasmospunk), namecoin (JeremyRand),CivX (turcol), NewYorkCoin (erasmospunk) -* fix `#603`_, `#608`_ +* fix `#603`, `#608` * other minor fixes and changes: FMCorz Version 1.8.7 (13 Sep 2018) @@ -179,12 +179,12 @@ Version 1.8.5 (18 Aug 2018) =========================== * require aiorpcX 0.7.3 which contains a couple of bugfixes -* fix `#552`_, `#577`_ +* fix `#552`, `#577` * fixed a session limiting bug reported by ghost43 * coin additions / updates: PIVX and Decred Testnets, BitcoinGreen (cunhasb) Monacoin (wakayamap) * proper generation input handling for various altcoins (erasmospunk) fixing - `#570`_ + `#570` Version 1.8.4 (14 Aug 2018) =========================== @@ -201,20 +201,20 @@ Version 1.8.3 (11 Aug 2018) * separate the DB and the BlockProcessor objects * comprehensive mempool tests -* fix `#521`_, `#565`_, `#567`_ +* fix `#521`, `#565`, `#567` Version 1.8.2 (09 Aug 2018) =========================== * require aiorpcX 0.7.1 which along with an ElectrumX change restores clean shutdown and flush functionality, particularly during initial sync -* fix `#564`_ +* fix `#564` Version 1.8.1 (08 Aug 2018) =========================== * require aiorpcX 0.7.0 which fixes a bug causing silent shutdown of ElectrumX -* fix `#557`_, `#559`_ +* fix `#557`, `#559` * tweaks related to log spew (I think mostly occurring with old versions of Python) @@ -222,8 +222,12 @@ Version 1.8 (06 Aug 2018) ========================== * require aiorpcX 0.6.2 -* fix query.py; move to contrib. Add :ref:`query ` function to RPC -* rewrite :command:`electrumx_rpc` so that proper command-line help is provided +<<<<<<< HEAD:docs/changelog.md +* fix query.py; move to contrib. Add `query ` function to RPC +======= +* fix query.py; move to contrib. Add `query` function to RPC +>>>>>>> WIP:docs/changelog.rst +* rewrite `electrumx_rpc` so that proper command-line help is provided * per-coin tx hash functions (erasmospunk) * coin additions / updates: Groestlcoin (Kefkius, erasmospunk), Decred (erasmonpsunk) @@ -231,27 +235,3 @@ Version 1.8 (06 Aug 2018) **Neil Booth** kyuupichan@gmail.com https://github.com/kyuupichan - -.. _#521: https://github.com/kyuupichan/electrumx/issues/521 -.. _#552: https://github.com/kyuupichan/electrumx/issues/552 -.. _#554: https://github.com/kyuupichan/electrumx/issues/554 -.. _#557: https://github.com/kyuupichan/electrumx/issues/557 -.. _#559: https://github.com/kyuupichan/electrumx/issues/559 -.. _#564: https://github.com/kyuupichan/electrumx/issues/564 -.. _#565: https://github.com/kyuupichan/electrumx/issues/565 -.. _#566: https://github.com/kyuupichan/electrumx/issues/566 -.. _#567: https://github.com/kyuupichan/electrumx/issues/567 -.. _#570: https://github.com/kyuupichan/electrumx/issues/570 -.. _#577: https://github.com/kyuupichan/electrumx/issues/577 -.. _#603: https://github.com/kyuupichan/electrumx/issues/603 -.. _#608: https://github.com/kyuupichan/electrumx/issues/608 -.. _#630: https://github.com/kyuupichan/electrumx/issues/630 -.. _#632: https://github.com/kyuupichan/electrumx/issues/630 -.. _#653: https://github.com/kyuupichan/electrumx/issues/653 -.. _#655: https://github.com/kyuupichan/electrumx/issues/655 -.. _#660: https://github.com/kyuupichan/electrumx/issues/660 -.. _#684: https://github.com/kyuupichan/electrumx/issues/684 -.. _#713: https://github.com/kyuupichan/electrumx/issues/713 -.. _#727: https://github.com/kyuupichan/electrumx/issues/727 -.. _#731: https://github.com/kyuupichan/electrumx/issues/731 -.. _#795: https://github.com/kyuupichan/electrumx/issues/795 diff --git a/docs/conf.py b/docs/conf.py index 9a7fcaa28..99801674d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -48,7 +48,7 @@ # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = '.md' # The master toctree document. master_doc = 'index' diff --git a/docs/environment.rst b/docs/environment.md similarity index 67% rename from docs/environment.rst rename to docs/environment.md index 76cdf4fa6..ac73ae236 100644 --- a/docs/environment.rst +++ b/docs/environment.md @@ -1,6 +1,3 @@ -.. _environment: - -===================== Environment Variables ===================== @@ -9,40 +6,39 @@ environment variables. Only a few are required to be given, the rest will have defaults if not specified. Many of the defaults around resource usage are conservative; I encourage you to review them. -.. note:: set :envvar:`SERVICES` appropriately to be able to connect to your server. For - clients across the internet to know what services you offer you must advertize your - services with :envvar:`REPORT_SERVICES`. - +**Note** +> set `SERVICES` appropriately to be able to connect to your server. For +> clients across the internet to know what services you offer you must advertize your +> services with `REPORT_SERVICES`. Required ======== These environment variables are always required: -.. envvar:: COIN +**COIN** - Must be a :attr:`NAME` from one of the :class:`Coin` classes in - `lib/coins.py`_. +Must be a `NAME` from one of the `Coin` classes in [lib/coins.py](../electrumx/lib/coins.py). -.. envvar:: DB_DIRECTORY +**DB_DIRECTORY** The path to the database directory. Relative paths should be relative to the parent process working directory. This is the directory of the `run` script if you use it. -.. envvar:: DAEMON_URL +**DAEMON_URL** A comma-separated list of daemon URLs. If more than one is provided ElectrumX will initially connect to the first, and failover to subsequent ones round-robin style if one stops working. - The generic form of a daemon URL is:: + The generic form of a daemon URL is: http://username:password@hostname:port/ The leading ``http://`` is optional, as is the trailing slash. The ``:port`` part is also optional and will default to the standard RPC - port for :envvar:`COIN` and :envvar:`NET` if omitted. + port for `COIN` and `NET` if omitted. For the ``run`` script @@ -50,12 +46,12 @@ For the ``run`` script The following are required if you use the ``run`` script: -.. envvar:: ELECTRUMX +**ELECTRUMX** The path to the electrumx_server script. Relative paths should be relative to the directory of the ``run`` script. -.. envvar:: USERNAME +**USERNAME** The username the server will run as. @@ -65,31 +61,32 @@ Services These two environment variables are comma-separated lists of individual *services*. -A **service** has the general form:: +A **service** has the general form: protocol://host:port -*protocol* is case-insensitive. The recognised protocols are:: - +*protocol* is case-insensitive. The recognised protocols are: +``` tcp Plaintext TCP sockets ssl SSL-encrypted TCP sockets ws Plaintext websockets wss SSL-encrypted websockets rpc Plaintext RPC - +``` In a services list, a protocol can be specified multiple times, with different hosts or ports. This might be useful for multi-homed hosts, or if you offer both Tor and clearnet services. *host* can be a hostname, an IPv4 address, or an IPv6 address enclosed in square brackets. -*port* is an integer from :const:`1` to :const:`65535` inclusive. +*port* is an integer from `1` to `65535` inclusive. Where documented, one or more of *protocol*, *host* and *port* can be omitted, in which case a default value will be assumed. -Here are some examples of valid services:: +Here are some examples of valid services: +``` tcp://host.domain.tld:50001 # Hostname, lowercase protocol, port SSL://23.45.67.78:50002 # An IPv4 address, upper-case protocol, port rpC://localhost # Host as a string, mixed-case protocol, default port @@ -98,47 +95,51 @@ Here are some examples of valid services:: rpc://:8000 # Default host, port given host.domain.tld:5151 # Default protocol, hostname, port rpc:// # RPC protocol, default host and port +``` -.. note:: ElectrumX will not serve any incoming connections until it has fully caught up - with your bitcoin daemon. The only exception is local **RPC** connections, - which are served at any time after the server has initialized. +**Note** +> ElectrumX will not serve any incoming connections until it has fully caught up +> with your bitcoin daemon. The only exception is local **RPC** connections, +> which are served at any time after the server has initialized. -.. envvar:: SERVICES +**SERVICES** - A comma-separated list of services ElectrumX will accept incoming connections for. +A comma-separated list of services ElectrumX will accept incoming connections for. - This environment variable determines what interfaces and ports the server listens on, so - must be set correctly for any connection to the server to succeed. If unset or empty, - ElectrumX will not listen for any incoming connections. +This environment variable determines what interfaces and ports the server listens on, so +must be set correctly for any connection to the server to succeed. If unset or empty, +ElectrumX will not listen for any incoming connections. - *protocol* can be any recognised protocol. +*protocol* can be any recognised protocol. - *host* defaults to all of the machine's interfaces, except if the protocol is **rpc**, - when it defaults to :const:`localhost`. +*host* defaults to all of the machine's interfaces, except if the protocol is **rpc**, +when it defaults to `localhost`. - *port* can only be defaulted for **rpc** where the default is :const:`8000`. +*port* can only be defaulted for **rpc** where the default is `8000`. - On most Unix systems ports below 1024 require elevated priveleges so choosing a higher - port is advisable. On Debian for example, this can be achieved by installinng - libcap2-bin package:: +On most Unix systems ports below 1024 require elevated priveleges so choosing a higher +port is advisable. On Debian for example, this can be achieved by installinng +libcap2-bin package: - sudo apt-get update && sudo apt-get -y install libcap2-bin - sudo setcap cap_net_bind_service=+ep /path/to/electrumx_server +``` +sudo apt-get update && sudo apt-get -y install libcap2-bin +sudo setcap cap_net_bind_service=+ep /path/to/electrumx_server +``` - If any listed service has protocol **ssl** or **wss** then :envvar:`SSL_CERTFILE` and - :envvar:`SSL_KEYFILE` must be defined. +If any listed service has protocol **ssl** or **wss** then `SSL_CERTFILE` and +`SSL_KEYFILE` must be defined. - Tor **onion** addresses are invalid in :envvar:`SERVICES`. +Tor **onion** addresses are invalid in `SERVICES`. - Here is an example value of the :envvar:`SERVICES` environment variable:: +Here is an example value of the `SERVICES` environment variable: - tcp://:50001,ssl://:50002,wss://:50004,rpc:// +`tcp://:50001,ssl://:50002,wss://:50004,rpc://` - This serves **tcp**, **ssl**, **wss** on all interfaces on ports 50001, 50002 and 50004 - respectively. **rpc** is served on its default host :const:`localhost` and default port - :const:`8000`. +This serves **tcp**, **ssl**, **wss** on all interfaces on ports 50001, 50002 and 50004 +respectively. **rpc** is served on its default host `localhost` and default port +`8000`. -.. envvar:: REPORT_SERVICES +**REPORT_SERVICES** A comma-separated list of services ElectrumX will advertize and other servers in the server network (if peer discovery is enabled), and any successful connection. @@ -148,74 +149,70 @@ Here are some examples of valid services:: server. If not set or empty, no services are advertized. The **rpc** protocol, special IP addresses (inlcuding private ones if peer discovery is - enabled), and :const:`localhost` are invalid in :envvar:`REPORT_SERVICES`. + enabled), and `localhost` are invalid in `REPORT_SERVICES`. - Here is an example value of the :envvar:`REPORT_SERVICES` environment variable:: + Here is an example value of the `REPORT_SERVICES` environment variable: tcp://sv.usebsv.com:50001,ssl://sv.usebsv.com:50002,wss://sv.usebsv.com:50004 - This advertizes **tcp**, **ssl**, **wss** services at :const:`sv.usebsv.com` on ports + This advertizes **tcp**, **ssl**, **wss** services at `sv.usebsv.com` on ports 50001, 50002 and 50004 respectively. -.. note:: Certificate Authority-signed certificates don't work over Tor, so you should - only have Tor services` in :envvar:`REPORT_SERVICES` if yours is self-signed. +**Note** +> Certificate Authority-signed certificates don't work over Tor, so you should +> only have Tor services in `REPORT_SERVICES` if yours is self-signed. -.. envvar:: SSL_CERTFILE +**SSL_CERTFILE** The filesystem path to your SSL certificate file. - :ref:`SSL certificates` - -.. envvar:: SSL_KEYFILE +**SSL_KEYFILE** The filesystem path to your SSL key file. - :ref:`SSL certificates` - Miscellaneous ============= These environment variables are optional: -.. envvar:: LOG_FORMAT +**LOG_FORMAT** - The Python logging `format string - `_ + The Python logging [format string](https://docs.python.org/3/library/logging.html#logrecord-attributes) to use. Defaults to ``%(levelname)s:%(name)s:%(message)s``. -.. envvar:: LOG_LEVEL +**LOG_LEVEL** The default Python logging level, a case-insensitive string. Useful values are 'debug', 'info', 'warning' and 'error'. -.. envvar:: ALLOW_ROOT +**ALLOW_ROOT** Set this environment variable to anything non-empty to allow running ElectrumX as root. -.. envvar:: NET +**NET** - Must be a :envvar:`NET` from one of the **Coin** classes in - `lib/coins.py`_. Defaults to ``mainnet``. + Must be a `NET` from one of the **Coin** classes in [lib/coins.py](../electrumx/lib/coins.py). + Defaults to ``mainnet``. -.. envvar:: DB_ENGINE +**DB_ENGINE** Database engine for the UTXO and history database. The default is ``leveldb``. The other alternative is ``rocksdb``. You will need to install the appropriate python package for your engine. The value is not case sensitive. -.. envvar:: DONATION_ADDRESS +**DONATION_ADDRESS** The server donation address reported to Electrum clients. Defaults to empty, which Electrum interprets as meaning there is none. -.. envvar:: BANNER_FILE +**BANNER_FILE** The path to a banner file to serve to clients in Electrum's "Console" tab. Relative file paths must be relative to - :envvar:`DB_DIRECTORY`. The banner file is re-read for each new + `DB_DIRECTORY`. The banner file is re-read for each new client. You can place several meta-variables in your banner file, which will be @@ -230,47 +227,47 @@ These environment variables are optional: + ``$DAEMON_SUBVERSION`` is replaced with the daemon's user agent string. For example, ``/BitcoinUnlimited:0.12.1(EB16; AD4)/``. + ``$DONATION_ADDRESS`` is replaced with the address from the - :envvar:`DONATION_ADDRESS` environment variable. + `DONATION_ADDRESS` environment variable. - See `here `_ + See [here](https://github.com/shsmith/electrumx-banner-updater>) for a script that updates a banner file periodically with useful statistics about fees, last block time and height, etc. -.. envvar:: TOR_BANNER_FILE +**TOR_BANNER_FILE** - As for :envvar:`BANNER_FILE` (which is also the default) but shown + As for `BANNER_FILE` (which is also the default) but shown to incoming connections believed to be to your Tor hidden service. -.. envvar:: ANON_LOGS +**ANON_LOGS** Set to anything non-empty to replace IP addresses in logs with redacted text like ``xx.xx.xx.xx:xxx``. By default IP addresses will be written to logs. -.. envvar:: LOG_SESSIONS +**LOG_SESSIONS** The number of seconds between printing session statistics to the - log. The output is identical to the :ref:`sessions` RPC command - except that :envvar:`ANON_LOGS` is honoured. Defaults to 3600. Set + log. The output is identical to the `sessions` RPC command + except that `ANON_LOGS` is honoured. Defaults to 3600. Set to zero to suppress this logging. -.. envvar:: REORG_LIMIT +**REORG_LIMIT** The maximum number of blocks to be able to handle in a chain reorganisation. ElectrumX retains some fairly compact undo information for this many blocks in levelDB. The default is a - function of :envvar:`COIN` and :envvar:`NET`; for Bitcoin mainnet it + function of `COIN` and `NET`; for Bitcoin mainnet it is 200. -.. envvar:: EVENT_LOOP_POLICY +**EVENT_LOOP_POLICY** The name of an event loop policy to replace the default asyncio policy, if any. At present only ``uvloop`` is accepted, in which - case you must have installed the `uvloop`_ Python package. + case you must have installed the `uvloop` Python package. If you are not sure what this means leave it unset. -.. envvar:: DROP_CLIENT +**DROP_CLIENT** Set a regular expression to disconnect any client based on their version string. For example to drop versions from 1.0 to 1.2 use @@ -288,13 +285,13 @@ Address subscriptions in ElectrumX are very cheap - they consume about two subscription-related defaults below are low and encourage you to raise them. -.. envvar:: MAX_SESSIONS +**MAX_SESSIONS** The maximum number of incoming connections. Once reached, TCP and SSL listening sockets are closed until the session count drops naturally to 95% of the limit. Defaults to 1,000. -.. envvar:: MAX_SEND +**MAX_SEND** The maximum size of a response message to send over the wire, in bytes. Defaults to 1,000,000 (except for AuxPoW coins, which default @@ -304,43 +301,43 @@ raise them. The Electrum protocol has a flaw in that address histories must be served all at once or not at all, an obvious avenue for abuse. - :envvar:`MAX_SEND` is a stop-gap until the protocol is improved to + `MAX_SEND` is a stop-gap until the protocol is improved to admit incremental history requests. Each history entry is approximately 100 bytes so the default is equivalent to a history limit of around 10,000 entries, which should be ample for most legitimate users. If you use a higher default bear in mind one client can request history for multiple addresses. Also note that the largest raw transaction you will be able to serve to a client is - just under half of :envvar:`MAX_SEND`, as each raw byte becomes 2 + just under half of `MAX_SEND`, as each raw byte becomes 2 hexadecimal ASCII characters on the wire. Very few transactions on Bitcoin mainnet are over 500KB in size. -.. envvar:: COST_SOFT_LIMIT -.. envvar:: COST_HARD_LIMIT -.. envvar:: REQUEST_SLEEP -.. envvar:: INITIAL_CONCURRENT +**COST_SOFT_LIMIT** \ +**COST_HARD_LIMIT** \ +**REQUEST_SLEEP** \ +**INITIAL_CONCURRENT** - All values are integers. :envvar:`COST_SOFT_LIMIT` defaults to :const:`1,000`, - :envvar:`COST_HARD_LIMIT` to :const:`10,000`, :envvar:`REQUEST_SLEEP` to :const:`2,500` - milliseconds, and :envvar:`INITIAL_CONCURRENT` to :const:`10` concurrent requests. + All values are integers. `COST_SOFT_LIMIT` defaults to `1,000`, + `COST_HARD_LIMIT` to `10,000`, `REQUEST_SLEEP` to `2,500` + milliseconds, and `INITIAL_CONCURRENT` to `10` concurrent requests. The server prices each request made to it based upon an estimate of the resources needed to process it. Factors include whether the request uses bitcoind, how much bandwidth it uses, and how hard it hits the databases. - To set a base for the units, a :func:`blockchain.scripthash.subscribe` subscription to - an address with a history of 2 or fewer transactions is costed at :const:`1.0` before - considering the bandwidth consumed. :func:`server.ping` is costed at :const:`0.1`. + To set a base for the units, a `blockchain.scripthash.subscribe` subscription to + an address with a history of 2 or fewer transactions is costed at `1.0` before + considering the bandwidth consumed. `server.ping` is costed at `0.1`. As the total cost of a session goes over the soft limit, its requests start to be throttled in two ways. First, the number of requests for that session that the server will process concurrently is reduced. Second, each request starts to sleep a little before being handled. - Before throttling starts, the server will process up to :envvar:`INITIAL_CONCURRENT` + Before throttling starts, the server will process up to `INITIAL_CONCURRENT` requests concurrently without sleeping. As the session cost ranges from - :envvar:`COST_SOFT_LIMIT` to :envvar:`COST_HARD_LIMIT`, concurrency drops linearly to - zero and each request's sleep time increases linearly up to :envvar:`REQUEST_SLEEP` + `COST_SOFT_LIMIT` to `COST_HARD_LIMIT`, concurrency drops linearly to + zero and each request's sleep time increases linearly up to `REQUEST_SLEEP` milliseconds. Once the hard limit is reached, the session is disconnected. In order that non-abusive sessions can continue to be served, a session's cost gradually @@ -356,34 +353,34 @@ raise them. requests the server has to manage in memory. It will also give a much worse experience for genuine connections. -.. envvar:: BANDWIDTH_UNIT_COST +**BANDWIDTH_UNIT_COST** - The number of bytes, sent and received, by a session that is deemed to cost :const:`1.0`. + The number of bytes, sent and received, by a session that is deemed to cost `1.0`. - The default value :const:`5,000` bytes, meaning the bandwidth cost assigned to a response + The default value `5,000` bytes, meaning the bandwidth cost assigned to a response of 100KB is 20. If your bandwidth is cheap you should probably raise this. -.. envvar:: REQUEST_TIMEOUT +**REQUEST_TIMEOUT** - An integer number of seconds defaulting to :const:`30`. If a request takes longer than + An integer number of seconds defaulting to `30`. If a request takes longer than this to respond to, either because of request limiting or because the request is expensive, the server rejects it and returns a timeout error to the client indicating that the server is busy. This can help prevent large backlogs of unprocessed requests building up under heavy load. -.. envvar:: SESSION_TIMEOUT +**SESSION_TIMEOUT** - An integer number of seconds defaulting to :const:`600`. Sessions that have not sent a + An integer number of seconds defaulting to `600`. Sessions that have not sent a request for longer than this are disconnected. Properly functioning clients should send - a :func:`server.ping` request once roughly 450 seconds have passed since the previous + a `server.ping` request once roughly 450 seconds have passed since the previous request, in order to avoid disconnection. Peer Discovery ============== -In response to the :func:`server.peers.subscribe` RPC call, ElectrumX +In response to the `server.peers.subscribe` RPC call, ElectrumX will only return peer servers that it has recently connected to and verified basic functionality. @@ -398,10 +395,9 @@ ElectrumX will perform peer-discovery by default and announce itself to other peers. If your server is private you may wish to disable some of this. -.. envvar:: PEER_DISCOVERY +**PEER_DISCOVERY** - This environment variable is case-insensitive and defaults to - ``on``. + This environment variable is case-insensitive and defaults to ``on``. If ``on``, ElectrumX will occasionally connect to and verify its network of peer servers. @@ -411,7 +407,7 @@ some of this. discovery is disabled and the server will only return itself in the peers list. -.. envvar:: PEER_ANNOUNCE +**PEER_ANNOUNCE** Set this environment variable to empty to disable announcing itself. If not defined, or non-empty, ElectrumX will announce itself to @@ -422,14 +418,14 @@ some of this. peer discovery if it notices it is not present in the peer's returned list. -.. envvar:: FORCE_PROXY +**FORCE_PROXY** By default peer discovery happens over the clear internet. Set this to non-empty to force peer discovery to be done via the proxy. This might be useful if you are running a Tor service exclusively and wish to keep your IP address private. -.. envvar:: TOR_PROXY_HOST +**TOR_PROXY_HOST** The host where your Tor proxy is running. Defaults to ``localhost``. @@ -437,13 +433,13 @@ some of this. If you are not running a Tor proxy just leave this environment variable undefined. -.. envvar:: TOR_PROXY_PORT +**TOR_PROXY_PORT** The port on which the Tor proxy is running. If not set, ElectrumX will autodetect any proxy running on the usual ports 9050 (Tor), 9150 (Tor browser bundle) and 1080 (socks). -.. envvar:: BLACKLIST_URL +**BLACKLIST_URL** URL to retrieve a list of blacklisted peers. If not set, a coin- specific default is used. @@ -460,9 +456,7 @@ because of Python overhead and also because leveldb consumes a lot of memory when flushing. So I recommend you do not set this over 60% of your available physical RAM: -.. _CACHE: - -.. envvar:: CACHE_MB +**CACHE_MB** The amount of cache, in MB, to use. The default is 1,200. @@ -476,6 +470,3 @@ your available physical RAM: caches, and also because of Python GC. I do not recommend raising this above 2000. - -.. _lib/coins.py: https://github.com/kyuupichan/electrumx/blob/master/electrumx/lib/coins.py -.. _uvloop: https://pypi.python.org/pypi/uvloop diff --git a/docs/features.rst b/docs/features.md similarity index 98% rename from docs/features.rst rename to docs/features.md index 996c14896..244782d4e 100644 --- a/docs/features.rst +++ b/docs/features.md @@ -57,7 +57,7 @@ blockchain indexer: eliminate CPU idling. As a Python program ElectrumX is unavoidably single-threaded in its essence; we must keep that CPU core busy. -Python's :mod:`asyncio` means ElectrumX has no (direct) use for threads +Python's `asyncio` means ElectrumX has no (direct) use for threads and associated complications. Roadmap diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 000000000..aaa379aa0 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,16 @@ +Documentation +============= + + + [Features](features.md) + + [Changelog](changelog.md) + + [HOWTO](HOWTO.md) + + [Environment](environment.md) + + Protocol + + [Protocol Basics](protocol-basics.md) + + [Protocol Changes](protocol-changes.md) + + [Protocol Methods](protocol-methods.md) + + [Protocol Removals](protocol-removed.md) + + [Peer Discovery](peer_discovery.md) + + [RPC Interface](rpc-interface.md) + + [Architecture](architecture.md) + + [Authors](authors.md) diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index d3c3de306..000000000 --- a/docs/index.rst +++ /dev/null @@ -1,66 +0,0 @@ -========= -ElectrumX -========= - -.. image:: https://travis-ci.org/kyuupichan/electrumx.svg?branch=master - :target: https://travis-ci.org/kyuupichan/electrumx -.. image:: https://coveralls.io/repos/github/kyuupichan/electrumx/badge.svg - :target: https://coveralls.io/github/kyuupichan/electrumx - -A reimplementation of Electrum-Server for a future with bigger blocks. - -The current version is |release|. - -Source Code -=========== - -The project is hosted on `GitHub -`_. and uses `Travis -`_ for Continuous -Integration. - -Please submit an issue on the `bug tracker -`_ if you have found a -bug or have a suggestion to improve the server. - -Authors and License -=================== - -Neil Booth wrote the vast majority of the code; see :ref:`Authors`. -Python version at least 3.6 is required. - -The code is released under the `MIT Licence -`_. - -Getting Started -=============== - -See :ref:`HOWTO`. - -There is also an `installer`_ available that simplifies the -installation on various Linux-based distributions, and a `Dockerfile`_ -available . - -.. _installer: https://github.com/bauerj/electrumx-installer -.. _Dockerfile: https://github.com/lukechilds/docker-electrumx - -Documentation -============= - -.. toctree:: - - features - changelog - HOWTO - environment - protocol - peer_discovery - rpc-interface - architecture - authors - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`search` diff --git a/docs/peer_discovery.rst b/docs/peer_discovery.md similarity index 83% rename from docs/peer_discovery.rst rename to docs/peer_discovery.md index 0270ce1aa..6f44cb1da 100644 --- a/docs/peer_discovery.rst +++ b/docs/peer_discovery.md @@ -1,14 +1,12 @@ -.. _Peer Discovery: - Peer Discovery ============== This was imlpemented in ElectrumX as of version 0.11.0. Support for IRC peer discovery was removed in ElectrumX version 1.2.1. -The :dfn:`peer database` is an in-memory store of peers with at least +The `peer database` is an in-memory store of peers with at least the following information about a peer, required for a response to the -:func:`server.peers.subscribe` RPC call: +`server.peers.subscribe` RPC call: * host name * ip address @@ -24,14 +22,13 @@ A list of hard-coded, well-known peers seeds the peer discovery process. Ideally it should have at least 4 servers that have shown commitment to reliable service. -In ElectrumX this is a per-coin property in `lib/coins.py -`_. +In ElectrumX this is a per-coin property in [lib/coins.py](../electrumx/lib/coins.py). server.peers.subscribe ---------------------- -:func:`server.peers.subscribe` is used by Electrum clients to get a +`server.peers.subscribe` is used by Electrum clients to get a list of peer servers, in preference to a hard-coded list of peer servers in the client, which it will fall back to if necessary. @@ -56,10 +53,10 @@ time has passed since the last successful connection to a peer, an Electrum server should make another attempt to connect, choosing either the TCP or SSL port. -On connecting it should issue :func:`server.peers.subscribe`, -:func:`blockchain.headers.subscribe`, and :func:`server.features` RPC +On connecting it should issue `server.peers.subscribe`, +`blockchain.headers.subscribe`, and `server.features` RPC calls to collect information about the server and its peers. If the -peer seems to not know of you, you can issue a :func:`server.add_peer` +peer seems to not know of you, you can issue a `server.add_peer` call to advertise yourself. Once this is done and replies received, terminate the connection. @@ -69,7 +66,7 @@ from any other source. On connecting, a server should confirm the peer is serving the same network, ideally via the genesis block hash of the -:func:`server.features` RPC call below. Also the height reported by +`server.features` RPC call below. Also the height reported by the peer should be within a small number of the expected value. If a peer is on the wrong network it should never be advertised to clients or other peers. Such invalid peers should perhaps be remembered for a @@ -96,7 +93,7 @@ can be configured or that it will try to autodetect. server.features --------------- -:func:`server.features` is a fairly new RPC call that a server can use +`server.features` is a fairly new RPC call that a server can use to advertise what services and features it offers. It is intended for use by Electrum clients as well as other peers. Peers will use it to gather peer information from the peer itself. @@ -109,10 +106,10 @@ If a key is missing the feature is presumed not to be offered. server.add_peer --------------- -:func:`server.add_peer` is intended for a new server to get itself in +`server.add_peer` is intended for a new server to get itself in the connected set. -A server receiving a :func:`server.add_peer` call should not replace +A server receiving a `server.add_peer` call should not replace existing information about the host(s) given, but instead schedule a separate connection to verify the information for itself. @@ -127,9 +124,9 @@ Notes for Implementors * it is very important to only accept peers that appear to be on the same network. At a minimum the genesis hash should be compared (if - the peer supports :func:`server.features`), and also that the peer's + the peer supports `server.features`), and also that the peer's reported height is within a few blocks of your own server's height. -* care should be taken with the :func:`server.add_peer` call. +* care should be taken with the `server.add_peer` call. Consider only accepting it once per connection. Clearnet peer requests should check the peer resolves to the requesting IP address, to prevent attackers from being able to trigger arbitrary @@ -146,7 +143,7 @@ Notes for Implementors source to at most a handful, to limit the effectiveness of malicious peers wanting to trigger arbitrary outgoing connections or fill your peer tables with junk data. -* in the response to :func:`server.peers.subscribe` calls, consider +* in the response to `server.peers.subscribe` calls, consider limiting the number of peers on similar IP subnets to protect against sybil attacks, and in the case of onion servers the total returned. diff --git a/docs/protocol-basics.rst b/docs/protocol-basics.md similarity index 67% rename from docs/protocol-basics.rst rename to docs/protocol-basics.md index 9dbbd5458..26940585a 100644 --- a/docs/protocol-basics.rst +++ b/docs/protocol-basics.md @@ -7,19 +7,18 @@ Message Stream Clients and servers communicate using **JSON RPC** over an unspecified underlying stream transport. Examples include TCP, SSL, WS and WSS. -Two standards `JSON RPC 1.0 -`_ and `JSON RPC 2.0 -`_ are specified; use of version +Two standards [JSON RPC 1.0](http://www.jsonrpc.org/specification_v1) and +[JSON RPC 2.0](http://www.jsonrpc.org/specification>) are specified; use of version 2.0 is encouraged but not required. Server support of batch requests is encouraged for version 1.0 but not required. -.. note:: A client or server should only indicate JSON RPC 2.0 by - setting the `jsonrpc - `_ member of - its messages to ``"2.0"`` if it supports the version 2.0 protocol in - its entirety. ElectrumX does and will expect clients advertizing so - to function correctly. Those that do not will be disconnected and - possibly blacklisted. +**Note** +>A client or server should only indicate JSON RPC 2.0 by +>setting the [jsonrpc](http://www.jsonrpc.org/specification#request_object) member of +>its messages to ``"2.0"`` if it supports the version 2.0 protocol in +>its entirety. ElectrumX does and will expect clients advertizing so +>to function correctly. Those that do not will be disconnected and +>possibly blacklisted. Clients making batch requests should limit their size depending on the nature of their query, because servers will limit response size as an @@ -48,7 +47,7 @@ Notifications ------------- Some RPC calls are subscriptions which, after the initial response, -will send a JSON RPC :dfn:`notification` each time the thing +will send a JSON RPC `notification` each time the thing subscribed to changes. The `method` of the notification is the same as the method of the subscription, and the `params` of the notification (and their names) are given in the documentation of the @@ -71,78 +70,73 @@ say from `protocol_min` to `protocol_max`, which may be the same. When a connection is made, both client and server must initially assume the protocol to use is their own `protocol_min`. -The client should send a :func:`server.version` RPC call as early as +The client should send a `server.version` RPC call as early as possible in order to negotiate the precise protocol version; see its description for more detail. All responses received in the stream from and including the server's response to this call will use its negotiated protocol version. -.. _script hashes: - Script Hashes ------------- -A :dfn:`script hash` is the hash of the binary bytes of the locking +A `script hash` is the hash of the binary bytes of the locking script (ScriptPubKey), expressed as a hexadecimal string. The hash function to use is given by the "hash_function" member of -:func:`server.features` (currently :func:`sha256` only). Like for +`server.features` (currently `sha256` only). Like for block and transaction hashes, when converting the big-endian binary hash to a hexadecimal string the least-significant byte appears first, and the most-significant byte last. -For example, the legacy Bitcoin address from the genesis block:: - - 1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa - -has P2PKH script:: - - 76a91462e907b15cbf27d5425399ebf6f0fb50ebb88f1888ac - -with SHA256 hash:: - - 6191c3b590bfcfa0475e877c302da1e323497acf3b42c08d8fa28e364edf018b - -which is sent to the server reversed as:: - - 8b01df4e368ea28f8dc0423bcf7a4923e3a12d307c875e47a0cfbf90b5c39161 - +For example, the legacy Bitcoin address from the genesis block: +``` +1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa +``` +has P2PKH script: +``` +76a91462e907b15cbf27d5425399ebf6f0fb50ebb88f1888ac +``` +with SHA256 hash: +``` +6191c3b590bfcfa0475e877c302da1e323497acf3b42c08d8fa28e364edf018b +``` +which is sent to the server reversed as: +``` +8b01df4e368ea28f8dc0423bcf7a4923e3a12d307c875e47a0cfbf90b5c39161 +``` By subscribing to this hash you can find P2PKH payments to that address. One public key, the genesis block public key, among the trillions for -that address is:: - - 04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb - 649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f - -which has P2PK script:: - - 4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb - 649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac - -with SHA256 hash:: - - 3318537dfb3135df9f3d950dbdf8a7ae68dd7c7dfef61ed17963ff80f3850474 - -which is sent to the server reversed as:: - - 740485f380ff6379d11ef6fe7d7cdd68aea7f8bd0d953d9fdf3531fb7d531833 - +that address is: +``` +04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb +649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f +``` +which has P2PK script: +``` +4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb +649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac +``` +with SHA256 hash: +``` +3318537dfb3135df9f3d950dbdf8a7ae68dd7c7dfef61ed17963ff80f3850474 +``` +which is sent to the server reversed as: +``` +740485f380ff6379d11ef6fe7d7cdd68aea7f8bd0d953d9fdf3531fb7d531833 +``` By subscribing to this hash you can find P2PK payments to the genesis block public key. -.. note:: The Genesis block coinbase is uniquely unspendable and - therefore not indexed. It will not show with the above P2PK script - hash subscription. - - -.. _status: +**Note** +>The Genesis block coinbase is uniquely unspendable and +>therefore not indexed. It will not show with the above P2PK script +>hash subscription. Status ------ -To calculate the `status` of a :ref:`script hash