Merge pull request #393 from HorizenOfficial/AddressIndexing_development

.travis.yml

@@ -3,12 +3,6 @@ language: shell
 os: linux
 
 .com.github.horizenofficial.zen.env-definitions:
-  - &docker_amd64_linux_ubuntu_xenial
-    env:
-      - DOCKER_ARCH=amd64
-      - DOCKER_TARGET_OS=linux
-      - DOCKER_FROM=ubuntu_xenial
-      - DOCKER_IS_DEB=true
   - &docker_amd64_linux_ubuntu_bionic
     env:
       - DOCKER_ARCH=amd64
@@ -21,77 +15,99 @@ os: linux
       - DOCKER_TARGET_OS=windows
       - DOCKER_FROM=ubuntu_bionic
       - DOCKER_IS_DEB=true
-  - &docker_amd64_linux_ubuntu_xenial_unit-tests
+  - &docker_amd64_linux_ubuntu_focal
     env:
       - DOCKER_ARCH=amd64
       - DOCKER_TARGET_OS=linux
-      - DOCKER_FROM=ubuntu_xenial
+      - DOCKER_FROM=ubuntu_focal
+      - DOCKER_IS_DEB=true
+  - &docker_amd64_linux_ubuntu_bionic_unit-tests
+    env:
+      - DOCKER_ARCH=amd64
+      - DOCKER_TARGET_OS=linux
+      - DOCKER_FROM=ubuntu_bionic
       - DOCKER_IS_DEB=true
       - TEST_CMD="./qa/zcash/full_test_suite.py"
       - TEST_ARGS="btest gtest sec-hard no-dot-so util-test secp256k1 libsnark univalue"
-  - &docker_amd64_linux_ubuntu_xenial_rpc-tests_1
+  - &docker_amd64_linux_ubuntu_bionic_rpc-tests_1
+    env:
+      - DOCKER_ARCH=amd64
+      - DOCKER_TARGET_OS=linux
+      - DOCKER_FROM=ubuntu_bionic
+      - DOCKER_IS_DEB=true
+      - TEST_CMD="./qa/zcash/full_test_suite.py"
+      - TEST_ARGS="rpc --rpc-extended --rpc-split=4:1"
+  - &docker_amd64_linux_ubuntu_bionic_rpc-tests_2
     env:
       - DOCKER_ARCH=amd64
       - DOCKER_TARGET_OS=linux
-      - DOCKER_FROM=ubuntu_xenial
+      - DOCKER_FROM=ubuntu_bionic
       - DOCKER_IS_DEB=true
       - TEST_CMD="./qa/zcash/full_test_suite.py"
-      - TEST_ARGS="rpc --rpc-extended --rpc-split=3:1"
-  - &docker_amd64_linux_ubuntu_xenial_rpc-tests_2
+      - TEST_ARGS="rpc --rpc-extended --rpc-split=4:2"
+  - &docker_amd64_linux_ubuntu_bionic_rpc-tests_3
     env:
       - DOCKER_ARCH=amd64
       - DOCKER_TARGET_OS=linux
-      - DOCKER_FROM=ubuntu_xenial
+      - DOCKER_FROM=ubuntu_bionic
       - DOCKER_IS_DEB=true
       - TEST_CMD="./qa/zcash/full_test_suite.py"
-      - TEST_ARGS="rpc --rpc-extended --rpc-split=3:2"
-  - &docker_amd64_linux_ubuntu_xenial_rpc-tests_3
+      - TEST_ARGS="rpc --rpc-extended --rpc-split=4:3"
+  - &docker_amd64_linux_ubuntu_bionic_rpc-tests_4
     env:
       - DOCKER_ARCH=amd64
       - DOCKER_TARGET_OS=linux
-      - DOCKER_FROM=ubuntu_xenial
+      - DOCKER_FROM=ubuntu_bionic
       - DOCKER_IS_DEB=true
       - TEST_CMD="./qa/zcash/full_test_suite.py"
-      - TEST_ARGS="rpc --rpc-extended --rpc-split=3:3"
-  - &docker_amd64_linux_ubuntu_xenial_test-fetch-params
+      - TEST_ARGS="rpc --rpc-extended --rpc-split=4:4"
+  - &docker_amd64_linux_ubuntu_bionic_test-fetch-params
     env:
       - DOCKER_ARCH=amd64
       - DOCKER_TARGET_OS=linux
-      - DOCKER_FROM=ubuntu_xenial
+      - DOCKER_FROM=ubuntu_bionic
       - DOCKER_IS_DEB=true
       - TEST_CMD="./qa/zen/test-fetch-params.sh"
-  - &docker_amd64_linux_ubuntu_bionic_unit-tests
+  - &docker_amd64_linux_ubuntu_focal_unit-tests
     env:
       - DOCKER_ARCH=amd64
       - DOCKER_TARGET_OS=linux
-      - DOCKER_FROM=ubuntu_bionic
+      - DOCKER_FROM=ubuntu_focal
       - DOCKER_IS_DEB=true
       - TEST_CMD="./qa/zcash/full_test_suite.py"
       - TEST_ARGS="btest gtest sec-hard no-dot-so util-test secp256k1 libsnark univalue"
-  - &docker_amd64_linux_ubuntu_bionic_rpc-tests_1
+  - &docker_amd64_linux_ubuntu_focal_rpc-tests_1
     env:
       - DOCKER_ARCH=amd64
       - DOCKER_TARGET_OS=linux
-      - DOCKER_FROM=ubuntu_bionic
+      - DOCKER_FROM=ubuntu_focal
       - DOCKER_IS_DEB=true
       - TEST_CMD="./qa/zcash/full_test_suite.py"
-      - TEST_ARGS="rpc --rpc-extended --rpc-split=3:1"
-  - &docker_amd64_linux_ubuntu_bionic_rpc-tests_2
+      - TEST_ARGS="rpc --rpc-extended --rpc-split=4:1"
+  - &docker_amd64_linux_ubuntu_focal_rpc-tests_2
     env:
       - DOCKER_ARCH=amd64
       - DOCKER_TARGET_OS=linux
-      - DOCKER_FROM=ubuntu_bionic
+      - DOCKER_FROM=ubuntu_focal
       - DOCKER_IS_DEB=true
       - TEST_CMD="./qa/zcash/full_test_suite.py"
-      - TEST_ARGS="rpc --rpc-extended --rpc-split=3:2"
-  - &docker_amd64_linux_ubuntu_bionic_rpc-tests_3
+      - TEST_ARGS="rpc --rpc-extended --rpc-split=4:2"
+  - &docker_amd64_linux_ubuntu_focal_rpc-tests_3
     env:
       - DOCKER_ARCH=amd64
       - DOCKER_TARGET_OS=linux
-      - DOCKER_FROM=ubuntu_bionic
+      - DOCKER_FROM=ubuntu_focal
       - DOCKER_IS_DEB=true
       - TEST_CMD="./qa/zcash/full_test_suite.py"
-      - TEST_ARGS="rpc --rpc-extended --rpc-split=3:3"
+      - TEST_ARGS="rpc --rpc-extended --rpc-split=4:3"
+  - &docker_amd64_linux_ubuntu_focal_rpc-tests_4
+    env:
+      - DOCKER_ARCH=amd64
+      - DOCKER_TARGET_OS=linux
+      - DOCKER_FROM=ubuntu_focal
+      - DOCKER_IS_DEB=true
+      - TEST_CMD="./qa/zcash/full_test_suite.py"
+      - TEST_ARGS="rpc --rpc-extended --rpc-split=4:4"
   - &amd64_osx_xcode9_4_unit-tests
     env:
       - TEST_CMD="./qa/zcash/full_test_suite.py"
@@ -113,14 +129,16 @@ os: linux
 .com.github.horizenofficial.zen.job-definitions:
   - &prepare_docker_amd64
     os: linux
-    dist: bionic
+    dist: focal
     cache: false
     script:
       - bash "${TRAVIS_BUILD_DIR}/contrib/ci-horizen/scripts/prepare/docker_image_build.sh"
       - bash "${TRAVIS_BUILD_DIR}/contrib/ci-horizen/scripts/prepare/docker_image_deploy.sh"
-  - &build_docker_amd64
+  - &build_docker_amd64_x-large
     os: linux
-    dist: bionic
+    dist: focal
+    vm:
+      size: x-large
     cache:
       directories:
         - "$HOME/.ccache"
@@ -130,7 +148,20 @@ os: linux
   - &test_docker_amd64
     if: env(SKIP_TESTS) != true
     os: linux
-    dist: bionic
+    dist: focal
+    cache:
+      directories:
+        - "$HOME/.zcash-params"
+    script:
+      - bash -c "RENAME_FOLDER='true' RENAME_SUFFIX='_clean' ${TRAVIS_BUILD_DIR}/contrib/ci-horizen/scripts/common/get_archive.sh ${B2_DL_DECOMPRESS_FOLDER} ${B2_DL_FILENAME}"
+      - bash -c "${TRAVIS_BUILD_DIR}/contrib/ci-horizen/scripts/common/travis_stay_alive.sh"
+      - bash -c '${TRAVIS_BUILD_DIR}/contrib/ci-horizen/scripts/test/run_test.sh "${TEST_CMD}" "${TEST_ARGS}"'
+  - &test_docker_amd64_x-large
+    if: env(SKIP_TESTS) != true
+    os: linux
+    dist: focal
+    vm:
+      size: x-large
     cache:
       directories:
         - "$HOME/.zcash-params"
@@ -161,10 +192,10 @@ jobs:
     # build and push docker build/test dependency images
     - stage: Prepare
       <<: *prepare_docker_amd64
-      <<: *docker_amd64_linux_ubuntu_xenial
+      <<: *docker_amd64_linux_ubuntu_bionic
     - stage: Prepare
       <<: *prepare_docker_amd64
-      <<: *docker_amd64_linux_ubuntu_bionic
+      <<: *docker_amd64_linux_ubuntu_focal
     - stage: Prepare
       <<: *prepare_docker_amd64
       <<: *docker_amd64_windows_ubuntu_bionic
@@ -179,13 +210,13 @@ jobs:
         - bash -c "${TRAVIS_BUILD_DIR}/contrib/ci-horizen/scripts/common/push_archive.sh ${B2_UL_COMPRESS_FOLDER} ${B2_UL_FILENAME}"
     # Docker based builds
     - stage: Build
-      <<: *build_docker_amd64
-      <<: *docker_amd64_linux_ubuntu_xenial
-    - stage: Build
-      <<: *build_docker_amd64
+      <<: *build_docker_amd64_x-large
       <<: *docker_amd64_linux_ubuntu_bionic
     - stage: Build
-      <<: *build_docker_amd64
+      <<: *build_docker_amd64_x-large
+      <<: *docker_amd64_linux_ubuntu_focal
+    - stage: Build
+      <<: *build_docker_amd64_x-large
       <<: *docker_amd64_windows_ubuntu_bionic
     # osx build
     - stage: Build
@@ -201,32 +232,38 @@ jobs:
         - bash -c "${TRAVIS_BUILD_DIR}/contrib/ci-horizen/scripts/common/push_archive.sh ${B2_UL_COMPRESS_FOLDER} ${B2_UL_FILENAME}"
     # Docker based tests
     - stage: Test
-      <<: *test_docker_amd64
-      <<: *docker_amd64_linux_ubuntu_xenial_unit-tests
+      <<: *test_docker_amd64_x-large
+      <<: *docker_amd64_linux_ubuntu_bionic_unit-tests
     - stage: Test
-      <<: *test_docker_amd64
-      <<: *docker_amd64_linux_ubuntu_xenial_rpc-tests_1
+      <<: *test_docker_amd64_x-large
+      <<: *docker_amd64_linux_ubuntu_bionic_rpc-tests_1
     - stage: Test
-      <<: *test_docker_amd64
-      <<: *docker_amd64_linux_ubuntu_xenial_rpc-tests_2
+      <<: *test_docker_amd64_x-large
+      <<: *docker_amd64_linux_ubuntu_bionic_rpc-tests_2
     - stage: Test
-      <<: *test_docker_amd64
-      <<: *docker_amd64_linux_ubuntu_xenial_rpc-tests_3
+      <<: *test_docker_amd64_x-large
+      <<: *docker_amd64_linux_ubuntu_bionic_rpc-tests_3
     - stage: Test
-      <<: *test_docker_amd64
-      <<: *docker_amd64_linux_ubuntu_xenial_test-fetch-params
+      <<: *test_docker_amd64_x-large
+      <<: *docker_amd64_linux_ubuntu_bionic_rpc-tests_4
     - stage: Test
       <<: *test_docker_amd64
-      <<: *docker_amd64_linux_ubuntu_bionic_unit-tests
+      <<: *docker_amd64_linux_ubuntu_bionic_test-fetch-params
     - stage: Test
-      <<: *test_docker_amd64
-      <<: *docker_amd64_linux_ubuntu_bionic_rpc-tests_1
+      <<: *test_docker_amd64_x-large
+      <<: *docker_amd64_linux_ubuntu_focal_unit-tests
     - stage: Test
-      <<: *test_docker_amd64
-      <<: *docker_amd64_linux_ubuntu_bionic_rpc-tests_2
+      <<: *test_docker_amd64_x-large
+      <<: *docker_amd64_linux_ubuntu_focal_rpc-tests_1
     - stage: Test
-      <<: *test_docker_amd64
-      <<: *docker_amd64_linux_ubuntu_bionic_rpc-tests_3
+      <<: *test_docker_amd64_x-large
+      <<: *docker_amd64_linux_ubuntu_focal_rpc-tests_2
+    - stage: Test
+      <<: *test_docker_amd64_x-large
+      <<: *docker_amd64_linux_ubuntu_focal_rpc-tests_3
+    - stage: Test
+      <<: *test_docker_amd64_x-large
+      <<: *docker_amd64_linux_ubuntu_focal_rpc-tests_4
     # osx based tests
     - stage: Test
       <<: *test_osx_xcode9_4

README.md

@@ -1,4 +1,4 @@
-Zen 2.0.23-bitcore
+Zen 2.0.24-bitcore
 ==============
 
 What is Horizen?

configure.ac

@@ -2,7 +2,7 @@ dnl require autoconf 2.60 (AS_ECHO/AS_ECHO_N)
 AC_PREREQ([2.60])
 define(_CLIENT_VERSION_MAJOR, 2)
 define(_CLIENT_VERSION_MINOR, 0)
-define(_CLIENT_VERSION_REVISION, 23)
+define(_CLIENT_VERSION_REVISION, 24)
 define(_CLIENT_VERSION_BUILD, 50)
 define(_ZC_BUILD_VAL, m4_if(m4_eval(_CLIENT_VERSION_BUILD < 25), 1, m4_incr(_CLIENT_VERSION_BUILD), m4_eval(_CLIENT_VERSION_BUILD < 50), 1, m4_eval(_CLIENT_VERSION_BUILD - 24), m4_eval(_CLIENT_VERSION_BUILD == 50), 1, , m4_eval(_CLIENT_VERSION_BUILD - 50)))
 define(_CLIENT_VERSION_SUFFIX, m4_if(m4_eval(_CLIENT_VERSION_BUILD < 25), 1, _CLIENT_VERSION_REVISION-beta$1, m4_eval(_CLIENT_VERSION_BUILD < 50), 1, _CLIENT_VERSION_REVISION-rc$1, m4_eval(_CLIENT_VERSION_BUILD == 50), 1, _CLIENT_VERSION_REVISION, _CLIENT_VERSION_REVISION-$1)))

contrib/ci-horizen/dockerfiles/amd64/linux/ubuntu_bionic/Dockerfile

@@ -7,22 +7,24 @@ COPY entrypoint.sh /usr/local/bin/entrypoint.sh
 SHELL ["/bin/bash", "-c"]
 
 RUN set -euxo pipefail \
-    && DEBIAN_FRONTEND=noninteractive \
+    && export DEBIAN_FRONTEND=noninteractive \
     && apt-get update \
     && apt-get -y --no-install-recommends install apt-utils \
     && apt-get -y --no-install-recommends dist-upgrade \
     && apt-get -y --no-install-recommends install aria2 autoconf automake bsdmainutils build-essential \
        ca-certificates cmake curl dirmngr fakeroot git g++-multilib gnupg2 help2man libc6-dev libgomp1 \
-       libtool lintian m4 ncurses-dev pigz pkg-config pv python python-dev python-pip python-setuptools \
-       python-wheel python-wheel-common python-zmq unzip wget zlib1g-dev \
-    && pip install b2==1.4.2 pyblake2 \
+       libtool lintian m4 ncurses-dev pigz pkg-config pv python-dev python-pip python-setuptools \
+       python-wheel python-wheel-common python-zmq time unzip wget zlib1g-dev \
+    && pip install b2==1.4.2 pyblake2 websocket-client2 \
     && BASEURL="https://github.com/tianon/gosu/releases/download/" \
-    && GOSU_VERSION="1.11" \
+    && GOSU_VERSION="1.13" \
     && DPKG_ARCH="$(dpkg --print-architecture | awk -F- '{ print $NF }')" \
     && curl -o /usr/local/bin/gosu -SL "${BASEURL}/${GOSU_VERSION}/gosu-${DPKG_ARCH}" \
     && curl -o /usr/local/bin/gosu.asc -SL "${BASEURL}/${GOSU_VERSION}/gosu-${DPKG_ARCH}.asc" \
     && export GNUPGHOME="$(mktemp -d)" \
-    && ( gpg2 --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 || \
+    && ( gpg2 --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 || \
+       gpg2 --batch --keyserver hkps://keyserver.ubuntu.com:443 --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 || \
+       gpg2 --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 || \
        gpg2 --batch --keyserver hkp://ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 || \
        gpg2 --batch --keyserver pgp.mit.edu --recv-key B42F6819007F00F88E364FD4036A9C25BF357DD4 || \
        gpg2 --batch --keyserver keyserver.pgp.com --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 || \

contrib/ci-horizen/dockerfiles/amd64/linux/ubuntu_bionic/entrypoint.sh

@@ -18,10 +18,16 @@ if [ ! "$USER_ID" == "$LOCAL_UID" ] || [ ! "$GRP_ID" == "$LOCAL_GID" ]; then
     echo "Warning: User zenbuilder with differing UID $LOCAL_UID/GID $LOCAL_GID already exists, most likely this container was started before with a different UID/GID. Re-create it to change UID/GID."
 fi
 
-echo "Starting with UID/GID: $LOCAL_UID:$LOCAL_GID"
-
 export HOME=/home/zenbuilder
 
+gcc -v
+echo
+lscpu
+echo
+free -h
+echo
+echo "Username: zenbuilder, HOME: $HOME, UID: $LOCAL_UID, GID: $LOCAL_GID"
+
 # Mount host directories
 for dir in .ccache .zcash-params build; do
     if [ -d "/mnt/${dir}" ]; then

contrib/ci-horizen/dockerfiles/amd64/linux/ubuntu_xenial/Dockerfile → contrib/ci-horizen/dockerfiles/amd64/linux/ubuntu_focal/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:xenial
+FROM ubuntu:focal
 
 MAINTAINER [email protected]
 
@@ -7,22 +7,24 @@ COPY entrypoint.sh /usr/local/bin/entrypoint.sh
 SHELL ["/bin/bash", "-c"]
 
 RUN set -euxo pipefail \
-    && DEBIAN_FRONTEND=noninteractive \
+    && export DEBIAN_FRONTEND=noninteractive \
     && apt-get update \
     && apt-get -y --no-install-recommends install apt-utils \
     && apt-get -y --no-install-recommends dist-upgrade \
     && apt-get -y --no-install-recommends install aria2 autoconf automake bsdmainutils build-essential \
        ca-certificates cmake curl dirmngr fakeroot git g++-multilib gnupg2 help2man libc6-dev libgomp1 \
-       libtool lintian m4 ncurses-dev pigz pkg-config pv python python-dev python-pip python-setuptools \
-       python-wheel python-wheel-common python-zmq unzip wget zlib1g-dev \
-    && pip install b2==1.4.2 pyblake2 \
+       libtool lintian m4 ncurses-dev pigz pkg-config pv python2-dev python-setuptools time unzip wget zlib1g-dev \
+    && curl -s https://bootstrap.pypa.io/pip/2.7/get-pip.py | python2 \
+    && pip2 install b2==1.4.2 pyblake2 pyzmq websocket-client2 \
     && BASEURL="https://github.com/tianon/gosu/releases/download/" \
-    && GOSU_VERSION="1.11" \
+    && GOSU_VERSION="1.13" \
     && DPKG_ARCH="$(dpkg --print-architecture | awk -F- '{ print $NF }')" \
     && curl -o /usr/local/bin/gosu -SL "${BASEURL}/${GOSU_VERSION}/gosu-${DPKG_ARCH}" \
     && curl -o /usr/local/bin/gosu.asc -SL "${BASEURL}/${GOSU_VERSION}/gosu-${DPKG_ARCH}.asc" \
     && export GNUPGHOME="$(mktemp -d)" \
-    && ( gpg2 --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 || \
+    && ( gpg2 --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 || \
+       gpg2 --batch --keyserver hkps://keyserver.ubuntu.com:443 --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 || \
+       gpg2 --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 || \
        gpg2 --batch --keyserver hkp://ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 || \
        gpg2 --batch --keyserver pgp.mit.edu --recv-key B42F6819007F00F88E364FD4036A9C25BF357DD4 || \
        gpg2 --batch --keyserver keyserver.pgp.com --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 || \

contrib/ci-horizen/dockerfiles/arm64/linux/ubuntu_xenial/entrypoint.sh → contrib/ci-horizen/dockerfiles/amd64/linux/ubuntu_focal/entrypoint.sh

@@ -18,10 +18,16 @@ if [ ! "$USER_ID" == "$LOCAL_UID" ] || [ ! "$GRP_ID" == "$LOCAL_GID" ]; then
     echo "Warning: User zenbuilder with differing UID $LOCAL_UID/GID $LOCAL_GID already exists, most likely this container was started before with a different UID/GID. Re-create it to change UID/GID."
 fi
 
-echo "Starting with UID/GID: $LOCAL_UID:$LOCAL_GID"
-
 export HOME=/home/zenbuilder
 
+gcc -v
+echo
+lscpu
+echo
+free -h
+echo
+echo "Username: zenbuilder, HOME: $HOME, UID: $LOCAL_UID, GID: $LOCAL_GID"
+
 # Mount host directories
 for dir in .ccache .zcash-params build; do
     if [ -d "/mnt/${dir}" ]; then