Integrate ain-grpc and metachain

* Import libain work * Prioritize aarch64 for M1 Mac * CI build changes * Add customtx and visitor impl * Invoke metachain in defid * Cross-compilation fixes for OSX and Windows * Local path for depends builds * DMC handler for FFI and RPC * Test framework update * Fix MC startup and lambda return * Support moving funds back and forth
DeFiCh · Nov 30, 2022 · 9a137e1 · 9a137e1
1 parent 8d233a4
commit 9a137e1
Show file tree

Hide file tree

Showing 38 changed files with 726 additions and 18 deletions.
diff --git a/.gitignore b/.gitignore
@@ -10,6 +10,12 @@ build
 *.tmp
 .DS_Store
 
+# Codegen
+libain_rpc.cpp
+libain_rpc.h
+libmc.cpp
+libmc.h
+
 # Build directory.
 build/
 out/

diff --git a/.python-version b/.python-version
diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh
@@ -27,7 +27,7 @@ export BASE_BUILD_DIR=${BASE_BUILD_DIR:-${TRAVIS_BUILD_DIR:-$BASE_ROOT_DIR}}
 export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out/$HOST}
 export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
 export WINEDEBUG=${WINEDEBUG:-fixme-all}
-export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3}
+export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 git unzip}
 export GOAL=${GOAL:-install}
 export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_BUILD_DIR}/qa-assets}
 export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH

diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh
@@ -49,3 +49,15 @@ DOCKER_EXEC echo "Number of CPUs \(nproc\): $(nproc)"
 ${CI_RETRY_EXE} DOCKER_EXEC apt-get update
 ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES
 
+# Install Rust toolchain
+DOCKER_EXEC "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y"
+DOCKER_EXEC ln -s /root/.cargo/bin/cargo /usr/bin/cargo
+DOCKER_EXEC ln -s /root/.cargo/bin/rustc /usr/bin/rustc
+DOCKER_EXEC ln -s /root/.cargo/bin/rustup /usr/bin/rustup
+DOCKER_EXEC rustup target add x86_64-unknown-linux-gnu
+
+DOCKER_EXEC "curl -L https://github.com/protocolbuffers/protobuf/releases/download/v3.20.0/protoc-3.20.0-linux-x86_64.zip -o /tmp/proto.zip"
+DOCKER_EXEC "unzip -o /tmp/proto.zip -d /tmp/proto"
+DOCKER_EXEC "chmod 755 -R /tmp/proto/bin"
+DOCKER_EXEC "cp /tmp/proto/bin/protoc /usr/local/bin/"
+DOCKER_EXEC "cp -R /tmp/proto/include/* /usr/local/include/"
diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh
@@ -25,3 +25,5 @@ fi
 if [ -z "$NO_DEPENDS" ]; then
   DOCKER_EXEC CONFIG_SHELL= make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS
 fi
+
+TARGET=${HOST} ./make.sh patch_codegen
diff --git a/configure.ac b/configure.ac
@@ -504,6 +504,8 @@ case $host in
      AC_CHECK_LIB([shlwapi],      [main],, AC_MSG_ERROR(libshlwapi missing))
      AC_CHECK_LIB([iphlpapi],      [main],, AC_MSG_ERROR(libiphlpapi missing))
      AC_CHECK_LIB([crypt32],      [main],, AC_MSG_ERROR(libcrypt32 missing))
+     AC_CHECK_LIB([bcrypt],      [main],, AC_MSG_ERROR(libbcrypt missing))
+     AC_CHECK_LIB([userenv],      [main],, AC_MSG_ERROR(libuserenv missing))
 
      # -static is interpreted by libtool, where it has a different meaning.
      # In libtool-speak, it's -all-static.
@@ -620,6 +622,9 @@ case $host in
      ;;
 esac
 
+AC_CHECK_LIB([ain_grpc],[main],AIN_GRPC_LIBS=-lain_grpc,AC_MSG_ERROR(libain_grpc missing))
+AC_CHECK_LIB([metachain],[main],METACHAIN_LIBS=-lmetachain,AC_MSG_ERROR(libmetachain missing))
+
 if test x$use_pkgconfig = xyes; then
   m4_ifndef([PKG_PROG_PKG_CONFIG], [AC_MSG_ERROR(PKG_PROG_PKG_CONFIG macro not found. Please install pkg-config and re-run autogen.sh.)])
   m4_ifdef([PKG_PROG_PKG_CONFIG], [
@@ -1417,7 +1422,9 @@ fi
 
 AM_CONDITIONAL([TARGET_DARWIN], [test x$TARGET_OS = xdarwin])
 AM_CONDITIONAL([BUILD_DARWIN], [test x$BUILD_OS = xdarwin])
+AM_CONDITIONAL([BUILD_NOT_DARWIN], [test x$BUILD_OS != xdarwin])
 AM_CONDITIONAL([TARGET_WINDOWS], [test x$TARGET_OS = xwindows])
+AM_CONDITIONAL([TARGET_NOT_WINDOWS], [test x$TARGET_OS != xwindows])
 AM_CONDITIONAL([ENABLE_WALLET],[test x$enable_wallet = xyes])
 AM_CONDITIONAL([ENABLE_TESTS],[test x$BUILD_TEST = xyes])
 AM_CONDITIONAL([ENABLE_FUZZ],[test x$enable_fuzz = xyes])
@@ -1487,6 +1494,8 @@ AC_SUBST(MINIUPNPC_CPPFLAGS)
 AC_SUBST(MINIUPNPC_LIBS)
 AC_SUBST(EVENT_LIBS)
 AC_SUBST(EVENT_PTHREADS_LIBS)
+AC_SUBST(AIN_GRPC_LIBS)
+AC_SUBST(METACHAIN_LIBS)
 AC_SUBST(ZMQ_LIBS)
 AC_SUBST(QR_LIBS)
 AC_SUBST(HAVE_GMTIME_R)

diff --git a/contrib/dockerfiles/x86_64-apple-darwin18.dockerfile b/contrib/dockerfiles/x86_64-apple-darwin18.dockerfile
@@ -14,10 +14,22 @@ RUN apt update && apt dist-upgrade -y
 RUN apt install -y software-properties-common build-essential git libtool autotools-dev automake \
 pkg-config bsdmainutils python3 libssl-dev libevent-dev libboost-system-dev \
 libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev \
-libminiupnpc-dev libzmq3-dev libqrencode-dev \
-curl cmake \
+libminiupnpc-dev libzmq3-dev libqrencode-dev curl cmake unzip \
 python3-dev python3-pip libcap-dev libbz2-dev libz-dev fonts-tuffy librsvg2-bin libtiff-tools imagemagick
 
+# install protobuf
+RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.20.0/protoc-3.20.0-linux-x86_64.zip
+RUN unzip -o protoc-3.20.0-linux-x86_64.zip -d ./proto
+RUN chmod 755 -R ./proto/bin
+RUN cp ./proto/bin/protoc /usr/local/bin/
+RUN cp -R ./proto/include/* /usr/local/include/
+
+# install rustlang
+RUN curl https://sh.rustup.rs -sSf | \
+    sh -s -- --default-toolchain stable -y
+ENV PATH=/root/.cargo/bin:$PATH
+RUN rustup target add x86_64-apple-darwin
+
 # For Berkeley DB - but we don't need as we do a depends build.
 # RUN apt install -y libdb-dev
 
@@ -43,6 +55,7 @@ WORKDIR /work
 COPY --from=depends-builder /work/depends ./depends
 COPY . .
 
+RUN ./make.sh patch_codegen ${TARGET}
 RUN ./autogen.sh
 
 # XREF: #make-configure

diff --git a/contrib/dockerfiles/x86_64-pc-linux-gnu.dockerfile b/contrib/dockerfiles/x86_64-pc-linux-gnu.dockerfile
@@ -12,6 +12,19 @@ COPY ./make.sh .
 RUN apt update && apt install -y apt-transport-https
 RUN export DEBIAN_FRONTEND=noninteractive && ./make.sh pkg-install-deps-x86_64
 
+# install protobuf
+RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.20.0/protoc-3.20.0-linux-x86_64.zip
+RUN unzip -o protoc-3.20.0-linux-x86_64.zip -d ./proto
+RUN chmod 755 -R ./proto/bin
+RUN cp ./proto/bin/protoc /usr/local/bin/
+RUN cp -R ./proto/include/* /usr/local/include/
+
+# install rustlang
+RUN curl https://sh.rustup.rs -sSf | \
+    sh -s -- --default-toolchain stable -y
+ENV PATH=/root/.cargo/bin:$PATH
+RUN rustup target add x86_64-unknown-linux-gnu
+
 # -----------
 FROM builder-base as depends-builder
 ARG TARGET
@@ -36,7 +49,7 @@ WORKDIR /work
 COPY . .
 RUN ./make.sh purge && rm -rf ./depends
 COPY --from=depends-builder /work/depends ./depends
-
+RUN ./make.sh patch_codegen ${TARGET}
 RUN export MAKE_COMPILER="CC=gcc CXX=g++" && \
     ./make.sh build-conf && ./make.sh build-make
 

diff --git a/contrib/dockerfiles/x86_64-w64-mingw32.dockerfile b/contrib/dockerfiles/x86_64-w64-mingw32.dockerfile
@@ -14,10 +14,26 @@ RUN apt update && apt dist-upgrade -y
 RUN apt install -y software-properties-common build-essential git libtool autotools-dev automake \
 pkg-config bsdmainutils python3 libssl-dev libevent-dev libboost-system-dev \
 libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev \
-libminiupnpc-dev libzmq3-dev libqrencode-dev \
-curl cmake \
+libminiupnpc-dev libzmq3-dev libqrencode-dev curl cmake unzip \
 g++-mingw-w64-x86-64 mingw-w64-x86-64-dev nsis
 
+# install protobuf
+RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.20.0/protoc-3.20.0-linux-x86_64.zip
+RUN unzip -o protoc-3.20.0-linux-x86_64.zip -d ./proto
+RUN chmod 755 -R ./proto/bin
+RUN cp ./proto/bin/protoc /usr/local/bin/
+RUN cp -R ./proto/include/* /usr/local/include/
+
+# install rustlang
+RUN curl https://sh.rustup.rs -sSf | \
+    sh -s -- --default-toolchain stable -y
+ENV PATH=/root/.cargo/bin:$PATH
+RUN rustup target add x86_64-pc-windows-gnu
+RUN printf '\n\
+    [target.x86_64-pc-windows-gnu]\n\
+    linker = "x86_64-w64-mingw32-gcc"\n\
+    ar = "x86_64-w64-mingw32-ar"\n' > /root/.cargo/config
+
 # Set the default mingw32 g++ compiler option to posix.
 RUN update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix
 

diff --git a/depends/Makefile b/depends/Makefile
@@ -96,7 +96,7 @@ zmq_packages_$(NO_ZMQ) = $(zmq_packages)
 
 rapidcheck_packages_$(RAPIDCHECK) = $(rapidcheck_packages)
 
-packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(wallet_packages_) $(upnp_packages_)
+packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(wallet_packages_) $(upnp_packages_) $(libmc_packages) $(libain_packages)
 native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages)
 
 ifneq ($(zmq_packages_),)

diff --git a/depends/packages/libain.mk b/depends/packages/libain.mk
@@ -0,0 +1,41 @@
+package=libain
+$(package)_version=master
+$(package)_git_path=https://github.com/DeFiCh/libain-rs
+
+define $(package)_fetch_cmds
+	if [ ! -z ${LIBAIN_PATH} ]; then \
+		echo "Using local path for libain-rs"; \
+		mkdir -p $$(base_build_dir)/$$(host)/$(package); \
+		ln -s $(LIBAIN_PATH) $$($(package)_extract_dir); \
+	else \
+		echo "Cloning from Git repository for libain-rs"; \
+		git clone --single-branch --branch master $$($(package)_git_path) $$($(package)_extract_dir); \
+	fi
+endef
+
+define $(package)_extract_cmds
+echo "Skipping extraction for libain source"
+endef
+
+# Perform target substitutions for Rust
+LIBAIN_TARGET := $(HOST)
+ifeq ($(LIBAIN_TARGET),x86_64-pc-linux-gnu)
+	LIBAIN_TARGET=x86_64-unknown-linux-gnu
+endif
+ifeq ($(LIBAIN_TARGET),arm-linux-gnueabihf)
+	LIBAIN_TARGET=arm-unknown-linux-gnueabihf
+endif
+ifeq ($(LIBAIN_TARGET),x86_64-apple-darwin18)
+	LIBAIN_TARGET=aarch64-apple-darwin
+endif
+ifeq ($(LIBAIN_TARGET),x86_64-w64-mingw32)
+	LIBAIN_TARGET=x86_64-pc-windows-gnu
+endif
+
+define $(package)_build_cmds
+	$(MAKE) build-grpc-pkg TARGET=$(LIBAIN_TARGET)
+endef
+
+define $(package)_stage_cmds
+	cp -r pkg/ain-grpc/* $($(package)_staging_dir)$(host_prefix)/
+endef
diff --git a/depends/packages/libmc.mk b/depends/packages/libmc.mk
@@ -0,0 +1,41 @@
+package=libmc
+$(package)_version=master
+$(package)_git_path=https://github.com/DeFiCh/metachain
+
+define $(package)_fetch_cmds
+	if [ ! -z ${LIBMC_PATH} ]; then \
+		echo "Using local path for metachain"; \
+		mkdir -p $$(base_build_dir)/$$(host)/$(package); \
+		ln -s $(LIBMC_PATH) $$($(package)_extract_dir); \
+	else \
+		echo "Cloning from Git repository for metachain"; \
+		git clone --single-branch --branch libmc $$($(package)_git_path) $$($(package)_extract_dir); \
+	fi
+endef
+
+define $(package)_extract_cmds
+echo "Skipping extraction for metachain source"
+endef
+
+# Perform target substitutions for Rust
+LIBMC_TARGET := $(HOST)
+ifeq ($(LIBMC_TARGET),x86_64-pc-linux-gnu)
+	LIBMC_TARGET=x86_64-unknown-linux-gnu
+endif
+ifeq ($(LIBMC_TARGET),arm-linux-gnueabihf)
+	LIBMC_TARGET=arm-unknown-linux-gnueabihf
+endif
+ifeq ($(LIBMC_TARGET),x86_64-apple-darwin18)
+	LIBMC_TARGET=aarch64-apple-darwin
+endif
+ifeq ($(LIBMC_TARGET),x86_64-w64-mingw32)
+	LIBMC_TARGET=x86_64-pc-windows-gnu
+endif
+
+define $(package)_build_cmds
+	$(MAKE) build-native-pkg TARGET=$(LIBMC_TARGET)
+endef
+
+define $(package)_stage_cmds
+	cp -r pkg/metachain/* $($(package)_staging_dir)$(host_prefix)/
+endef
diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk
@@ -8,6 +8,10 @@ zmq_packages=zeromq
 
 upnp_packages=miniupnpc
 
+libain_packages = libain
+
+libmc_packages = libmc
+
 darwin_native_packages = native_biplist native_ds_store native_mac_alias
 
 ifneq ($(build_os),darwin)

diff --git a/make.sh b/make.sh
@@ -112,6 +112,21 @@ build_deps() {
     popd >/dev/null
 }
 
+patch_codegen() {
+    local target=${1:-${TARGET}}
+    # Required for patching C++ glue emitted from Rust build
+    printf "#ifndef DEFI_RUST_RPC_H\n#define DEFI_RUST_RPC_H\n\n" > "$(pwd)/src/libain_rpc.h"
+    cat "$(pwd)/depends/${target}/include/libain_rpc.hpp" >> "$(pwd)/src/libain_rpc.h"
+    printf "\n#endif // DEFI_RUST_RPC_H" >> "$(pwd)/src/libain_rpc.h"
+    printf "#include <libain_rpc.h>\n" > "$(pwd)/src/libain_rpc.cpp"
+    cat "$(pwd)/depends/${target}/libain_rpc.cpp" >> "$(pwd)/src/libain_rpc.cpp"
+    printf "#ifndef DEFI_METACHAIN_H\n#define DEFI_METACHAIN_H\n\n" > "$(pwd)/src/libmc.h"
+    cat "$(pwd)/depends/${target}/include/libmc.hpp" >> "$(pwd)/src/libmc.h"
+    printf "\n#endif // DEFI_METACHAIN_H" >> "$(pwd)/src/libmc.h"
+    printf "#include <libmc.h>\n" > "$(pwd)/src/libmc.cpp"
+    cat "$(pwd)/depends/${target}/libmc.cpp" >> "$(pwd)/src/libmc.cpp"
+}
+
 build_conf() {
     local target=${1:-${TARGET}}
     local make_conf_opts=${MAKE_CONF_ARGS:-}
@@ -138,6 +153,7 @@ build_make() {
 
 build() {
     build_deps "$@"
+    patch_codegen "$@"
     build_conf "$@"
     build_make "$@"
 }
@@ -389,7 +405,7 @@ pkg_install_deps_x86_64() {
         libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev \
         libminiupnpc-dev libzmq3-dev libqrencode-dev wget \
         libdb-dev libdb++-dev libdb5.3 libdb5.3-dev libdb5.3++ libdb5.3++-dev \
-        curl cmake
+        curl cmake unzip
 }
 
 pkg_install_llvm() {

diff --git a/src/Makefile.am b/src/Makefile.am
@@ -372,6 +372,12 @@ libdefi_server_a_SOURCES = \
   index/txindex.cpp \
   interfaces/chain.cpp \
   init.cpp \
+  libain_rpc.h \
+  libain_rpc.cpp \
+  libmc.h \
+  libmc.cpp \
+  dmc_handler.h \
+  dmc_handler.cpp \
   dbwrapper.cpp \
   masternodes/accounts.cpp \
   masternodes/accountshistory.cpp \
@@ -672,9 +678,20 @@ nodist_libdefi_util_a_SOURCES = $(srcdir)/obj/build.h
 defid_SOURCES = defid.cpp
 defid_CPPFLAGS = $(AM_CPPFLAGS) $(DEFI_INCLUDES)
 defid_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
+# FIXME(wafflespeanut): Allowing multiple definitions because symbols conflict across Rust static libs
 defid_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)
+if TARGET_DARWIN
+defid_LDFLAGS += -framework Security -framework SystemConfiguration -framework CoreFoundation
+endif
+if BUILD_NOT_DARWIN
+defid_LDFLAGS += -Wl,--allow-multiple-definition
+endif
+if TARGET_NOT_WINDOWS
+defid_LDFLAGS += -lain_grpc -lmetachain -ldl
+endif
 
 if TARGET_WINDOWS
+defid_LDFLAGS += -luserenv -lbcrypt
 defid_SOURCES += defid-res.rc
 endif
 
@@ -683,7 +700,6 @@ endif
 defid_LDADD = \
   $(LIBDEFI_SERVER) \
   $(LIBDEFI_WALLET) \
-  $(LIBDEFI_SERVER) \
   $(LIBDEFI_COMMON) \
   $(LIBUNIVALUE) \
   $(LIBDEFI_SPV) \
@@ -695,7 +711,7 @@ defid_LDADD = \
   $(LIBMEMENV) \
   $(LIBSECP256K1)
 
-defid_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS)
+defid_LDADD +=$(AIN_GRPC_LIBS) $(METACHAIN_LIBS) $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS)
 
 # defi-cli binary #
 defi_cli_SOURCES = defi-cli.cpp

diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include
@@ -74,8 +74,20 @@ bench_bench_defi_SOURCES += bench/coin_selection.cpp
 bench_bench_defi_SOURCES += bench/wallet_balance.cpp
 endif
 
-bench_bench_defi_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS)
+bench_bench_defi_LDADD += $(AIN_GRPC_LIBS) $(METACHAIN_LIBS) $(BOOST_LIBS) $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS)
 bench_bench_defi_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)
+if TARGET_DARWIN
+bench_bench_defi_LDFLAGS += -framework Security -framework SystemConfiguration -framework CoreFoundation
+endif
+if BUILD_NOT_DARWIN
+bench_bench_defi_LDFLAGS += -Wl,--allow-multiple-definition
+endif
+if TARGET_NOT_WINDOWS
+bench_bench_defi_LDFLAGS += -lain_grpc -lmetachain -ldl
+endif
+if TARGET_WINDOWS
+bench_bench_defi_LDFLAGS += -luserenv -lbcrypt
+endif
 
 CLEAN_DEFI_BENCH = bench/*.gcda bench/*.gcno $(GENERATED_BENCH_FILES)