More build updates

.github/workflows/wheels.yml

@@ -20,7 +20,7 @@ jobs:
       CIBW_BEFORE_ALL_LINUX: yum install -y swig || apk add swig
       CIBW_BEFORE_ALL_MACOS: brew install gnu-sed swig automake
       CIBW_BEFORE_ALL_WINDOWS: choco install swig --version=3.0.12 --no-progress --allow-downgrade -y
-      CIBW_BEFORE_BUILD_WINDOWS: .\tools\msvc\wheel_preamble.bat
+      CIBW_BEFORE_BUILD_WINDOWS: .\tools\msvc\swig.bat
       CIBW_REPAIR_WHEEL_COMMAND_LINUX: auditwheel repair --only-plat -w {dest_dir} {wheel}
       LIBWALLY_DIR: "."
       SWIG_PATH: "C:\\ProgramData\\chocolatey\\lib\\swig\\tools\\install\\swigwin-3.0.12"

.gitignore

@@ -78,6 +78,7 @@ docs/source/psbt_members.rst
 docs/source/script.rst
 docs/source/symmetric.rst
 docs/source/transaction.rst
+docs/source/transaction_members.rst
 docs/source/elements.rst
 docs/source/*_int.rst
 .idea/

README.md

@@ -114,34 +114,39 @@ installed.
 
 ### Python
 
-For non-development use, you can install wally with `pip` as follows:
+For non-development use, you can install wally from PyPI with `pip` as follows:
 
 ```
 pip install wallycore==0.9.2
 ```
 
-For python development, you can build and install wally using:
+For development, you can build and install wally using:
 
 ```
 $ pip install .
 ```
 
-It is suggested you only install this way into a virtualenv while the library
-is under heavy development.
-
 If you wish to explicitly choose the python version to use, set the
-`PYTHON_VERSION` environment variable (to e.g. `3`, `3.7` etc) before
+`PYTHON_VERSION` environment variable (to e.g. `3.9`, `3.10` etc) before
 running `pip` or (when compiling manually) `./configure`.
 
 You can also install the binary [wally releases](https://github.com/ElementsProject/libwally-core/releases)
-using the released wheel files without having to compile the library, e.g.:
+using the released wheel files, for example if you don't wish to install from PyPI over the network:
 
 ```
 pip install wallycore-<version_and_architecture>.whl
 ```
 
-The script `tools/build_python_manylinux_wheels.sh` builds the Linux release files
-and can be used as an example for your own python projects.
+Each wally release includes a signed `requirements.txt` file. It is strongly
+suggested that you verify and use this file when installing, with:
+
+```
+pip -r requirements.txt
+```
+
+Doing so ensures that the wheel you install is the version you expect and an
+official build. This will detect, for example, if PyPI is hacked and a
+malicious wallycore package uploaded.
 
 ### Android
 

configure.ac

@@ -12,6 +12,9 @@ AH_BOTTOM([#include "ccan_config.h"
 
 LDPATH_VAR=LD_LIBRARY_PATH
 case $host_os in
+cygwin*|mingw*)
+    is_win="yes"
+    ;;
 *darwin*)
     is_osx="yes"
     LDPATH_VAR=DYLD_LIBRARY_PATH
@@ -212,12 +215,6 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[int main(void){static int a[2];return *((int*)((
               have_unaligned=1, have_unaligned=0, have_unaligned=0)
 AC_DEFINE_UNQUOTED([HAVE_UNALIGNED_ACCESS], [$have_unaligned], [Define if we have unaligned access])
 
-if test "x$is_osx" != "xyes"; then
-    # Assume we are using gcc (i.e. have this attribute) if cross-compiling
-    AC_COMPILE_IFELSE([AC_LANG_SOURCE([extern int foo(int) __attribute__((weak)); int main(void){return 0;}])],
-                      [AC_DEFINE([HAVE_ATTRIBUTE_WEAK], 1, [Define if we have __attribute__((weak))])])
-fi
-
 AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <sys/mman.h>]],[[mmap(0,0,0,0,0,0)]])],
                [AC_DEFINE(HAVE_MMAP, 1, [Define if we have mmap])])
 
@@ -339,6 +336,9 @@ if test "x$swig_python" = "xyes"; then
     if test "x$pythonexists" != "xyes"; then
         AC_MSG_FAILURE([ERROR: No usable Python was found for swig-python])
     fi
+    if test "x$elements_abi" = "xno"; then
+        AC_MSG_FAILURE([ERROR: Python wrapper can not be enabled when Elements ABI is disabled])
+    fi
     SWIG_PYTHON
     AX_CHECK_COMPILE_FLAG([-DSWIG_PYTHON_BUILD=1], [AM_CFLAGS="$AM_CFLAGS -DSWIG_PYTHON_BUILD=1"])
 fi
@@ -349,6 +349,9 @@ AC_ARG_ENABLE(swig-java,
 AM_CONDITIONAL([USE_SWIG_JAVA], [test "x$swig_java" = "xyes"])
 
 if test "x$swig_java" = "xyes"; then
+    if test "x$elements_abi" = "xno"; then
+        AC_MSG_FAILURE([ERROR: Java wrapper can not be enabled when Elements ABI is disabled])
+    fi
     saved_JAVA_HOME=$JAVA_HOME
     if test x"$cross_compiling" = "xyes"; then
         # For cross compiling we assume the users host O/S Java install is not
@@ -389,12 +392,19 @@ AC_SUBST([JAVAC_TARGET])
 
 AC_SUBST([AM_CFLAGS])
 
+AM_CONDITIONAL([SHARED_BUILD_ENABLED], [test "x$enable_shared" = "xyes"])
 if test "x$enable_static" = "xyes"; then
     CTEST_EXTRA_STATIC='$(libwallycore_la_LIBADD)'
+    dnl Windows static builds require SECP256K1_STATIC to be defined.
+    dnl As a result, you can't build both a static (.lib) and dynamic (.dll)
+    dnl library with a single 'configure; make' invocation.
+    if test "x$is_win" = "xyes" -a "x$enable_shared" = "xyes"; then
+        AC_MSG_ERROR([Windows builds cannot build shared and static builds at the same time])
+    fi
+    AX_CHECK_COMPILE_FLAG([-DSECP256K1_STATIC=1], [AM_CFLAGS="$AM_CFLAGS -DSECP256K1_STATIC=1"])
 fi
 AC_SUBST([CTEST_EXTRA_STATIC])
 
-AM_CONDITIONAL([SHARED_BUILD_ENABLED], [test "x$enable_shared" = "xyes"])
 
 AC_CONFIG_FILES([
  Makefile

setup.py

@@ -40,49 +40,44 @@
     # Run the autotools/make build up front to generate our sources,
     # then build using the standard Python ext module machinery.
     # (Windows requires source generation to be done separately).
-    import multiprocessing
     import subprocess
 
     abs_path = os.path.dirname(os.path.abspath(__file__)) + '/'
 
-    def call(args):
-        subprocess.check_call(args, cwd=abs_path, env=configure_env)
+    def call(args, cwd=abs_path):
+        subprocess.check_call(args, cwd=cwd, env=configure_env)
 
     call(['./tools/cleanup.sh'])
     call(['./tools/autogen.sh'])
     call(['./configure'] + CONFIGURE_ARGS)
-    call(['make', '-j{}'.format(multiprocessing.cpu_count())])
+    call(['make', 'swig_python/swig_python_wrap.c'], abs_path + 'src/')
 
 define_macros=[
     ('SWIG_PYTHON_BUILD', None),
     ('WALLY_CORE_BUILD', None),
-    ('HAVE_CONFIG_H', None),
-    ('SECP256K1_BUILD', None),
-    ('BUILD_ELEMENTS', None)
+    ('BUILD_ELEMENTS', None),
     ]
-if is_windows:
-    define_macros.append(('USE_ECMULT_STATIC_PRECOMPUTATION', None))
-    define_macros.append(('ECMULT_WINDOW_SIZE', 15))
 
 include_dirs=[
     './',
     './src',
     './src/ccan',
     './src/secp256k1/include',
     ]
+
 if is_windows:
-    shutil.copyfile('./src/amalgamation/windows_config/libsecp256k1-config.h', 'src/secp256k1/src/libsecp256k1-config.h')
     include_dirs = ['./src/amalgamation/windows_config'] + include_dirs
-
-extra_compile_args = ['-flax-vector-conversions']
+    extra_compile_args = []
+else:
+    extra_compile_args = ['-flax-vector-conversions']
 
 wally_ext = Extension(
     '_wallycore',
     define_macros=define_macros,
     include_dirs=include_dirs,
     extra_compile_args=extra_compile_args,
     sources=[
-        'src/swig_python/swig_wrap.c' if is_windows else 'src/swig_python/swig_python_wrap.c',
+        'src/swig_python/swig_python_wrap.c',
         'src/amalgamation/combined.c',
         'src/amalgamation/combined_ccan.c',
         'src/amalgamation/combined_ccan2.c',

src/Makefile.am

@@ -24,6 +24,7 @@ include_HEADERS += $(top_srcdir)/include/wally_psbt_members.h
 include_HEADERS += $(top_srcdir)/include/wally_script.h
 include_HEADERS += $(top_srcdir)/include/wally_symmetric.h
 include_HEADERS += $(top_srcdir)/include/wally_transaction.h
+include_HEADERS += $(top_srcdir)/include/wally_transaction_members.h
 
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = wallycore.pc
@@ -177,6 +178,7 @@ libwallycore_la_SOURCES = \
 libwallycore_la_INCLUDES = \
     include/wally.hpp \
     include/wally_address.h \
+    include/wally_anti_exfil.h \
     include/wally_bip32.h \
     include/wally_bip38.h \
     include/wally_bip39.h \
@@ -188,9 +190,11 @@ libwallycore_la_INCLUDES = \
     include/wally_elements.h \
     include/wally_map.h \
     include/wally_psbt.h \
+    include/wally_psbt_members.h \
     include/wally_script.h \
     include/wally_symmetric.h \
-    include/wally_transaction.h
+    include/wally_transaction.h \
+    include/wally_transaction_members.h
 
 if SHARED_BUILD_ENABLED
 LT_VER_CURRENT = 0   # increment at every ABI change (whether breaking or non-breaking)

src/amalgamation/combined.c

@@ -1,4 +1,40 @@
-#define SECP256K1_BUILD 1
+/*
+ * secp2556k1-zkp configuration
+ */
+#define ENABLE_MODULE_ECDH 1
+#define ENABLE_MODULE_EXTRAKEYS 1
+#define ENABLE_MODULE_SCHNORRSIG 1
+#define ENABLE_MODULE_GENERATOR 1
+#define ENABLE_MODULE_ECDSA_S2C 1
+#ifdef BUILD_ELEMENTS
+#define ENABLE_MODULE_RANGEPROOF 1
+#define ENABLE_MODULE_RECOVERY 1
+#define ENABLE_MODULE_SURJECTIONPROOF 1
+#define ENABLE_MODULE_WHITELIST 1
+#endif
+
+#if (defined(__clang__) || defined(__GNUC__)) && (defined(__x86_64__) || defined(__amd64__))
+#define USE_ASM_X86_64 1
+#endif
+
+#undef PACKAGE
+#undef PACKAGE_BUGREPORT
+#undef PACKAGE_NAME
+#undef PACKAGE_STRING
+#undef PACKAGE_TARNAME
+#undef PACKAGE_URL
+#undef PACKAGE_VERSION
+#undef VERSION
+#if defined(__clang__)
+#pragma clang diagnostic ignored "-Wunused-function"
+#elif defined(__GNUC__)
+#pragma GCC diagnostic ignored "-Wunused-function"
+#endif
+#include "src/secp256k1/src/secp256k1.c"
+#include "src/secp256k1/src/precomputed_ecmult_gen.c"
+#include "src/secp256k1/src/precomputed_ecmult.c"
+#include "ccan/ccan/crypto/sha256/sha256.c"
+
 #include "internal.c"
 #include "address.c"
 #include "aes.c"
@@ -29,40 +65,11 @@
 #include "transaction.c"
 #include "wif.c"
 #include "wordlist.c"
-#undef PACKAGE
-#undef PACKAGE_BUGREPORT
-#undef PACKAGE_NAME
-#undef PACKAGE_STRING
-#undef PACKAGE_TARNAME
-#undef PACKAGE_URL
-#undef PACKAGE_VERSION
-#undef VERSION
-#undef SECP256K1_BUILD
-#include "src/secp256k1/src/secp256k1.c"
-#include "src/secp256k1/src/precomputed_ecmult_gen.c"
-#include "src/secp256k1/src/precomputed_ecmult.c"
-#include "ccan/ccan/crypto/sha256/sha256.c"
 
 void wally_silence_unused_warnings(void)
 {
     assert_sign_assumptions();
     assert_bip32_assumptions();
     assert_bip38_assumptions();
     assert_tx_assumptions();
-    secp256k1_fe_get_bounds(NULL, 0);
-    secp256k1_fe_inv_var(NULL, NULL);
-    secp256k1_ge_set_all_gej_var(NULL, NULL, 0);
-    secp256k1_gej_has_quad_y_var(NULL);
-    secp256k1_ge_is_valid_var(NULL);
-    secp256k1_ge_set_infinity(NULL);
-    secp256k1_ec_commit_verify(NULL, NULL, NULL, NULL, 0);
-    secp256k1_ecmult_multi_var(NULL, NULL, NULL, NULL, NULL, NULL, 0);
-    secp256k1_ecmult_strauss_batch_single(NULL, NULL, NULL, NULL, NULL, NULL, 0);
-    secp256k1_ecmult_pippenger_batch_single(NULL, NULL, NULL, NULL, NULL, NULL, 0);
-    secp256k1_pippenger_scratch_size(0, 0);
-    secp256k1_scalar_chacha20(NULL, NULL, NULL, 0);
-    secp256k1_sha256_initialize_tagged(NULL, NULL, 0);
-    tx_elements_input_issuance_proof_init(NULL, NULL, 0, NULL, 0);
-    tx_elements_output_proof_init(NULL, NULL, 0, NULL, 0);
-    witness_stack_from_bytes(NULL, NULL, NULL);
 }

src/amalgamation/windows_config/config.h

@@ -1,36 +1,19 @@
 #ifndef LIBWALLYCORE_CONFIG_H
 #define LIBWALLYCORE_CONFIG_H
+
+/* config.h for Windows. Assumes a little-endian intel-ish target */
 #include <stddef.h>
 
-#define HAVE_ATTRIBUTE_WEAK 1
-#define HAVE_BIG_ENDIAN 0
-#define HAVE_BSWAP_64 0
-#define HAVE_BYTESWAP_H 0
-#define HAVE_DLFCN_H 1
-#define HAVE_INTTYPES_H 1
-#define HAVE_LITTLE_ENDIAN 1
-#define HAVE_MEMORY_H 1
-#define HAVE_MMAP 1
-#define HAVE_PTHREAD 1
-#define HAVE_PTHREAD_PRIO_INHERIT 1
-#define HAVE_PYTHON "2.7"
-#define HAVE_STDINT_H 1
-#define HAVE_STDLIB_H 1
-#define HAVE_STRINGS_H 1
-#define HAVE_STRING_H 1
-#define HAVE_SYS_STAT_H 1
-#define HAVE_SYS_TYPES_H 1
-#define HAVE_UNISTD_H 1
-#define STDC_HEADERS 1
-#define VERSION "0.6"
-#if (!defined(_SSIZE_T_DECLARED)) && (!defined(_ssize_t)) && (!defined(ssize_t))
-#define ssize_t long long
+#ifndef _WIN32
+#error windows_config/config.h is only intended for windows builds
 #endif
 
-#define alignment_ok(p, n) ((size_t)(p) % (n) == 0)
+#define HAVE_UNALIGNED_ACCESS 1
 
-void wally_clear(void *p, size_t len);
+#if (!defined(_SSIZE_T_DECLARED)) && (!defined(_ssize_t)) && (!defined(ssize_t))
+#define ssize_t long long
+#endif
 
-#define CCAN_CLEAR_MEMORY(p, len) wally_clear(p, len)
+#include "ccan_config.h"
 
-#endif /*LIBWALLYCORE_CONFIG_H*/
+#endif /* LIBWALLYCORE_CONFIG_H */

src/ecdh.c

@@ -1,6 +1,5 @@
 #include "internal.h"
 #include <include/wally_crypto.h>
-#include <secp256k1.h>
 #include <secp256k1_ecdh.h>
 
 int wally_ecdh(const unsigned char *pub_key, size_t pub_key_len,