depends: Update from upstream

.python-version

@@ -0,0 +1 @@
+3.5.6

.travis.yml

@@ -48,7 +48,7 @@ jobs:
       sudo: false
       cache: false
       language: python
-      python: '3.6'
+      python: '3.5' # Oldest supported version according to doc/dependencies.md
       install:
         - set -o errexit; source .travis/lint_04_install.sh
       before_script:
@@ -141,7 +141,7 @@ jobs:
     - stage: test
       env: >-
         HOST=x86_64-apple-darwin14
-        PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools-git"
+        PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools"
         OSX_SDK=10.11
         RUN_UNIT_TESTS=false
         RUN_FUNCTIONAL_TESTS=false

configure.ac

@@ -81,8 +81,8 @@ AC_PATH_TOOL(RANLIB, ranlib)
 AC_PATH_TOOL(STRIP, strip)
 AC_PATH_TOOL(GCOV, gcov)
 AC_PATH_PROG(LCOV, lcov)
-dnl Python 3.x is supported from 3.4 on (see https://github.com/bitcoin/bitcoin/issues/7893)
-AC_PATH_PROGS([PYTHON], [python3.7 python3.6 python3.5 python3.4 python3 python])
+dnl Python 3.5 is specified in .python-version and should be used if available, see doc/dependencies.md
+AC_PATH_PROGS([PYTHON], [python3.5 python3.6 python3.7 python3.8 python3 python])
 AC_PATH_PROG(GENHTML, genhtml)
 AC_PATH_PROG([GIT], [git])
 AC_PATH_PROG(CCACHE,ccache)

depends/Makefile

@@ -6,6 +6,7 @@ BASE_CACHE ?= $(BASEDIR)/built
 SDK_PATH ?= $(BASEDIR)/SDKs
 NO_QT ?=
 NO_WALLET ?=
+NO_ZMQ ?=
 NO_UPNP ?=
 FALLBACK_DOWNLOAD_PATH ?= https://bitcoincore.org/depends-sources
 
@@ -14,7 +15,7 @@ HOST ?= $(BUILD)
 PATCHES_PATH = $(BASEDIR)/patches
 BASEDIR = $(CURDIR)
 HASH_LENGTH:=11
-DOWNLOAD_CONNECT_TIMEOUT:=10
+DOWNLOAD_CONNECT_TIMEOUT:=30
 DOWNLOAD_RETRIES:=3
 HOST_ID_SALT ?= salt
 BUILD_ID_SALT ?= salt
@@ -92,6 +93,7 @@ $(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null)
 qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch)_$(host_os)_packages)
 wallet_packages_$(NO_WALLET) = $(wallet_packages)
 upnp_packages_$(NO_UPNP) = $(upnp_packages)
+zmq_packages_$(NO_ZMQ) = $(zmq_packages)
 
 packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_)
 native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages)
@@ -100,6 +102,10 @@ ifneq ($(qt_packages_),)
 native_packages += $(qt_native_packages)
 endif
 
+ifneq ($(zmq_packages_),)
+packages += $(zmq_packages)
+endif
+
 all_packages = $(packages) $(native_packages)
 
 meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk
@@ -136,6 +142,7 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_
             -e 's|@LDFLAGS@|$(strip $(host_LDFLAGS) $(host_$(release_type)_LDFLAGS))|' \
             -e 's|@allow_host_packages@|$(ALLOW_HOST_PACKAGES)|' \
             -e 's|@no_qt@|$(NO_QT)|' \
+            -e 's|@no_zmq@|$(NO_ZMQ)|' \
             -e 's|@no_wallet@|$(NO_WALLET)|' \
             -e 's|@no_upnp@|$(NO_UPNP)|' \
             -e 's|@debug@|$(DEBUG)|' \
@@ -185,4 +192,6 @@ download-win:
 	@$(MAKE) -s HOST=x86_64-w64-mingw32 download-one
 download: download-osx download-linux download-win
 
+$(foreach package,$(all_packages),$(eval $(call ext_add_stages,$(package))))
+
 .PHONY: install cached clean clean-all download-one download-osx download-linux download-win download check-packages check-sources

depends/README.md

@@ -34,7 +34,7 @@ No other options are needed, the paths are automatically configured.
 
 #### For macOS cross compilation
 
-    sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils cmake imagemagick libcap-dev libz-dev libbz2-dev python-setuptools
+    sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils cmake imagemagick libcap-dev libz-dev libbz2-dev python3-setuptools
 
 #### For Win32/Win64 cross compilation
 
@@ -44,7 +44,7 @@ No other options are needed, the paths are automatically configured.
 
 Common linux dependencies:
 
-    sudo apt-get install make automake cmake curl g++-multilib libtool binutils-gold bsdmainutils pkg-config python3
+    sudo apt-get install make automake cmake curl g++-multilib libtool binutils-gold bsdmainutils pkg-config python3 patch
 
 For linux ARM cross compilation:
 
@@ -69,6 +69,7 @@ The following can be set when running make: make FOO=bar
     SDK_PATH: Path where sdk's can be found (used by macOS)
     FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up
     NO_QT: Don't download/build/cache qt and its dependencies
+    NO_ZMQ: Don't download/build/cache packages needed for enabling zeromq
     NO_WALLET: Don't download/build/cache libs needed to enable the wallet
     NO_UPNP: Don't download/build/cache packages needed for enabling upnp
     DEBUG: disable some optimizations and enable more runtime checking

depends/builders/darwin.mk

@@ -1,13 +1,13 @@
-build_darwin_CC: = $(shell xcrun -f clang)
-build_darwin_CXX: = $(shell xcrun -f clang++)
-build_darwin_AR: = $(shell xcrun -f ar)
-build_darwin_RANLIB: = $(shell xcrun -f ranlib)
-build_darwin_STRIP: = $(shell xcrun -f strip)
-build_darwin_OTOOL: = $(shell xcrun -f otool)
-build_darwin_NM: = $(shell xcrun -f nm)
+build_darwin_CC:=$(shell xcrun -f clang)
+build_darwin_CXX:=$(shell xcrun -f clang++)
+build_darwin_AR:=$(shell xcrun -f ar)
+build_darwin_RANLIB:=$(shell xcrun -f ranlib)
+build_darwin_STRIP:=$(shell xcrun -f strip)
+build_darwin_OTOOL:=$(shell xcrun -f otool)
+build_darwin_NM:=$(shell xcrun -f nm)
 build_darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool)
-build_darwin_SHA256SUM = shasum -a 256
-build_darwin_DOWNLOAD = curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o
+build_darwin_SHA256SUM=shasum -a 256
+build_darwin_DOWNLOAD=curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o
 
 #darwin host on darwin builder. overrides darwin host preferences.
 darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION)

depends/config.site.in

@@ -33,6 +33,10 @@ if test -z $with_gui && test -n "@no_qt@"; then
   with_gui=no
 fi
 
+if test -z $enable_zmq && test -n "@no_zmq@"; then
+  enable_zmq=no
+fi
+
 if test x@host_os@ = xdarwin; then
   BREW=no
   PORT=no
@@ -67,7 +71,7 @@ fi
 if test -n "@CXX@" -a -z "${CXX}"; then
   CXX="@CXX@"
 fi
-PYTHONPATH=$depends_prefix/native/lib/python/dist-packages:$PYTHONPATH
+PYTHONPATH=$depends_prefix/native/lib/python3/dist-packages:$PYTHONPATH
 
 if test -n "@AR@"; then
   AR=@AR@

depends/funcs.mk

@@ -76,8 +76,9 @@ $(1)_download_path_fixed=$(subst :,\:,$$($(1)_download_path))
 
 
 #default commands
+# The default behavior for tar will try to set ownership when running as uid 0 and may not succeed, --no-same-owner disables this behavior
 $(1)_fetch_cmds ?= $(call fetch_file,$(1),$(subst \:,:,$$($(1)_download_path_fixed)),$$($(1)_download_file),$($(1)_file_name),$($(1)_sha256_hash))
-$(1)_extract_cmds ?= mkdir -p $$($(1)_extract_dir) && echo "$$($(1)_sha256_hash)  $$($(1)_source)" > $$($(1)_extract_dir)/.$$($(1)_file_name).hash &&  $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$$($(1)_file_name).hash && tar --strip-components=1 -xf $$($(1)_source)
+$(1)_extract_cmds ?= mkdir -p $$($(1)_extract_dir) && echo "$$($(1)_sha256_hash)  $$($(1)_source)" > $$($(1)_extract_dir)/.$$($(1)_file_name).hash &&  $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$$($(1)_file_name).hash && tar --no-same-owner --strip-components=1 -xf $$($(1)_source)
 $(1)_preprocess_cmds ?=
 $(1)_build_cmds ?=
 $(1)_config_cmds ?=
@@ -170,15 +171,15 @@ $($(1)_extracted): | $($(1)_fetched)
 	$(AT)mkdir -p $$(@D)
 	$(AT)cd $$(@D); $(call $(1)_extract_cmds,$(1))
 	$(AT)touch $$@
-$($(1)_preprocessed): | $($(1)_dependencies) $($(1)_extracted)
+$($(1)_preprocessed): | $($(1)_extracted)
 	$(AT)echo Preprocessing $(1)...
 	$(AT)mkdir -p $$(@D) $($(1)_patch_dir)
 	$(AT)$(foreach patch,$($(1)_patches),cd $(PATCHES_PATH)/$(1); cp $(patch) $($(1)_patch_dir) ;)
 	$(AT)cd $$(@D); $(call $(1)_preprocess_cmds, $(1))
 	$(AT)touch $$@
-$($(1)_configured): | $($(1)_preprocessed)
+$($(1)_configured): | $($(1)_dependencies) $($(1)_preprocessed)
 	$(AT)echo Configuring $(1)...
-	$(AT)rm -rf $(host_prefix); mkdir -p $(host_prefix)/lib; cd $(host_prefix); $(foreach package,$($(1)_all_dependencies), tar xf $($(package)_cached); )
+	$(AT)rm -rf $(host_prefix); mkdir -p $(host_prefix)/lib; cd $(host_prefix); $(foreach package,$($(1)_all_dependencies), tar --no-same-owner -xf $($(package)_cached); )
 	$(AT)mkdir -p $$(@D)
 	$(AT)+cd $$(@D); $($(1)_config_env) $(call $(1)_config_cmds, $(1))
 	$(AT)touch $$@
@@ -213,6 +214,14 @@ $(1): | $($(1)_cached_checksum)
 
 endef
 
+stages = fetched extracted preprocessed configured built staged postprocessed cached cached_checksum
+
+define ext_add_stages
+$(foreach stage,$(stages),
+          $(1)_$(stage): $($(1)_$(stage))
+          .PHONY: $(1)_$(stage))
+endef
+
 # These functions create the build targets for each package. They must be
 # broken down into small steps so that each part is done for all packages
 # before moving on to the next step. Otherwise, a package's info

depends/packages.md

@@ -14,8 +14,9 @@ Each package is required to define at least these variables:
     placeholder such as 1.0 can be used.
 
     $(package)_download_path:
-    Location of the upstream source, without the file-name. Usually http or
-    ftp.
+    Location of the upstream source, without the file-name. Usually http, https
+    or ftp. Secure transmission options like https should be preferred if
+    available.
 
     $(package)_file_name:
     The upstream source filename available at the download path.

depends/packages/bdb.mk

@@ -1,6 +1,6 @@
 package=bdb
 $(package)_version=4.8.30
-$(package)_download_path=http://download.oracle.com/berkeley-db
+$(package)_download_path=https://download.oracle.com/berkeley-db
 $(package)_file_name=db-$($(package)_version).NC.tar.gz
 $(package)_sha256_hash=12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef
 $(package)_build_subdir=build_unix
@@ -10,6 +10,7 @@ $(package)_config_opts=--disable-shared --enable-cxx --disable-replication
 $(package)_config_opts_mingw32=--enable-mingw
 $(package)_config_opts_linux=--with-pic
 $(package)_cxxflags=-std=c++11
+$(package)_cppflags_mingw32=-DUNICODE -D_UNICODE
 endef
 
 define $(package)_preprocess_cmds

depends/packages/expat.mk

@@ -1,11 +1,11 @@
 package=expat
-$(package)_version=2.2.5
-$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_2_2_5/
+$(package)_version=2.2.6
+$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_2_2_6/
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
-$(package)_sha256_hash=d9dc32efba7e74f788fcc4f212a43216fc37cf5f23f4c2339664d473353aedf6
+$(package)_sha256_hash=17b43c2716d521369f82fc2dc70f359860e90fa440bea65b3b85f0b246ea81f2
 
 define $(package)_set_vars
-$(package)_config_opts=--disable-static
+$(package)_config_opts=--disable-static --without-docbook
 endef
 
 define $(package)_config_cmds

depends/packages/fontconfig.mk

@@ -1,6 +1,6 @@
 package=fontconfig
 $(package)_version=2.12.1
-$(package)_download_path=http://www.freedesktop.org/software/fontconfig/release/
+$(package)_download_path=https://www.freedesktop.org/software/fontconfig/release/
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=b449a3e10c47e1d1c7a6ec6e2016cca73d3bd68fbbd4f0ae5cc6b573f7d6c7f3
 $(package)_dependencies=freetype expat

depends/packages/freetype.mk

@@ -1,6 +1,6 @@
 package=freetype
 $(package)_version=2.7.1
-$(package)_download_path=http://download.savannah.gnu.org/releases/$(package)
+$(package)_download_path=https://download.savannah.gnu.org/releases/$(package)
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=3a3bb2c4e15ffb433f2032f50a5b5a92558206822e22bfe8cbe339af4aa82f88
 

depends/packages/libX11.mk

@@ -1,6 +1,6 @@
 package=libX11
 $(package)_version=1.6.2
-$(package)_download_path=http://xorg.freedesktop.org/releases/individual/lib/
+$(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=2aa027e837231d2eeea90f3a4afe19948a6eb4c8b2bec0241eba7dbc8106bd16
 $(package)_dependencies=libxcb xtrans xextproto xproto

depends/packages/libXau.mk

@@ -1,6 +1,6 @@
 package=libXau
 $(package)_version=1.0.8
-$(package)_download_path=http://xorg.freedesktop.org/releases/individual/lib/
+$(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=fdd477320aeb5cdd67272838722d6b7d544887dfe7de46e1e7cc0c27c2bea4f2
 $(package)_dependencies=xproto

depends/packages/libXext.mk

@@ -1,6 +1,6 @@
 package=libXext
 $(package)_version=1.3.2
-$(package)_download_path=http://xorg.freedesktop.org/releases/individual/lib/
+$(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=f829075bc646cdc085fa25d98d5885d83b1759ceb355933127c257e8e50432e0
 $(package)_dependencies=xproto xextproto libX11 libXau

depends/packages/libxcb.mk

@@ -1,6 +1,6 @@
 package=libxcb
 $(package)_version=1.10
-$(package)_download_path=http://xcb.freedesktop.org/dist
+$(package)_download_path=https://xcb.freedesktop.org/dist
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=98d9ab05b636dd088603b64229dd1ab2d2cc02ab807892e107d674f9c3f2d5b5
 $(package)_dependencies=xcb_proto libXau xproto

depends/packages/miniupnpc.mk

@@ -1,6 +1,6 @@
 package=miniupnpc
 $(package)_version=2.0.20180203
-$(package)_download_path=http://miniupnp.free.fr/files
+$(package)_download_path=https://miniupnp.tuxfamily.org/files/
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
 $(package)_sha256_hash=90dda8c7563ca6cd4a83e23b3c66dbbea89603a1675bfdb852897c2c9cc220b7
 

depends/packages/native_biplist.mk

@@ -3,13 +3,13 @@ $(package)_version=1.0.3
 $(package)_download_path=https://bitbucket.org/wooster/biplist/downloads
 $(package)_file_name=biplist-$($(package)_version).tar.gz
 $(package)_sha256_hash=4c0549764c5fe50b28042ec21aa2e14fe1a2224e239a1dae77d9e7f3932aa4c6
-$(package)_install_libdir=$(build_prefix)/lib/python/dist-packages
+$(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages
 
 define $(package)_build_cmds
-    python setup.py build
+    python3 setup.py build
 endef
 
 define $(package)_stage_cmds
     mkdir -p $($(package)_install_libdir) && \
-    python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir)
+    python3 setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir)
 endef

depends/packages/native_cctools.mk

@@ -5,7 +5,7 @@ $(package)_file_name=$($(package)_version).tar.gz
 $(package)_sha256_hash=a09c9ba4684670a0375e42d9d67e7f12c1f62581a27f28f7c825d6d7032ccc6a
 $(package)_build_subdir=cctools
 $(package)_clang_version=3.7.1
-$(package)_clang_download_path=http://llvm.org/releases/$($(package)_clang_version)
+$(package)_clang_download_path=https://llvm.org/releases/$($(package)_clang_version)
 $(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz
 $(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz
 $(package)_clang_sha256_hash=99b28a6b48e793705228a390471991386daa33a9717cd9ca007fcdde69608fd9
@@ -22,12 +22,12 @@ define $(package)_extract_cmds
   echo "$($(package)_clang_sha256_hash)  $($(package)_source_dir)/$($(package)_clang_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \
   $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \
   mkdir -p toolchain/bin toolchain/lib/clang/3.5/include && \
-  tar --strip-components=1 -C toolchain -xf $($(package)_source_dir)/$($(package)_clang_file_name) && \
+  tar --no-same-owner --strip-components=1 -C toolchain -xf $($(package)_source_dir)/$($(package)_clang_file_name) && \
   rm -f toolchain/lib/libc++abi.so* && \
   echo "#!/bin/sh" > toolchain/bin/$(host)-dsymutil && \
   echo "exit 0" >> toolchain/bin/$(host)-dsymutil && \
   chmod +x toolchain/bin/$(host)-dsymutil && \
-  tar --strip-components=1 -xf $($(package)_source)
+  tar --no-same-owner --strip-components=1 -xf $($(package)_source)
 endef
 
 define $(package)_set_vars

depends/packages/native_cdrkit.mk

@@ -1,6 +1,6 @@
 package=native_cdrkit
 $(package)_version=1.1.11
-$(package)_download_path=http://distro.ibiblio.org/fatdog/source/600/c
+$(package)_download_path=https://distro.ibiblio.org/fatdog/source/600/c
 $(package)_file_name=cdrkit-$($(package)_version).tar.bz2
 $(package)_sha256_hash=b50d64c214a65b1a79afe3a964c691931a4233e2ba605d793eb85d0ac3652564
 $(package)_patches=cdrkit-deterministic.patch

depends/packages/native_ds_store.mk

@@ -3,14 +3,14 @@ $(package)_version=1.1.2
 $(package)_download_path=https://github.com/al45tair/ds_store/archive/
 $(package)_file_name=v$($(package)_version).tar.gz
 $(package)_sha256_hash=3b3ecb7bf0a5157f5b6010bc3af7c141fb0ad3527084e63336220d22744bc20c
-$(package)_install_libdir=$(build_prefix)/lib/python/dist-packages
+$(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages
 $(package)_dependencies=native_biplist
 
 define $(package)_build_cmds
-    python setup.py build
+    python3 setup.py build
 endef
 
 define $(package)_stage_cmds
     mkdir -p $($(package)_install_libdir) && \
-    python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir)
+    python3 setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir)
 endef

depends/packages/native_mac_alias.mk

@@ -3,13 +3,13 @@ $(package)_version=2.0.7
 $(package)_download_path=https://github.com/al45tair/mac_alias/archive/
 $(package)_file_name=v$($(package)_version).tar.gz
 $(package)_sha256_hash=6f606d3b6bccd2112aeabf1a063f5b5ece87005a5d7e97c8faca23b916e88838
-$(package)_install_libdir=$(build_prefix)/lib/python/dist-packages
+$(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages
 
 define $(package)_build_cmds
-    python setup.py build
+    python3 setup.py build
 endef
 
 define $(package)_stage_cmds
     mkdir -p $($(package)_install_libdir) && \
-    python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir)
+    python3 setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir)
 endef

depends/packages/native_protobuf.mk

@@ -5,7 +5,7 @@ $(package)_file_name=protobuf-$($(package)_version).tar.bz2
 $(package)_sha256_hash=ee445612d544d885ae240ffbcbf9267faa9f593b7b101f21d58beceb92661910
 
 define $(package)_set_vars
-$(package)_config_opts=--disable-shared
+$(package)_config_opts=--disable-shared --without-zlib
 endef
 
 define $(package)_config_cmds