Merge pull request #5426 from kittywhiskers/guix_p3

backport: merge bitcoin#25643, bitcoin#23583, bitcoin#23618, bitcoin#23603, bitcoin#23817, bitcoin#24520, bitcoin#24508, bitcoin#25639, bitcoin#26018, bitcoin#25437, bitcoin#25484, bitcoin#23585, bitcoin#24549, bitcoin#24733, bitcoin#21991, bitcoin#22526, bitcoin#25633, bitcoin#24597, bitcoin#24955, bitcoin#25099, bitcoin#24552, bitcoin#21851, bitcoin#25389, bitcoin#25357, partial bitcoin#22318 (guix backports: part 3)

Author: PastaPastaPasta

.gitlab-ci.yml

@@ -42,13 +42,13 @@ builder-image:
   image: $CI_REGISTRY_IMAGE:builder-$CI_COMMIT_REF_SLUG
   variables:
     SDK_URL: https://bitcoincore.org/depends-sources/sdks
-    XCODE_VERSION: "12.1"
-    XCODE_BUILD_ID: 12A7403
+    XCODE_VERSION: "12.2"
+    XCODE_BUILD_ID: 12B45b
     MAKEJOBS: -j4
   before_script:
     - echo HOST=$HOST
     - |
-      if [ "$HOST" = "x86_64-apple-darwin19" ]; then
+      if [ "$HOST" = "x86_64-apple-darwin" ]; then
         mkdir -p depends/SDKs
         mkdir -p depends/sdk-sources
         OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz"
@@ -193,12 +193,12 @@ x86_64-pc-linux-gnu-nowallet:
     HOST: x86_64-pc-linux-gnu
     DEP_OPTS: "NO_WALLET=1"
 
-x86_64-apple-darwin19:
+x86_64-apple-darwin:
   extends:
     - .build-depends-template
     - .skip-in-fast-mode-template
   variables:
-    HOST: x86_64-apple-darwin19
+    HOST: x86_64-apple-darwin
 
 ###
 
@@ -302,7 +302,7 @@ mac-build:
     - .build-template
     - .skip-in-fast-mode-template
   needs:
-    - x86_64-apple-darwin19
+    - x86_64-apple-darwin
   variables:
     BUILD_TARGET: mac
 

ci/test/00_setup_env_mac.sh

@@ -7,10 +7,10 @@
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_macos_cross
-export HOST=x86_64-apple-darwin19
+export HOST=x86_64-apple-darwin
 export PACKAGES="cmake libcap-dev libz-dev libbz2-dev python3-dev python3-setuptools"
-export XCODE_VERSION=12.1
-export XCODE_BUILD_ID=12A7403
+export XCODE_VERSION=12.2
+export XCODE_BUILD_ID=12B45b
 export RUN_UNIT_TESTS=false
 export RUN_INTEGRATION_TESTS=false
 export GOAL="all deploy"

ci/test/00_setup_env_mac_host.sh

@@ -7,7 +7,7 @@
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_macos
-export HOST=x86_64-apple-darwin19
+export HOST=x86_64-apple-darwin
 export PIP_PACKAGES="zmq lief"
 export RUN_UNIT_TESTS=true
 export RUN_INTEGRATION_TESTS=false

configure.ac

@@ -20,6 +20,7 @@ BITCOIN_TX_NAME=dash-tx
 BITCOIN_WALLET_TOOL_NAME=dash-wallet
 
 dnl Unless the user specified ARFLAGS, force it to be cr
+dnl This is also the default as-of libtool 2.4.7
 AC_ARG_VAR(ARFLAGS, [Flags for the archiver, defaults to <cr> if not set])
 if test "x${ARFLAGS+set}" != "xset"; then
   ARFLAGS="cr"

contrib/containers/ci/Dockerfile

@@ -37,7 +37,7 @@ RUN pip3 install \
     codespell==1.17.1 \
     flake8==3.8.3 \
     jinja2 \
-    lief==0.12.0 \
+    lief==0.12.1 \
     pyzmq \
     vulture==2.3 \
     yq \

contrib/containers/guix/scripts/guix-start

@@ -13,8 +13,8 @@ if [[ ! -d "$WORKSPACE_PATH" ]]; then
     exit 1
 fi
 
-XCODE_VERSION="12.1"
-XCODE_RELEASE="12A7403"
+XCODE_VERSION="12.2"
+XCODE_RELEASE="12B45b"
 XCODE_ARCHIVE="Xcode-${XCODE_VERSION}-${XCODE_RELEASE}-extracted-SDK-with-libcxx-headers"
 
 # Check if macOS SDK is present, if not, download it

contrib/devtools/security-check.py

@@ -206,12 +206,9 @@ def check_MACHO_control_flow(binary) -> bool:
 ]
 
 BASE_MACHO = [
-    ('PIE', check_PIE),
     ('NOUNDEFS', check_MACHO_NOUNDEFS),
-    ('NX', check_NX),
     ('LAZY_BINDINGS', check_MACHO_LAZY_BINDINGS),
     ('Canary', check_MACHO_Canary),
-    ('CONTROL_FLOW', check_MACHO_control_flow),
 ]
 
 CHECKS = {
@@ -226,7 +223,10 @@ def check_MACHO_control_flow(binary) -> bool:
         lief.ARCHITECTURES.X86: BASE_PE,
     },
     lief.EXE_FORMATS.MACHO: {
-        lief.ARCHITECTURES.X86: BASE_MACHO,
+        lief.ARCHITECTURES.X86: BASE_MACHO + [('PIE', check_PIE),
+                                              ('NX', check_NX),
+                                              ('CONTROL_FLOW', check_MACHO_control_flow)],
+        lief.ARCHITECTURES.ARM64: BASE_MACHO,
     }
 }
 

contrib/devtools/symbol-check.py

@@ -217,7 +217,7 @@ def check_MACHO_min_os(binary) -> bool:
     return False
 
 def check_MACHO_sdk(binary) -> bool:
-    if binary.build_version.sdk == [10, 15, 6]:
+    if binary.build_version.sdk == [11, 0, 0]:
         return True
     return False
 

contrib/devtools/test-security-check.py

@@ -116,21 +116,34 @@ def test_MACHO(self):
         executable = 'test1'
         cc = determine_wellknown_cmd('CC', 'clang')
         write_testcode(source)
+        arch = get_arch(cc, source, executable)
+
+        if arch == lief.ARCHITECTURES.X86:
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']),
+                (1, executable+': failed NOUNDEFS LAZY_BINDINGS Canary PIE NX CONTROL_FLOW'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all']),
+                (1, executable+': failed NOUNDEFS LAZY_BINDINGS PIE NX CONTROL_FLOW'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all']),
+                (1, executable+': failed NOUNDEFS LAZY_BINDINGS PIE CONTROL_FLOW'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all']),
+                (1, executable+': failed LAZY_BINDINGS PIE CONTROL_FLOW'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all']),
+                (1, executable+': failed PIE CONTROL_FLOW'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']),
+                (1, executable+': failed PIE'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']),
+                (0, ''))
+        else:
+            # arm64 darwin doesn't support non-PIE binaries, control flow or executable stacks
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector']),
+                (1, executable+': failed NOUNDEFS LAZY_BINDINGS Canary'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all']),
+                (1, executable+': failed NOUNDEFS LAZY_BINDINGS'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all']),
+                (1, executable+': failed LAZY_BINDINGS'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-bind_at_load','-fstack-protector-all']),
+                (0, ''))
 
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']),
-            (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS Canary CONTROL_FLOW'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all']),
-            (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS CONTROL_FLOW'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all']),
-            (1, executable+': failed PIE NOUNDEFS LAZY_BINDINGS CONTROL_FLOW'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all']),
-            (1, executable+': failed PIE LAZY_BINDINGS CONTROL_FLOW'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all']),
-            (1, executable+': failed PIE CONTROL_FLOW'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']),
-            (1, executable+': failed PIE'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']),
-            (0, ''))
 
         clean_files(source, executable)
 

contrib/gitian-build.py

@@ -74,8 +74,8 @@ def build():
 
     if args.macos:
         print('\nCompiling ' + args.version + ' MacOS')
-        subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/depends-sources/sdks/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz'])
-        subprocess.check_output(["echo 'be17f48fd0b08fb4dcd229f55a6ae48d9f781d210839b4ea313ef17dd12d6ea5 inputs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz' | sha256sum -c"], shell=True)
+        subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/depends-sources/sdks/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz'])
+        subprocess.check_output(["echo 'df75d30ecafc429e905134333aeae56ac65fac67cb4182622398fd717df77619 inputs/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz' | sha256sum -c"], shell=True)
         subprocess.check_call(['bin/gbuild', '--fetch-tags',  '-j', args.jobs, '-m', args.memory, '--commit', 'dash='+args.commit, '--url', 'dash='+args.url, '../dash/contrib/gitian-descriptors/gitian-osx.yml'])
         subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-unsigned', '--destination', '../gitian.sigs/', '../dash/contrib/gitian-descriptors/gitian-osx.yml'])
         subprocess.check_call('mv build/out/dashcore-*-osx-unsigned.tar.gz inputs/', shell=True)
@@ -218,7 +218,7 @@ def main():
     args.macos = 'm' in args.os
 
     # Disable for MacOS if no SDK found
-    if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz'):
+    if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz'):
         print('Cannot build for MacOS, SDK does not exist. Will build for other OSes')
         args.macos = False