From 05fe9dbc4d0930b9f7138db32117df61c73a832b Mon Sep 17 00:00:00 2001 From: Giovanni Petrantoni Date: Thu, 18 Jun 2020 14:32:31 +0900 Subject: [PATCH 1/3] bump go daemon tag version --- .appveyor.yml | 2 +- .travis.yml | 2 +- azure-pipelines.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 29d8a80db0..22b01411ed 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -40,7 +40,7 @@ install: # install and build go-libp2p-daemon - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_p2pd.sh - - bash build_p2pd.sh p2pdCache + - bash build_p2pd.sh p2pdCache v0.2.4 build_script: - nimble install -y --depsOnly diff --git a/.travis.yml b/.travis.yml index 201eca668b..a184925926 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,7 +45,7 @@ install: # install and build go-libp2p-daemon - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_p2pd.sh - - bash build_p2pd.sh p2pdCache v0.2.1 + - bash build_p2pd.sh p2pdCache v0.2.4 script: - nimble install -y --depsOnly diff --git a/azure-pipelines.yml b/azure-pipelines.yml index caa9581bd9..a881136bee 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -170,7 +170,7 @@ steps: echo "##vso[task.prependpath]$(GOPATH)/bin" curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_p2pd.sh - bash build_p2pd.sh p2pdCache + bash build_p2pd.sh p2pdCache v0.2.4 displayName: 'Building Go libp2p daemon' - bash: | From 1552dcb177524d95b3d311267da7b2f2497cca71 Mon Sep 17 00:00:00 2001 From: Giovanni Petrantoni Date: Thu, 18 Jun 2020 15:04:31 +0900 Subject: [PATCH 2/3] enable noise in daemonapi --- libp2p/daemon/daemonapi.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/libp2p/daemon/daemonapi.nim b/libp2p/daemon/daemonapi.nim index b15a0b2a0f..68570ddb3a 100644 --- a/libp2p/daemon/daemonapi.nim +++ b/libp2p/daemon/daemonapi.nim @@ -736,6 +736,7 @@ proc newDaemonApi*(flags: set[P2PDaemonFlags] = {}, if i > 0: opt.add "," opt.add $address args.add(opt) + args.add("-noise=true") args.add("-listen=" & $api.address) # We are trying to get absolute daemon path. From 8d2577ca5f0020a5c919e211dc1c7996986d266f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= Date: Thu, 18 Jun 2020 19:38:01 +0200 Subject: [PATCH 3/3] CI changes - Travis: use Go 1.14 - azure-pipelines.yml: big cleanup - Azure: bump cache keys - build 64-bit p2pd on 32-bit Windows - install both Mingw-w64 architectures --- .travis.yml | 3 +- azure-pipelines.yml | 182 +++++++++++++------------------------------- libp2p.nimble | 17 +++-- 3 files changed, 63 insertions(+), 139 deletions(-) diff --git a/.travis.yml b/.travis.yml index a184925926..c116e17670 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ git: # when multiple CI builds are queued, the tested commit needs to be in the last X commits cloned with "--depth X" depth: 10 -go: "1.12.x" +go: "1.14.x" matrix: include: @@ -51,3 +51,4 @@ script: - nimble install -y --depsOnly - nimble test - nimble examples_build + diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a881136bee..5e412bcde9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,186 +13,108 @@ strategy: VM: 'windows-latest' PLATFORM: x64 TEST_LANG: c - # Windows_cpp_64bit: - # VM: 'windows-latest' - # UCPU: amd64 - # PLATFORM: x64 - # TEST_LANG: cpp - #Linux_64bit: - #VM: 'ubuntu-16.04' - #PLATFORM: x64 - #TEST_LANG: c - # Linux_cpp_64bit: - # VM: 'ubuntu-16.04' - # UCPU: amd64 - # PLATFORM: x64 - # TEST_LANG: cpp - - # TODO this requires gcc -m32 as the Ubuntu image is 64-bit - # Linux_32bit: - # VM: 'ubuntu-16.04' - # ARCH: x86 - # PLATFORM: x86 - # TEST_LANG: c - - # TODO: go version on Azure is problematic, - # we have xerrors package issue - # MacOS_64bit: - # VM: 'macOS-10.14' - # PLATFORM: x64 - # TEST_LANG: c pool: vmImage: $(VM) variables: - GOPATH: '$(system.defaultWorkingDirectory)/go' V: 0 # Scripts verbosity, 1 for debugging build scripts steps: - task: CacheBeta@1 displayName: 'cache Nim binaries' inputs: - key: NimBinaries | $(Agent.OS) | $(PLATFORM) | "$(Build.SourceBranchName)" | "v3" + key: NimBinaries | $(Agent.OS) | $(PLATFORM) | "$(Build.SourceBranchName)" | "v4" path: NimBinaries - task: CacheBeta@1 displayName: 'cache Go libp2p daemon' inputs: - key: p2pdCache | $(Agent.OS) | $(PLATFORM) + key: p2pdCache | $(Agent.OS) | $(PLATFORM) | "v3" path: p2pdCache - task: CacheBeta@1 displayName: 'cache MinGW-w64' inputs: - key: mingwCache | 8_1_0 | $(PLATFORM) + key: mingwCache | 8_1_0 | $(PLATFORM) | "v1" path: mingwCache - condition: eq(variables['Agent.OS'], 'Windows_NT') - powershell: | Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem' -Name 'LongPathsEnabled' -Value 1 displayName: 'long path support' - condition: eq(variables['Agent.OS'], 'Windows_NT') + - bash: | set -e - echo "PATH=${PATH}" + + # custom MinGW-w64 versions for both 32-bit and 64-bit, since we need a 64-bit build of p2pd echo "Installing MinGW-w64" + + install_mingw() { + mkdir -p mingwCache + cd mingwCache + if [[ ! -e "$MINGW_FILE" ]]; then + rm -f *.7z + curl -OLsS "$MINGW_URL" + fi + 7z x -y -bd "$MINGW_FILE" >/dev/null + mkdir -p /c/custom + mv "$MINGW_DIR" /c/custom/ + cd .. + } + + # 32-bit + MINGW_FILE="i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z" + MINGW_URL="https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/8.1.0/threads-posix/dwarf/${MINGW_FILE}" + MINGW_DIR="mingw32" + install_mingw + + # 64-bit + MINGW_FILE="x86_64-8.1.0-release-posix-seh-rt_v6-rev0.7z" + MINGW_URL="https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win64/Personal%20Builds/mingw-builds/8.1.0/threads-posix/seh/${MINGW_FILE}" + MINGW_DIR="mingw64" + install_mingw + if [[ $PLATFORM == "x86" ]]; then - MINGW_FILE="i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z" - MINGW_URL="https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/8.1.0/threads-posix/dwarf/${MINGW_FILE}" MINGW_DIR="mingw32" else - MINGW_FILE="x86_64-8.1.0-release-posix-seh-rt_v6-rev0.7z" - MINGW_URL="https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win64/Personal%20Builds/mingw-builds/8.1.0/threads-posix/seh/${MINGW_FILE}" MINGW_DIR="mingw64" fi - mkdir -p mingwCache - pushd mingwCache - if [[ ! -e "$MINGW_FILE" ]]; then - rm -f *.7z - curl -OLsS "$MINGW_URL" - fi - 7z x -y -bd "$MINGW_FILE" >/dev/null - mkdir -p /c/custom - mv "$MINGW_DIR" /c/custom/ - popd - - # Workaround https://developercommunity.visualstudio.com/content/problem/891929/windows-2019-cygheap-base-mismatch-detected-git-ba.html - echo "##vso[task.prependpath]/usr/bin" - echo "##vso[task.prependpath]/mingw64/bin" - - echo "##vso[task.setvariable variable=MINGW_DIR;]$MINGW_DIR" - displayName: 'Install dependencies (Windows)' - condition: eq(variables['Agent.OS'], 'Windows_NT') - - powershell: | - # export custom mingw PATH to other tasks - echo "##vso[task.prependpath]c:\custom\$(MINGW_DIR)\bin" - displayName: 'Mingw PATH (Windows)' - condition: eq(variables['Agent.OS'], 'Windows_NT') - - - bash: | - set -e + export PATH="/c/custom/${MINGW_DIR}/bin:${PATH}" echo "PATH=${PATH}" - export ncpu= - case '$(Agent.OS)' in - 'Linux') - ncpu=$(nproc) - ;; - 'Darwin') - ncpu=$(sysctl -n hw.ncpu) - ;; - 'Windows_NT') - ncpu=$NUMBER_OF_PROCESSORS - ;; - esac - [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 - echo "Found ${ncpu} cores" - echo "##vso[task.setvariable variable=ncpu;]$ncpu" - displayName: 'Detecting number of cores' + which gcc + gcc -v - - bash: | - set -e - echo "PATH=${PATH}" + # detect number of cores + export ncpu="$NUMBER_OF_PROCESSORS" + [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=2 + echo "Found ${ncpu} cores" # build nim from our own branch - this to avoid the day-to-day churn and # regressions of the fast-paced Nim development while maintaining the # flexibility to apply patches - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh + curl -OLsS https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh env MAKE="mingw32-make -j${ncpu}" ARCH_OVERRIDE=$(PLATFORM) bash build_nim.sh Nim csources dist/nimble NimBinaries - displayName: 'Building Nim (Windows)' - condition: eq(variables['Agent.OS'], 'Windows_NT') - - powershell: | - echo "##vso[task.prependpath]$pwd\Nim\bin" - displayName: 'Set env variable (Windows)' - condition: eq(variables['Agent.OS'], 'Windows_NT') - - - bash: | - set -e + export PATH="${PWD}/Nim/bin:${PATH}" echo "PATH=${PATH}" - # build nim from our own branch - this to avoid the day-to-day churn and - # regressions of the fast-paced Nim development while maintaining the - # flexibility to apply patches - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh - env MAKE="make -j${ncpu}" ARCH_OVERRIDE=$(PLATFORM) bash build_nim.sh Nim csources dist/nimble NimBinaries - displayName: 'Building Nim (POSIX)' - condition: ne(variables['Agent.OS'], 'Windows_NT') - - bash: | - echo "##vso[task.prependpath]$PWD/Nim/bin" - displayName: 'Set env variable (Posix)' - condition: ne(variables['Agent.OS'], 'Windows_NT') - - - bash: | - set -e # install and build go-libp2p-daemon go version - echo "##vso[task.prependpath]$(GOPATH)/bin" - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_p2pd.sh - bash build_p2pd.sh p2pdCache v0.2.4 - displayName: 'Building Go libp2p daemon' - - - bash: | - set -e + export GOPATH="${PWD}/go" + export PATH="${GOPATH}/bin:${PATH}" echo "PATH=${PATH}" + + curl -OLsS https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_p2pd.sh + # we can't seem to be able to build a 32-bit p2pd + env PATH="/c/custom/mingw64/bin:${PATH}" bash build_p2pd.sh p2pdCache v0.2.4 + + # install dependencies nimble refresh nimble install -y --depsOnly - displayName: 'Building dependencies (Posix)' - condition: ne(variables['Agent.OS'], 'Windows_NT') - - powershell: | - echo $Env:Path - nimble refresh - nimble install -y --depsOnly - displayName: 'Building dependencies (Windows)' - condition: eq(variables['Agent.OS'], 'Windows_NT') - - - bash: | - set -e - echo "PATH=${PATH}" - which gcc - gcc -v + # run tests nimble test nimble examples_build - displayName: 'Testing the package' + displayName: 'build and test' + diff --git a/libp2p.nimble b/libp2p.nimble index 914da3e2c5..2675990285 100644 --- a/libp2p.nimble +++ b/libp2p.nimble @@ -17,18 +17,19 @@ requires "nim >= 1.2.0", "stew >= 0.1.0" proc runTest(filename: string, verify: bool = true, sign: bool = true) = - var excstr: string = "nim c -r --opt:speed -d:debug --verbosity:0 --hints:off -d:chronicles_log_level=info" - excstr.add(" ") - excstr.add("-d:libp2p_pubsub_sign=" & $sign) - excstr.add(" ") - excstr.add("-d:libp2p_pubsub_verify=" & $verify) - excstr.add(" ") - excstr.add("tests/" & filename) + var excstr = "nim c -r --opt:speed -d:debug --verbosity:0 --hints:off -d:chronicles_log_level=info" + excstr.add(" --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off") + excstr.add(" -d:libp2p_pubsub_sign=" & $sign) + excstr.add(" -d:libp2p_pubsub_verify=" & $verify) + excstr.add(" tests/" & filename) exec excstr rmFile "tests/" & filename.toExe proc buildSample(filename: string) = - exec "nim c --opt:speed --threads:on -d:debug --verbosity:0 --hints:off examples/" & filename + var excstr = "nim c --opt:speed --threads:on -d:debug --verbosity:0 --hints:off" + excstr.add(" --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off") + excstr.add(" examples/" & filename) + exec excstr rmFile "examples" & filename.toExe task testnative, "Runs libp2p native tests":