CI: update nightly builds to use universal mac update (#6071)

.circleci/config.yml

@@ -715,7 +715,7 @@ commands:
             command: |
               if [ "${CIRCLE_BRANCH}" = "rel/nightly" ]
               then
-                export NO_BUILD="true"
+                export NIGHTLY_BUILD="true"
               fi
               export PATH=$(echo "$PATH" | sed -e "s|:${HOME}/\.go_workspace/bin||g" | sed -e 's|:/usr/local/go/bin||g')
               export GOPATH="<< parameters.build_dir >>/go"

Makefile

@@ -179,19 +179,24 @@ ifeq ($(OS_TYPE),darwin)
 	mkdir -p $(GOPATH1)/bin-darwin-arm64
 	CROSS_COMPILE_ARCH=arm64 GOBIN=$(GOPATH1)/bin-darwin-arm64 MACOSX_DEPLOYMENT_TARGET=12.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch arm64 -mmacos-version-min=12.0" --host=aarch64-apple-darwin' $(MAKE)
 
+	# same for buildsrc-special
+	cd tools/block-generator && \
+	CROSS_COMPILE_ARCH=amd64 GOBIN=$(GOPATH1)/bin-darwin-amd64 MACOSX_DEPLOYMENT_TARGET=12.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch x86_64 -mmacos-version-min=12.0" --host=x86_64-apple-darwin' $(MAKE)
+	CROSS_COMPILE_ARCH=arm64 GOBIN=$(GOPATH1)/bin-darwin-arm64 MACOSX_DEPLOYMENT_TARGET=12.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch arm64 -mmacos-version-min=12.0" --host=aarch64-apple-darwin' $(MAKE)
+
 	# lipo together
-	mkdir -p $(GOPATH1)/bin-darwin-universal
+	mkdir -p $(GOPATH1)/bin
 	for binary in $$(ls $(GOPATH1)/bin-darwin-arm64); do \
 		if [ -f $(GOPATH1)/bin-darwin-amd64/$$binary ]; then \
-			lipo -create -output $(GOPATH1)/bin-darwin-universal/$$binary \
+			lipo -create -output $(GOPATH1)/bin/$$binary \
 			$(GOPATH1)/bin-darwin-arm64/$$binary \
 			$(GOPATH1)/bin-darwin-amd64/$$binary; \
 		else \
 			echo "Warning: Binary $$binary exists in arm64 but not in amd64"; \
 		fi \
 	done
 else
-	$(error OS_TYPE must be darwin for universal builds)
+	echo "OS_TYPE must be darwin for universal builds, skipping"
 endif
 
 deps:

cmd/updater/update.sh

@@ -185,10 +185,6 @@ function get_updater_url() {
         UNAME=$(uname -m)
         if [[ "${UNAME}" = "x86_64" ]]; then
             ARCH="amd64"
-        elif [[ "${UNAME}" = "armv6l" ]]; then
-            ARCH="arm"
-        elif [[ "${UNAME}" = "armv7l" ]]; then
-            ARCH="arm"
         elif [[ "${UNAME}" = "aarch64" ]]; then
             ARCH="arm64"
         else

scripts/release/mule/Makefile.mule

@@ -1,15 +1,13 @@
 # This file is imported into go-algorand/Makefile.
 
 PKG_DIR = $(SRCPATH)/tmp/node_pkgs/$(OS_TYPE)/$(ARCH)
+PKG_DIR_UNIVERSAL = $(SRCPATH)/tmp/node_pkgs/$(OS_TYPE)/universal
 
-.PHONY: ci-clean ci-setup ci-build
+.PHONY: ci-clean ci-build
 
 ci-clean: clean
 	rm -rf tmp
 
-ci-setup:
-	mkdir -p $(PKG_DIR)
-
 ci-test:
 ifeq ($(ARCH), amd64)
 	RACE=-race
@@ -28,7 +26,16 @@ ci-integration:
 	SRCROOT=$(SRCPATH) \
 	test/scripts/e2e.sh -c $(CHANNEL) -n
 
-ci-build: ci-clean build ci-setup
+ci-build-universal: ci-clean universal
+	echo $(PKG_DIR_UNIVERSAL)
+	mkdir -p $(PKG_DIR_UNIVERSAL)
+	CHANNEL=$(CHANNEL) PKG_ROOT=$(PKG_DIR_UNIVERSAL) NO_BUILD=True VARIATIONS=$(OS_TYPE)-universal \
+	scripts/build_packages.sh $(OS_TYPE)/universal && \
+	mkdir -p $(PKG_DIR_UNIVERSAL)/data && \
+	cp installer/genesis/devnet/genesis.json $(PKG_DIR_UNIVERSAL)/data
+
+ci-build: ci-clean
+	mkdir -p $(PKG_DIR)
 	CHANNEL=$(CHANNEL) PKG_ROOT=$(PKG_DIR) NO_BUILD=True VARIATIONS=$(OS_TYPE)-$(ARCH) \
 	scripts/build_packages.sh $(OS_TYPE)/$(ARCH) && \
 	mkdir -p $(PKG_DIR)/data && \

scripts/release/mule/README.md

@@ -25,7 +25,6 @@ In addition, make sure that the following AWS credentials are set in environment
 # Build Stages
 
 - [package](#package)
-- [upload](#upload)
 - [test](#test)
 - [sign](#sign)
 - [deploy](#deploy)
@@ -49,20 +48,6 @@ In addition, make sure that the following AWS credentials are set in environment
     - package-docker
         + packages docker image
 
-## upload
-
-- see `./go-algorand/package-upload.yaml`
-
-- customizable environment variables:
-
-    + `CHANNEL`
-    + `STAGING`
-    + `VERSION`
-
-#### `mule` jobs
-
-    - package-upload
-
 ## test
 
 - see `./go-algorand/package-test.yaml`
@@ -180,10 +165,6 @@ Let's look at some examples.
 
     mule -f package.yaml package
 
-### Uploading
-
-    STAGING=the-staging-area CHANNEL=beta VERSION=latest mule -f package-upload.yaml package-upload
-
 ### Testing
 
 1. As part of the test suite, the `verify_package_string.sh` test needs the `BRANCH` as well as the `SHA`:

scripts/travis/deploy_packages.sh

@@ -11,7 +11,7 @@
 set -e
 
 SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
-
+OSARCH=$("${SCRIPTPATH}/../osarchtype.sh")
 # Get the go build version.
 GOLANG_VERSION=$("${SCRIPTPATH}/../get_golang_version.sh")
 
@@ -24,11 +24,20 @@ then
     exit 1
 fi
 
+if [ "${NIGHTLY_BUILD}" == "true" ]; then
+    # we want to rebuild universal binaries for nightly builds
+    NO_BUILD=true
+    if [ "${OSARCH}" == "darwin/arm64" ]; then
+        make universal
+        OSARCH="darwin/universal"
+    fi
+fi
+
 if [ -z "${NO_BUILD}" ] || [ "${NO_BUILD}" != "true" ]; then
     scripts/travis/build.sh
 fi
 
 export RELEASE_GENESIS_PROCESS=true
 export NO_BUILD=true
 export SkipCleanCheck=1
-scripts/deploy_version.sh "${TRAVIS_BRANCH}" "$(./scripts/osarchtype.sh)"
+scripts/deploy_version.sh "${TRAVIS_BRANCH}" "${OSARCH}"

test/muleCI/mule.yaml

@@ -93,11 +93,8 @@ agents:
 
 tasks:
   - task: shell.Make
-    name: build.darwin-arm64
-    target: ci-build
-  - task: shell.Make
-    name: build.darwin-amd64
-    target: ci-build
+    name: build.darwin-universal
+    target: ci-build-universal
   - task: docker.Make
     name: build.amd64
     agent: cicd.ubuntu.amd64
@@ -124,9 +121,9 @@ tasks:
 
   # Stash tasks
   - task: stash.Stash
-    name: darwin-arm64
+    name: darwin-universal
     bucketName: go-algorand-ci-cache
-    stashId: ${JENKINS_JOB_CACHE_ID}/darwin-arm64
+    stashId: ${JENKINS_JOB_CACHE_ID}/darwin-universal
     globSpecs:
       - tmp/node_pkgs/**/*
   - task: stash.Stash
@@ -135,12 +132,6 @@ tasks:
     stashId: ${JENKINS_JOB_CACHE_ID}/linux-amd64
     globSpecs:
       - tmp/node_pkgs/**/*
-  - task: stash.Stash
-    name: darwin-amd64
-    bucketName: go-algorand-ci-cache
-    stashId: ${JENKINS_JOB_CACHE_ID}/darwin-amd64
-    globSpecs:
-      - tmp/node_pkgs/**/*
   - task: stash.Stash
     name: linux-arm64
     bucketName: go-algorand-ci-cache
@@ -164,13 +155,9 @@ tasks:
     bucketName: go-algorand-ci-cache
     stashId: ${JENKINS_JOB_CACHE_ID}/linux-amd64
   - task: stash.Unstash
-    name: darwin-amd64
+    name: darwin-universal
     bucketName: go-algorand-ci-cache
-    stashId: ${JENKINS_JOB_CACHE_ID}/darwin-amd64
-  - task: stash.Unstash
-    name: darwin-arm64
-    bucketName: go-algorand-ci-cache
-    stashId: ${JENKINS_JOB_CACHE_ID}/darwin-arm64
+    stashId: ${JENKINS_JOB_CACHE_ID}/darwin-universal
   - task: stash.Unstash
     name: packages
     bucketName: go-algorand-ci-cache
@@ -187,14 +174,10 @@ tasks:
     target: mule-sign
 
 jobs:
-  build-darwin-arm64:
-    tasks:
-      - shell.Make.build.darwin-arm64
-      - stash.Stash.darwin-arm64
-  build-darwin-amd64:
+  build-darwin-universal:
     tasks:
-      - shell.Make.build.darwin-amd64
-      - stash.Stash.darwin-amd64
+      - shell.Make.build.darwin-universal
+      - stash.Stash.darwin-universal
   build-linux-amd64:
     tasks:
       - docker.Make.build.amd64
@@ -207,8 +190,7 @@ jobs:
     tasks:
       - stash.Unstash.linux-amd64
       - stash.Unstash.linux-arm64
-      - stash.Unstash.darwin-arm64
-      - stash.Unstash.darwin-amd64
+      - stash.Unstash.darwin-universal
       - docker.Make.deb.amd64
       - docker.Make.rpm.amd64
       - stash.Stash.packages

util/s3/s3Helper.go

@@ -189,7 +189,17 @@ func (helper *Helper) GetPackageVersion(channel string, pkg string, specificVers
 	osName := runtime.GOOS
 	arch := runtime.GOARCH
 	prefix := fmt.Sprintf("%s_%s_%s-%s_", pkg, channel, osName, arch)
-	return helper.GetPackageFilesVersion(channel, prefix, specificVersion)
+
+	maxVersion, maxVersionName, err = helper.GetPackageFilesVersion(channel, prefix, specificVersion)
+	// For darwin, we want to also look at universal binaries
+	if osName == "darwin" {
+		universalPrefix := fmt.Sprintf("%s_%s_%s-%s_", pkg, channel, osName, "universal")
+		universalMaxVersion, universalMaxVersionName, universalErr := helper.GetPackageFilesVersion(channel, universalPrefix, specificVersion)
+		if universalMaxVersion > maxVersion {
+			return universalMaxVersion, universalMaxVersionName, universalErr
+		}
+	}
+	return maxVersion, maxVersionName, err
 }
 
 // GetPackageFilesVersion return the package version