Trac #29104: Add to tox.ini some homebrew toxenvs (with a fresh insta…

…ll not in /usr/local), add homebrew packages

#29053 adds `tox.ini` to support testing the sage distribution in
isolated environments via docker.

This ticket adds toxenvs `local-homebrew-macos-{minimal,standard}` for
testing with (best-effort) isolated installations of homebrew on macOS
(not using docker).

To test:
{{{
    brew install tox
    tox -e local-homebrew-macos-standard
}}}

See comments in tox.ini for more examples.

An example run of that is here:
https://github.com/mkoeppe/sage/actions/runs/32812271

URL: https://trac.sagemath.org/29104
Reported by: mkoeppe
Ticket author(s): Matthias Koeppe
Reviewer(s): Dima Pasechnik, John Palmieri

.gitignore

@@ -106,3 +106,4 @@ gitlab-build-docker.log
 #######################
 
 /.tox
+/prefix

.homebrew-build-env

@@ -0,0 +1,36 @@
+# Source this -*- shell-script -*- to set some environment variables
+# that activate keg-only homebrew package installations
+
+HOMEBREW=`brew --prefix` || return 1
+for l in gettext; do
+    if [ -d "$HOMEBREW/opt/$l/bin" ]; then
+        PATH="$HOMEBREW/opt/$l/bin:$PATH"
+    fi
+done
+export PATH
+PKG_CONFIG_PATH="$HOMEBREW/lib/pkgconfig:$PKG_CONFIG_PATH"
+# libpng.pc depends on zlib.pc
+for l in openblas openssl readline sqlite zlib; do
+    if [ -d "$HOMEBREW/opt/$l/lib/pkgconfig" ]; then
+       PKG_CONFIG_PATH="$HOMEBREW/opt/$l/lib/pkgconfig:$PKG_CONFIG_PATH"
+    fi
+done
+export PKG_CONFIG_PATH
+LDFLAGS="-L$HOMEBREW/lib $LDFLAGS"
+CPPFLAGS="-I$HOMEBREW/include $CPPFLAGS"
+for l in readline; do
+    if [ -d "$HOMEBREW/opt/$l/lib" ]; then
+        LDFLAGS="-L$HOMEBREW/opt/$l/lib $LDFLAGS"
+    fi
+    if [ -d "$HOMEBREW/opt/$l/include" ]; then
+        CPPFLAGS="-I$HOMEBREW/opt/$l/include $CPPFLAGS"
+    fi
+done
+export LDFLAGS
+export CPPFLAGS
+for l in gettext; do
+    if [ -d "$HOMEBREW/opt/$l/share/aclocal" ]; then
+        ACLOCAL_PATH="$HOMEBREW/opt/$l/share/aclocal:$ACLOCAL_PATH"
+    fi
+done
+export ACLOCAL_PATH

build/pkgs/arb/distros/homebrew.txt

@@ -0,0 +1,3 @@
+## This package depends on ntl, the homebrew package of which we cannot use
+## because it is built with NTL_THREADS.  See https://trac.sagemath.org/ticket/29339
+# sagemath/science/arb

build/pkgs/boost/distros/homebrew.txt

@@ -0,0 +1 @@
+boost

build/pkgs/boost_cropped/distros/homebrew.txt

@@ -0,0 +1 @@
+boost

build/pkgs/cmake/distros/homebrew.txt

@@ -0,0 +1 @@
+cmake

build/pkgs/flint/distros/homebrew.txt

@@ -0,0 +1,3 @@
+## This package depends on ntl, the homebrew package of which we cannot use
+## because it is built with NTL_THREADS.  See https://trac.sagemath.org/ticket/29339
+# sagemath/science/flint

build/pkgs/freetype/distros/homebrew.txt

@@ -0,0 +1 @@
+freetype

build/pkgs/gfortran/distros/homebrew.txt

@@ -0,0 +1 @@
+gcc

build/pkgs/git/distros/homebrew.txt

@@ -0,0 +1 @@
+git

build/pkgs/glpk/distros/homebrew.txt

@@ -0,0 +1,2 @@
+## glpk 4.65 should not be used by Sage as long as it is not patched by Homebrew
+# glpk

build/pkgs/gmp/distros/homebrew.txt

@@ -0,0 +1 @@
+gmp

build/pkgs/gsl/distros/homebrew.txt

@@ -0,0 +1 @@
+gsl

build/pkgs/homebrew-bootstrap.txt

@@ -0,0 +1,2 @@
+# Packages needed for ./bootstrap
+gettext autoconf automake libtool pkg-config

build/pkgs/homebrew.txt

@@ -0,0 +1,12 @@
+# This file, build/pkgs/homebrew.txt, contains names of homebrew packages
+# needed for installation of Sage from source (in addition to XCode).
+#
+# In addition, the files build/pkgs/SPKG/homebrew.txt contain the names
+# of packages that provide the equivalent of SPKG.
+#
+# See build/bin/sage-spkg, where this information is processed
+# for use in "sage -info SPKG".
+#
+# Everything on a line after a # character is ignored.
+
+# No packages needed

build/pkgs/isl/distros/homebrew.txt

@@ -0,0 +1 @@
+isl

build/pkgs/libpng/distros/homebrew.txt

@@ -0,0 +1 @@
+libpng

build/pkgs/mpc/distros/homebrew.txt

@@ -0,0 +1 @@
+libmpc

build/pkgs/mpfi/distros/homebrew.txt

@@ -0,0 +1 @@
+mpfi

build/pkgs/mpfr/distros/homebrew.txt

@@ -0,0 +1 @@
+mpfr

build/pkgs/ninja_build/distros/homebrew.txt

@@ -0,0 +1 @@
+ninja

build/pkgs/ntl/distros/homebrew.txt

@@ -0,0 +1,3 @@
+## We cannot use ntl on homebrew because it is built with NTL_THREADS.
+## See https://trac.sagemath.org/ticket/29339
+# ntl

build/pkgs/openblas/distros/homebrew.txt

@@ -0,0 +1 @@
+openblas

build/pkgs/openssl/distros/homebrew.txt

@@ -0,0 +1 @@
+openssl

build/pkgs/pari/distros/homebrew.txt

@@ -0,0 +1,2 @@
+## cannot currently be used by Sage because the pari packages are not in Homebrew
+#pari

build/pkgs/patch/distros/homebrew.txt

@@ -0,0 +1 @@
+gpatch

build/pkgs/pcre/distros/homebrew.txt

@@ -0,0 +1 @@
+pcre

build/pkgs/pkgconf/distros/homebrew.txt

@@ -0,0 +1 @@
+pkg-config

build/pkgs/r/distros/homebrew.txt

@@ -0,0 +1 @@
+r

build/pkgs/readline/distros/homebrew.txt

@@ -0,0 +1 @@
+readline

build/pkgs/sqlite/distros/homebrew.txt

@@ -0,0 +1 @@
+sqlite

build/pkgs/xz/distros/homebrew.txt

@@ -0,0 +1 @@
+xz

build/pkgs/yasm/distros/alpine.txt

@@ -0,0 +1 @@
+yasm

build/pkgs/yasm/distros/homebrew.txt

@@ -0,0 +1 @@
+yasm

build/pkgs/yasm/distros/opensuse.txt

@@ -0,0 +1 @@
+yasm

build/pkgs/zeromq/distros/homebrew.txt

@@ -0,0 +1 @@
+zeromq

build/pkgs/zlib/distros/homebrew.txt

@@ -0,0 +1 @@
+zlib

tox.ini

@@ -7,8 +7,13 @@
 [tox]
 ### Test that the system packages listed in debian.txt/fedora.txt files of standard spkg exist
 ### and satisfy the requirements tested by spkg-configure.m4, then compile a few packages.
+###
 envlist =
     check_configure,
+    {
+    ### "docker" toxenvs copy sources from the source tree subject to the exclusions in
+    ### the file ".dockerignore".  This should work out of non-clean source trees, and all
+    ### "docker" toxenvs can be run in parallel.
     docker-{ubuntu-{trusty,xenial,bionic,latest,eoan,rolling,focal,devel},
             debian-{jessie,stretch,buster,bullseye,sid},
             fedora-{26,27,28,29,30,31,32},
@@ -22,15 +27,44 @@ envlist =
              # windows-amd64  #  https://hub.docker.com/u/winamd64/
              # Other architectures built by official images:
              # (but not officially supported by Docker, Inc.)
-             arm32v5,ppc64le,s390x,i386}
+             arm32v5,ppc64le,s390x,i386},
     ### "local" targets should be run from a source tree that is freshly checked out
     ### (for example, by 'git worktree add ...') or has been cleaned by 'make bdist-clean' --
     ### because they build within the source tree (because we have no VPATH support).
     ### Only one "local" target can be run at a time.
     ### However, "local" targets install in a separate prefix (SAGE_LOCAL=.tox/TOXENV/local)
     ### rather than "local/" and also place log files into .tox/TOXENV/log, where TOXENV
     ### is the name of the environment.
-   -{minimal,standard,maximal},
+    #
+    # The "local-homebrew" toxenvs create an "isolated" homebrew installation (not in /usr/local).
+    # (However, many configure scripts still look for stuff in /usr/local.)
+    #
+    # By default, it runs "make build" (after bootstrapping and configuring)
+    # with SAGE_CHECK=yes and SAGE_CHECK_PACKAGES set to a good default,
+    # and V=0 so that screen output is limited.
+    #
+    #   $ tox -e local-homebrew-macos-standard
+    #
+    # The value of MAKE is picked up from the environment, so you can do:
+    #
+    #   $ MAKE="make -j8" tox -e local-homebrew-macos-standard
+    #
+    # Build targets can be passed as positional arguments (separated from tox options by "--"):
+    #
+    #   $ tox -e local-homebrew-macos-standard -- ppl
+    #   $ tox -e local-homebrew-macos-standard -- build ptest
+    #
+    # Also make variables can be passed as positional arguments; for example, to run the
+    # test suite:
+    #
+    #   $ tox -e local-homebrew-macos-standard -- SAGE_CHECK=no build ptest SAGE_NUM_THREADS=4
+    #
+    # Or to rebuild a package with verbose output:
+    #
+    #   $ tox -e local-homebrew-macos-standard -- ppl-clean ppl V=1
+    #
+    local-homebrew-macos
+   }-{minimal,standard,maximal},
    ### The "local-direct" toxenv passes the whole environment on to the sage build.
    ### Whatever is in PATH etc. will be used.
    local-direct
@@ -39,7 +73,9 @@ skipsdist = true
 
 [testenv]
 passenv =
-    EXTRA_DOCKER_BUILD_ARGS
+    docker:       EXTRA_DOCKER_BUILD_ARGS
+    local:        MAKE
+    local-direct: *
 setenv =
     # Set this to 'force' instead of 'yes' to make it an error if an spkg with spkg-configure and system package
     # is not recognized.
@@ -159,16 +195,37 @@ setenv =
     ###
     ### "local" envs
     ###
+    homebrew:         SYSTEM=homebrew
     local:            HOME={envdir}
+    local:            SETENV=:
     local-!direct:    PATH=/usr/bin:/bin:/usr/sbin:/sbin
+    # brew caches downloaded files in ${HOME}/Library/Caches. We share it between different toxenvs.
+    local-homebrew:   HOMEBREW={envdir}/homebrew
+    local-homebrew:   PATH={env:HOMEBREW}/bin:/usr/bin:/bin:/usr/sbin:/sbin
+    local-homebrew:   SETENV=. .homebrew-build-env
 
 # environment will be skipped if regular expression does not match against the sys.platform string
 platform =
     local-macos:   darwin
 
 whitelist_externals =
-    bash
-    docker
+                   bash
+    docker:        docker
+    homebrew:      brew
+
+commands_pre =
+    # https://docs.brew.sh/Installation
+    homebrew:      bash -c 'if [ ! -d {env:HOME}/Library/Caches ]; then mkdir -p {toxworkdir}/Caches && mkdir -p {env:HOME}/Library && ln -sf {toxworkdir}/Caches {env:HOME}/Library/; fi'
+    homebrew:      bash -c 'if [ ! -x {env:HOMEBREW}/bin/brew ]; then mkdir -p {envdir}/homebrew && cd {envdir}/homebrew && curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 ; fi'
+    # homebrew-mininmal: Minimal for bootstrapping and build.
+    # gettext and pkg-config are needed so that their aclocal macros are available.
+    homebrew:      bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/homebrew*.txt`; {env:HOMEBREW}/bin/brew install $PACKAGES; {env:HOMEBREW}/bin/brew upgrade $PACKAGES'
+    homebrew-standard:   bash -c 'PACKAGES=`sed "s/#.*//;" build/pkgs/*/distros/homebrew.txt`; {env:HOMEBREW}/bin/brew install $PACKAGES; {env:HOMEBREW}/bin/brew upgrade $PACKAGES'
+    # Install symbolic links "config.log" and "logs" in SAGE_ROOT so that log files are written into the tox log directory.
+    local:         bash -c 'touch {envlogdir}/config.log; ln -sf {envlogdir}/config.log .; if [ ! -d logs -o -L logs ]; then rm -f logs; ln -sf {envlogdir} logs; fi'
+    # Install a symbolic link "prefix" in SAGE_ROOT for convenient inspection; it is not used in the build.
+    local:         bash -c 'if [ ! -d prefix -o -L prefix ]; then rm -f prefix; ln -sf {envdir}/local prefix; fi'
+
 commands =
     docker:        bash -c 'build/bin/write-dockerfile.sh {env:SYSTEM} "{env:TYPE_PATTERN:}" {env:WITH_SYSTEM_SPKG} {env:IGNORE_MISSING_SYSTEM_PACKAGES} > {envdir}/Dockerfile'
     # From https://hub.docker.com/r/multiarch/ubuntu-core/
@@ -179,6 +236,9 @@ commands =
     docker:        --build-arg BASE_IMAGE={env:FULL_BASE_IMAGE_AND_TAG}           \
     docker-conda:  --build-arg USE_CONDARC="{env:CONDARC}"                        \
     docker:        {env:EXTRA_DOCKER_BUILD_ARGS:}
+    # pathpy checksuite needs tox. #28728: gap fails its test suite.
+    # linbox/cysignals testsuites fail.
+    local:         bash -c 'export PATH={env:PATH} && {env:SETENV} && ./bootstrap && ./configure --prefix={envdir}/local && make -k V=0 base-toolchain && SAGE_CHECK=yes SAGE_CHECK_PACKAGES="!cython,!r,!python3,!python2,!nose,!pathpy,!gap,!cysignals,!linbox" make -k V=0 {posargs:build}'
 
 [testenv:check_configure]
 ## Test that configure behaves properly