Merge branch 'develop' into t/20622/public/combinat/species-refactoring

.ci/README.md

@@ -0,0 +1,9 @@
+# Continuous Integration (CI)
+
+We support several implementations of CI. All these implementations rely on
+[docker](https://docker.com) in some way. This directory contains bits which
+are shared between these CI implementations. The relevant docker files can be
+found in `/docker/`.
+
+* [CircleCI](https://circleci.com) is configured in `/.circleci/`.
+* [GitLab CI](https://gitlab.com) is configured in `/.gitlab-ci.yml`.

.ci/build-docker.sh

@@ -0,0 +1,59 @@
+#!/bin/sh
+
+# This script gets called from CI to build several flavours of docker images
+# which contain Sage.
+
+# ****************************************************************************
+#       Copyright (C) 2018 Julian Rüth <julian.rueth@fsfe.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+# ****************************************************************************
+
+set -ex
+
+# We speed up the build process by copying built artifacts from ARTIFACT_BASE
+# during docker build. See /docker/Dockerfile for more details.
+ARTIFACT_BASE=${ARTIFACT_BASE:-sagemath/sagemath-dev:develop}
+
+# Seed our cache with $ARTIFACT_BASE if it exists.
+docker pull "$ARTIFACT_BASE" > /dev/null || true
+
+docker_build() {
+    # Docker's --cache-from does not really work with multi-stage builds: https://github.com/moby/moby/issues/34715
+    # So we just have to rely on the local cache.
+    time docker build -f docker/Dockerfile \
+--build-arg "--build-arg "MAKEFLAGS=${MAKEFLAGS}" --build-arg "SAGE_NUM_THREADS=${SAGE_NUM_THREADS}" --build-arg "MAKEFLAGS_DOCBUILD=${MAKEFLAGS}" --build-arg "SAGE_NUM_THREADS_DOCBUILD=${SAGE_NUM_THREADS_DOCBUILD}" --build-arg ARTIFACT_BASE=$ARTIFACT_BASE $@
+}
+
+# We use a multi-stage build /docker/Dockerfile. For the caching to be
+# effective, we populate the cache by building the run/build-time-dependencies
+# and the make-all target. (Just building the last target is not enough as
+# intermediate targets could be discarded from the cache [depending on the
+# docker version] and therefore the caching would fail for our actual builds
+# below.)
+docker_build --target run-time-dependencies --tag run-time-dependencies:$DOCKER_TAG .
+docker_build --target build-time-dependencies --tag build-time-dependencies:$DOCKER_TAG .
+docker_build --target make-all --tag make-all:$DOCKER_TAG .
+
+# Copy docs out of the docker image to save them into browseable GitLab artifacts
+container=$(docker create make-all:$DOCKER_TAG)
+docker cp $container:/home/sage/sage/local/share/doc/sage/html html_
+# GitLab's browser does not like symlinks, so we flatten them
+rsync html_/ html/ -a --copy-links
+
+# Build the release image without build artifacts.
+docker_build --target sagemath --tag "$DOCKER_IMAGE_CLI" .
+# Tag the sagemath:$DOCKER_TAG image that CI has just built as
+# sagemath:$COMMIT_HASH so we can refer to it uniquely later.
+docker tag "$DOCKER_IMAGE_CLI" "$DOCKER_IMAGE_BINDER"
+# Display the layers of this image
+docker history "$DOCKER_IMAGE_CLI"
+# Build the developer image with the build artifacts intact.
+# Note: It's important to build the dev image last because it might be tagged as ARTIFACT_BASE.
+docker_build --target sagemath-dev --tag "$DOCKER_IMAGE_DEV" .
+# Display the layers of this image
+docker history "$DOCKER_IMAGE_DEV"

.ci/describe-system.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+# ****************************************************************************
+#       Copyright (C) 2018 Julian Rüth <julian.rueth@fsfe.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+# ****************************************************************************
+
+set +e -x
+
+docker info
+docker run docker sh -c "
+  set -x
+  uname -a
+  df -h
+  cat /proc/cpuinfo
+  cat /proc/meminfo
+  cat /proc/sys/vm/overcommit_memory
+  cat /proc/sys/vm/overcommit_ratio"

.ci/head-tail.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+
+OFFSET=1024
+# This script reads from stdin and prints to stdout as long as a the output
+# does not exceed a certain number of bytes. When reading an EOF it prints the
+# last $OFFSET lines if they have not been printed normally already.
+# This script expects one argument, the number of bytes.
+
+# Heavily inspired by a simlar strategy in make, https://stackoverflow.com/a/44849696/812379.
+
+# ****************************************************************************
+#       Copyright (C) 2018 Julian Rüth <julian.rueth@fsfe.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+# ****************************************************************************
+
+stdbuf -i0 -o0 -e0 awk -v limit=$1 -v firstMissingNR=-1 -v offset=$OFFSET -v bytes=0 \
+'{
+  if (bytes < limit) {
+    # this probably gets multi-byte characters wrong, but that probably does
+    # not matter for our purposes. (We add 1 for a UNIX newline.)
+    bytes += length($0) + 1;
+    print;
+  } else {
+    if (firstMissingNR == -1){
+      print "[…output truncated…]";
+      firstMissingNR = NR;
+    }
+    a[NR] = $0;
+    delete a[NR-offset];
+    printf "." > "/dev/stderr"
+    if (/^sage-logger/){
+      print > "/dev/stderr"
+    }
+  }
+}
+END {
+  if (firstMissingNR != -1) {
+    print "" > "/dev/stderr";
+    for(i = NR-offset+1 > firstMissingNR ? NR-offset-1 : firstMissingNR; i<=NR ; i++){ print a[i]; }
+  }
+}
+'
+

.ci/protect-secrets.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+# This script protects all environment variables that start with "SECRET_".
+# It puts them in a temporary file. The name of the variable contains the path
+# of that file.  This filename can then safely be used in `cat` even if `set
+# -x` has been turned on. Also you can run "export" to understand the
+# environment without danger.
+# Be careful, however, not to use this like the following:
+# docker login $DOCKER_USER $(cat $SECRET_DOCKER_PASS)
+# as this would expose the password if `set -x` has been turned on.
+
+# ****************************************************************************
+#       Copyright (C) 2018 Julian Rüth <julian.rueth@fsfe.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+# ****************************************************************************
+
+set -eo pipefail
+set +x
+
+function encrypt {
+    RET=`mktemp`
+    eval " echo \$$1" > "$RET"
+    echo $RET
+}
+
+for name in `awk 'END { for (name in ENVIRON) { print name; } }' < /dev/null`; do
+case "$name" in
+  SECRET_*)
+    export $name="$(encrypt $name)"
+    echo "Protected $name"
+    ;;
+esac
+done
+
+unset encrypt

.ci/pull-gitlab.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+# This script gets called from CI to pull the Sage docker images that were
+# built during the "build" phase to pull all the connected docker daemon
+# (likely a docker-in-docker.)
+# This script expects a single parameter, the base name of the docker image
+# such as sagemath or sagemath-dev.
+# The variable $DOCKER_IMAGE is set to the full name of the pulled image;
+# source this script to use it.
+
+# ****************************************************************************
+#       Copyright (C) 2018 Julian Rüth <julian.rueth@fsfe.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+# ****************************************************************************
+
+set -ex
+
+# Pull the built images from the gitlab registry and give them the original
+# names they had after built.
+# Note that "set -x" prints the $CI_BUILD_TOKEN here but GitLab removes it
+# automatically from the log output.
+docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
+docker pull $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG
+export DOCKER_IMAGE="${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG"
+docker tag $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG $DOCKER_IMAGE

.ci/push-dockerhub.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+# This script gets called from CI to push our docker images to
+# $DOCKER_NAMESPACE/sagemath* on the Docker Hub.
+# This script expects a single parameter, the base name of the docker image
+# such as sagemath or sagemath-dev.
+
+# ****************************************************************************
+#       Copyright (C) 2018 Julian Rüth <julian.rueth@fsfe.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+# ****************************************************************************
+
+set -ex
+
+[ -z "$DOCKER_TAG" ] && (echo "Can not push untagged build."; exit 0)
+
+# Push the built images to the docker hub (and fail silently if
+# DOCKER_USER/SECRET_DOCKER_PASS have not been configured.)
+if [ -z "$DOCKER_USER" -o -z "$SECRET_DOCKER_PASS" ]; then
+    echo "DOCKER_USER/SECRET_DOCKER_PASS variables have not been configured in your Continuous Integration setup. Not pushing built images to Docker Hub."
+else
+  cat "$SECRET_DOCKER_PASS" | docker login -u $DOCKER_USER --password-stdin
+  docker push ${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG
+
+  # For historical reasons, we also provide a -py3 tag. It's identical to the non-py3 tag.
+  docker tag ${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG ${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG-py3
+  docker push ${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG-py3
+fi

.ci/push-gitlab.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+# This script gets called from CI to push our docker images to registry
+# configured in GitLab. (Mostly, so we can pull them again to push them to the
+# Docker Hub.)
+# This script expects a single parameter, the base name of the docker image
+# such as sagemath or sagemath-dev.
+
+# ****************************************************************************
+#       Copyright (C) 2018 Julian Rüth <julian.rueth@fsfe.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+# ****************************************************************************
+
+set -ex
+
+# Note that "set -x" prints the $CI_BUILD_TOKEN here but GitLab removes it
+# automatically from the log output.
+docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
+docker tag ${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG
+docker push $CI_REGISTRY_IMAGE/$1:$DOCKER_TAG

.ci/setup-make-parallelity.sh

@@ -0,0 +1,65 @@
+#!/bin/sh
+
+# Source this to set CPUTHREADS (the number of apparent cores) and RAMTHREADS
+# (free RAM divided by the maximum amount needed per thread typically)
+# From this this script infers reasonable defaults for SAGE_NUM_THREADS and
+# MAKEFLAGS.
+
+# We do exactly the same for CPUTHREADS_DOCBUILD, RAMTHREADS_DOCBUILD,
+# SAGE_NUM_THREADS_DOCBUILD, MAKEFLAGS_DOCBUILD. As the docbuild needs
+# substantially more RAM as of May 2018.
+
+# ****************************************************************************
+#       Copyright (C) 2018 Julian Rüth <julian.rueth@fsfe.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+# ****************************************************************************
+
+set -ex
+
+if [ -z "$CPUTHREADS" ]; then
+    # Determine the number of threads that can run simultaneously on this system
+    # (we might not have nproc available.)
+    # Note that this value is incorrect for some CI providers (notably CircleCI:
+    # https://circleci.com/docs/2.0/configuration-reference/#resource_class) which
+    # provision fewer vCPUs than shown in /proc/cpuinfo. So it is probably better
+    # to set CPUTHREADS manuall in your CI configuration.
+    CPUTHREADS=`docker run docker cat /proc/cpuinfo | grep -E '^processor' | wc -l`
+fi
+if [ -z "$CPUTHREADS_DOCBUILD" ]; then
+    CPUTHREADS_DOCBUILD=$CPUTHREADS
+fi
+
+if [ -z "$RAMTHREADS" ]; then
+    RAMTHREADS=$(( `docker run docker cat /proc/meminfo | grep MemTotal | awk '{ print $2 }'` / 1048576 ))
+    if [ $RAMTHREADS = 0 ];then
+        RAMTHREADS=1;
+    fi
+fi
+if [ -z "$RAMTHREADS_DOCBUILD" ]; then
+    RAMTHREADS_DOCBUILD=$(( `docker run docker cat /proc/meminfo | grep MemTotal | awk '{ print $2 }'` / 2097152 ))
+    if [ $RAMTHREADS_DOCBUILD = 0 ];then
+        RAMTHREADS_DOCBUILD=1;
+    fi
+fi
+
+# On CI machines with their virtual CPUs, it seems to be quite beneficial to
+# overcommit on CPU usage. We only need to make sure that we do not exceed RAM
+# (as there is no swap.)
+if [ $CPUTHREADS -lt $RAMTHREADS ]; then
+    export SAGE_NUM_THREADS=$((CPUTHREADS + 1))
+else
+    export SAGE_NUM_THREADS=$RAMTHREADS
+fi
+if [ $CPUTHREADS_DOCBUILD -lt $RAMTHREADS_DOCBUILD ]; then
+    export SAGE_NUM_THREADS_DOCBUILD=$((CPUTHREADS_DOCBUILD + 1))
+else
+    export SAGE_NUM_THREADS_DOCBUILD=$RAMTHREADS_DOCBUILD
+fi
+# Set -j and -l for make (though -l is probably ignored by Sage)
+export MAKEFLAGS="-j $SAGE_NUM_THREADS -l $((CPUTHREADS - 1)).8"
+export MAKEFLAGS_DOCBUILD="-j $SAGE_NUM_THREADS_DOCBUILD -l $((CPUTHREADS_DOCBUILD - 1)).8"

.ci/test-cli.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+# This script gets called from CI to run minimal tests on the sagemath image.
+
+# Usage: ./test-cli.sh IMAGE-NAME
+
+# ****************************************************************************
+#       Copyright (C) 2018 Julian Rüth <julian.rueth@fsfe.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+# ****************************************************************************
+
+set -ex
+
+echo "Checking that Sage starts and can calculate 1+1…"
+# Calculate 1+1 (remove startup messages and leading & trailing whitespace)
+TWO=`docker run "$1" sage -c "'print(1+1)'" | tail -1 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'`
+[ "x$TWO" = "x2" ]
+
+echo "Checking that some binaries that should be distributed with Sage are on the PATH…"
+# We could also run minimal tests on these but we don't yet.
+# Check that Singular and GAP are present
+docker run "$1" which Singular
+docker run "$1" which gap
+# Check that jupyter is present (for binder)
+docker run "$1" which jupyter