Merge branch 'develop' into init_files

sagemath · May 13, 2024 · 2e368bc · 2e368bc
2 parents d2fc9a1 + ffbbea9
commit 2e368bc
Show file tree

Hide file tree

Showing 559 changed files with 12,482 additions and 11,130 deletions.
diff --git a/.ci/create-changes-html.sh b/.ci/create-changes-html.sh
@@ -51,7 +51,7 @@ diffParagraphs.forEach(paragraph => {
 EOF
 echo '</head>' >> CHANGES.html
 echo '<body>' >> CHANGES.html
-(cd $DOC_REPOSITORY && git diff $BASE_DOC_COMMIT -- *.html) > diff.txt
+(cd $DOC_REPOSITORY && git diff $BASE_DOC_COMMIT -- "*.html") > diff.txt
 python3 - << EOF
 import os, re, html
 with open('diff.txt', 'r') as f:

diff --git a/.ci/docker-exec-script.sh b/.ci/docker-exec-script.sh
@@ -0,0 +1,14 @@
+#!/bin/sh -x
+if [ $# -lt 3 ]; then
+    echo >&2 "usage: docker-exec-script.sh CONTAINER WORKDIR [VAR=VALUE...] SCRIPT"
+    exit 1
+fi
+CONTAINER=$1
+WORKDIR=$2
+shift 2
+(echo "cd \"$WORKDIR\"";
+ while [ $# -gt 1 ]; do
+     echo "export \"$1\""
+     shift
+ done;
+ cat "$1") | docker exec -i $CONTAINER bash -ex
diff --git a/.ci/merge-fixes.sh b/.ci/merge-fixes.sh
@@ -1,5 +1,8 @@
 #!/bin/sh
-# Apply open PRs labeled "blocker" from sagemath/sage as patches.
+# Apply open PRs labeled "p: CI Fix" from sagemath/sage as patches.
+# (policy set by vote in 2024-03,
+#  https://groups.google.com/g/sage-devel/c/OKwwUGyKveo/m/vpyCXYBqAAAJ)
+#
 # This script is invoked by various workflows in .github/workflows
 #
 # The repository variable SAGE_CI_FIXES_FROM_REPOS can be set
@@ -20,15 +23,15 @@ for REPO in ${SAGE_CI_FIXES_FROM_REPOSITORIES:-sagemath/sage}; do
             echo "Nothing to do for 'none' in SAGE_CI_FIXES_FROM_REPOSITORIES"
             ;;
         */*)
-            echo "Getting open PRs with 'blocker' status from https://github.com/$REPO/pulls?q=is%3Aopen+label%3A%22p%3A+blocker+%2F+1%22"
+            echo "Getting open PRs with 'p: CI Fix' label from https://github.com/$REPO/pulls?q=is%3Aopen+label%3A%22p%3A+CI+Fix%22"
             GH="gh -R $REPO"
             REPO_FILE="upstream/ci-fixes-${REPO%%/*}-${REPO##*/}"
-            PRs="$($GH pr list --label "p: blocker / 1" --json number --jq '.[].number' | tee $REPO_FILE)"
+            PRs="$($GH pr list --label "p: CI Fix" --json number --jq '.[].number' | tee $REPO_FILE)"
             date -u +"%Y-%m-%dT%H:%M:%SZ" > $REPO_FILE.date  # Record the date, for future reference
             if [ -z "$PRs" ]; then
-                echo "Nothing to do: Found no open PRs with 'blocker' status in $REPO."
+                echo "Nothing to do: Found no open PRs with 'p: CI Fix' label in $REPO."
             else
-                echo "Found open PRs with 'blocker' status in $REPO: $(echo $PRs)"
+                echo "Found open PRs with 'p: CI Fix' label in $REPO: $(echo $PRs)"
                 git tag -f test_base
                 git commit -q -m "Uncommitted changes" --no-allow-empty -a
                 for a in $PRs; do

diff --git a/build/bin/write-dockerfile.sh → .ci/write-dockerfile.sh b/build/bin/write-dockerfile.sh → .ci/write-dockerfile.sh
@@ -27,13 +27,13 @@ for SPKG in $(sage-package list --has-file=spkg-configure.m4 $SAGE_PACKAGE_LIST_
         CONFIGURE_ARGS+="--with-system-${SPKG}=${WITH_SYSTEM_SPKG} "
     fi
 done
-echo "# Automatically generated by SAGE_ROOT/build/bin/write-dockerfile.sh"
+echo "# Automatically generated by SAGE_ROOT/.ci/write-dockerfile.sh"
 echo "# the :comments: separate the generated file into sections"
 echo "# to simplify writing scripts that customize this file"
 ADD="ADD $__CHOWN"
 RUN=RUN
 cat <<EOF
-ARG BASE_IMAGE
+ARG BASE_IMAGE=$(eval echo "${FULL_BASE_IMAGE_AND_TAG}")
 FROM \${BASE_IMAGE} as with-system-packages
 EOF
 case $SYSTEM in
@@ -54,8 +54,21 @@ case $SYSTEM in
             1|y*|Y*)
                 ;;
             *)
+                #
+                # The Ubuntu Docker images are "minimized", meaning that some large
+                # bits such as documentation has been removed. We have to unminimize
+                # once (which reinstalls the full versions of some minimized packages),
+                # or e.g. the maxima documentation (which we depend on for correct operation)
+                # will be missing.
+                #
+                # But we only have to do this once. To save time in incremental builds,
+                # we remove the unminimize binary here after it has done its job.
+                #
                 cat <<EOF
-RUN (yes | unminimize) || echo "(ignored)"
+RUN if command -v unminimize > /dev/null; then  \
+        (yes | unminimize) || echo "(ignored)"; \
+        rm -f "$(command -v unminimize)";       \
+    fi
 EOF
                 if [ -n "$DIST_UPGRADE" ]; then
                     cat <<EOF
@@ -143,7 +156,7 @@ EOF
                 ;;
             *)
                 cat <<EOF
-ARG USE_CONDARC=condarc.yml
+ARG USE_CONDARC=${USE_CONDARC-condarc.yml}
 ADD *condarc*.yml /tmp/
 RUN echo \${CONDARC}; cd /tmp && conda config --stdin < \${USE_CONDARC}
 RUN conda update -n base conda
@@ -230,24 +243,41 @@ cat <<EOF
 
 FROM with-system-packages as bootstrapped
 #:bootstrapping:
-RUN if [ -d /sage ]; then echo "### Incremental build from \$(cat /sage/VERSION.txt)" && mv /sage /sage-old && mkdir /sage && for a in local logs; do if [ -d /sage-old/\$a ]; then mv /sage-old/\$a /sage; fi; done; rm -rf /sage-old; else mkdir -p /sage; fi
+$ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap bootstrap-conda configure.ac sage .homebrew-build-env tox.ini Pipfile.m4 .gitignore /new/
+$ADD config/config.rpath /new/config/config.rpath
+$ADD src/doc/bootstrap /new/src/doc/bootstrap
+$ADD src/bin /new/src/bin
+$ADD src/Pipfile.m4 src/pyproject.toml.m4 src/requirements.txt.m4 src/setup.cfg.m4 src/VERSION.txt /new/src/
+$ADD m4 /new/m4
+$ADD pkgs /new/pkgs
+$ADD build /new/build
+$ADD .ci /new/.ci
+$ADD .upstream.d /new/.upstream.d
+RUN if [ -d /sage ]; then                                               \
+        echo "### Incremental build from \$(cat /sage/VERSION.txt)" &&  \
+        if command -v git; then                                         \
+            (cd /new &&                                                 \
+             echo /src >> .gitignore &&                                 \
+             ./.ci/retrofit-worktree.sh worktree-image /sage);          \
+        else                                                            \
+            for a in local logs; do                                     \
+                if [ -d /sage/\$a ]; then mv /sage/\$a /new/; fi;       \
+            done;                                                       \
+            rm -rf /sage;                                               \
+            mv /new /sage;                                              \
+        fi;                                                             \
+    else                                                                \
+        mv /new /sage;                                                  \
+    fi
 WORKDIR /sage
-$ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap bootstrap-conda configure.ac sage .homebrew-build-env tox.ini Pipfile.m4 ./
-$ADD config/config.rpath config/config.rpath
-$ADD src/doc/bootstrap src/doc/bootstrap
-$ADD src/bin src/bin
-$ADD src/Pipfile.m4 src/pyproject.toml.m4 src/requirements.txt.m4 src/setup.cfg.m4 src/VERSION.txt src/
-$ADD m4 ./m4
-$ADD pkgs pkgs
-$ADD build ./build
-$ADD .upstream.d ./.upstream.d
-ARG BOOTSTRAP=./bootstrap
+
+ARG BOOTSTRAP=${BOOTSTRAP-./bootstrap}
 $RUN sh -x -c "\${BOOTSTRAP}" $ENDRUN $THEN_SAVE_STATUS
 
 FROM bootstrapped as configured
 #:configuring:
 RUN $CHECK_STATUS_THEN mkdir -p logs/pkgs; rm -f config.log; ln -s logs/pkgs/config.log config.log
-ARG EXTRA_CONFIGURE_ARGS=""
+ARG EXTRA_CONFIGURE_ARGS="${CONFIGURE_ARGS}"
 EOF
 if [ ${WITH_SYSTEM_SPKG} = "force" ]; then
     cat <<EOF
@@ -286,7 +316,16 @@ ENV MAKE="make -j\${NUMPROC}"
 ARG USE_MAKEFLAGS="-k V=0"
 ENV SAGE_CHECK=warn
 ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!gap,!cysignals,!linbox,!git,!ppl,!cmake,!rpy2,!sage_sws2rst"
-$ADD src src
+$ADD .gitignore /new/.gitignore
+$ADD src /new/src
+RUN if command -v git; then                             \
+        cd /new && rm -rf .git &&                       \
+        ./.ci/retrofit-worktree.sh worktree-pre /sage;  \
+    else                                                \
+        rm -rf /sage/src;                               \
+        mv /new/src /sage/src;                          \
+    fi
+
 ARG TARGETS="build"
 $RUN $CHECK_STATUS_THEN make SAGE_SPKG="sage-spkg -y -o" \${USE_MAKEFLAGS} \${TARGETS} $ENDRUN $THEN_SAVE_STATUS