Skip to content
This repository has been archived by the owner on Jan 30, 2023. It is now read-only.

Commit

Permalink
Merge branch 't/29233/bootstrap____make_distclean_broken' into t/2904…
Browse files Browse the repository at this point in the history
…1/at___bootstrap_time__generate_src_requirements_txt__src_constraints_txt__src_setup_cfg__install_requires__from_build_pkgs
  • Loading branch information
Matthias Koeppe committed Feb 22, 2020
2 parents cfcc9fd + 0dc97db commit bae1a82
Show file tree
Hide file tree
Showing 733 changed files with 22,729 additions and 5,048 deletions.
12 changes: 12 additions & 0 deletions .lgtm.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,18 @@ queries:
- exclude: py/call/wrong-named-class-argument
- exclude: py/call/wrong-number-class-arguments
- exclude: py/unsafe-cyclic-import
path_classifiers:
imports_only:
- "**/all.py"
- "**/*catalog*.py"
- "**/species/library.py"
- "**/categories/basic.py"
- "**/combinat/ribbon.py"
- "**/interacts/geometry.py"
- "**/matroids/advanced.py"
- "**/matroids/named_matroids.py"
- "**/modular/congroup.py"
- "**/quadratic_forms/quadratic_form__mass.py"
extraction:
python:
python_setup:
Expand Down
5 changes: 4 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ misc-clean:
rm -f aclocal.m4 config.log confcache
rm -rf autom4te.cache
rm -f build/make/Makefile build/make/Makefile-auto
rm -f src/lib/pkgconfig/*

bdist-clean: clean
$(MAKE) misc-clean
Expand All @@ -101,6 +102,8 @@ distclean: build-clean
# source tarball
bootstrap-clean:
rm -rf config configure build/make/Makefile-auto.in
rm -f src/requirements.txt src/constraints.txt src/install-requires.txt
rm -f src/doc/en/installation/*.txt

# Remove absolutely everything which isn't part of the git repo
maintainer-clean: distclean bootstrap-clean
Expand Down Expand Up @@ -192,7 +195,7 @@ ptestoptional: all
ptestoptionallong: all
$(PTESTALL) --long --logfile=logs/ptestoptionallong.log

configure: configure.ac src/bin/sage-version.sh m4/*.m4 build/pkgs/*/spkg-configure.m4
configure: bootstrap src/doc/bootstrap configure.ac src/bin/sage-version.sh m4/*.m4 build/pkgs/*/spkg-configure.m4 build/pkgs/*.txt build/pkgs/*/distros/*.txt
./bootstrap -d

install: all
Expand Down
2 changes: 1 addition & 1 deletion VERSION.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
SageMath version 9.1.beta0, Release Date: 2020-01-10
SageMath version 9.1.beta5, Release Date: 2020-02-21
25 changes: 12 additions & 13 deletions bootstrap
Original file line number Diff line number Diff line change
Expand Up @@ -75,25 +75,24 @@ install_config_rpath() {
bootstrap () {
rm -f m4/sage_spkg_configures.m4
spkg_configures=""
for filename in $(find build/pkgs -type f -name spkg-configure.m4); do
for filename in $(find build/pkgs -type f -name spkg-configure.m4 | sort); do
pkgname="$(echo $filename | cut -d/ -f3)"
echo "m4_sinclude([$filename])" >> m4/sage_spkg_configures.m4
spkg_configures="$spkg_configures
SAGE_SPKG_CONFIGURE_$(echo ${pkgname} | tr '[a-z]' '[A-Z]')"
done
for filename in $(find build/pkgs -type f -name type); do
pkgtype="$(cat $filename)"
pkgname="$(echo $filename | cut -d/ -f3)"
case "$pkgtype" in
optional|experimental)
spkg_configures="$spkg_configures
SAGE_SPKG_ENABLE([$pkgname], [$pkgtype])" ;;
esac
done
echo "$spkg_configures" >> m4/sage_spkg_configures.m4

# https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format
(cd src && for list in requirements.txt constraints.txt install-requires.txt; do
(echo "# Automatically generated by ../bootstrap; do not edit"
echo "#"
for filename in $(find ../build/pkgs -type f -name sagelib-$list); do
echo "# From $filename:"
cat $filename
echo ""
done) > $list
done)

SAGE_ROOT="$SAGE_ROOT" src/bootstrap && \
install_config_rpath && \
aclocal -I m4 && \
automake --add-missing --copy build/make/Makefile-auto && \
Expand Down Expand Up @@ -156,7 +155,7 @@ save () {
# Create configure tarball
echo "Creating $NEWCONFBALL..."
mkdir -p upstream
tar zcf "$NEWCONFBALL" configure config/* build/make/Makefile-auto.in
tar zcf "$NEWCONFBALL" configure config/* build/make/Makefile-auto.in src/doc/en/installation/*.txt

# Update version
echo "$NEWCONFVERSION" >$PKG/package-version.txt
Expand Down
35 changes: 35 additions & 0 deletions build/bin/sage-print-system-package-command
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
#!/usr/bin/env bash
#
system=$1
command=$2
if [ -z "$system" -o -z "$command" ]; then
echo >&2 "usage: $0 {debian|arch|conda|...} {install|remove|...} PACKAGES..."
exit 1
fi
shift 2
system_packages="$*"
if [ -z "$system_packages" ]; then
# No command needed to install nothing
exit 0
fi
shopt -s extglob
case $system:$command in
debian*|ubuntu*)
echo "sudo apt-get $command $system_packages"
;;
@(fedora*|redhat*|centos*):install)
echo "sudo yum install $system_packages"
;;
arch*:install)
echo "sudo pacman -S $system_packages"
;;
*conda*:install)
echo "conda install $system_packages"
;;
homebrew*:install)
echo "brew install $system_packages"
;;
*)
echo "# $command the following packages: $system_packages"
;;
esac
39 changes: 39 additions & 0 deletions build/bin/sage-spkg
Original file line number Diff line number Diff line change
Expand Up @@ -349,10 +349,49 @@ fi
if [ $INFO -ne 0 -a "$USE_LOCAL_SCRIPTS" = yes ]; then
cat "$PKG_SCRIPTS/SPKG.txt"
if [ -r "$PKG_SCRIPTS/type" ] ; then
echo
echo "== Type =="
echo
cat "$PKG_SCRIPTS/type"
echo
fi
echo "== Equivalent System Packages =="
echo
PKG_DISTROS="$PKG_SCRIPTS"/distros
for system_package_file in "$PKG_DISTROS"/debian*.txt "$PKG_DISTROS"/fedora*.txt "$PKG_DISTROS"/*conda*.txt "$PKG_DISTROS"/homebrew*.txt "$PKG_DISTROS"/arch*.txt ; do
if [ -f "$system_package_file" ]; then
system=$(basename "$system_package_file" .txt)
system_packages="$(echo $(sed 's/#.*//;' $system_package_file))"
case $system in
debian)
# Generic
echo "Debian/Ubuntu:"
;;
fedora)
# Generic
echo "Fedora/Redhat/CentOS:"
;;
*)
echo "$system:"
;;
esac
echo -n " "
sage-print-system-package-command $system install $system_packages
fi
done
if [ -z "$system" ]; then
echo "(none known)"
else
echo
if [ -f "$PKG_SCRIPTS"/spkg-configure.m4 ]; then
echo "If the system package is installed, ./configure will check whether it can be used."
else
echo "However, these system packages will not be used for building Sage"
echo "because spkg-configure.m4 has not been written for this package;"
echo "see https://trac.sagemath.org/ticket/27330"
fi
fi
echo
exit 0
fi

Expand Down
169 changes: 169 additions & 0 deletions build/bin/write-dockerfile.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
#! /usr/bin/env bash
## Write a Dockerfile to stdout that tests that the packages listed in the debian.txt/fedora.txt files of standard spkg exist
## and satisfy the requirements tested by spkg-configure.m4
## This is called by $SAGE_ROOT/tox.ini
set -e
SYSTEM="${1:-debian}"
shopt -s extglob
TYPE_PATTERN="${2:-standard}"
WITH_SYSTEM_SPKG="${3:-yes}"
IGNORE_MISSING_SYSTEM_PACKAGES="${4:-no}"
#
STRIP_COMMENTS="sed s/#.*//;"
SAGE_ROOT=.
SYSTEM_PACKAGES=$(echo $(${STRIP_COMMENTS} $SAGE_ROOT/build/pkgs/$SYSTEM{,-bootstrap}.txt))
CONFIGURE_ARGS="--enable-option-checking "
for PKG_SCRIPTS in build/pkgs/*; do
if [ -d $PKG_SCRIPTS ]; then
PKG_BASE=$(basename $PKG_SCRIPTS)
SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/distros/$SYSTEM.txt
PKG_TYPE=$(cat $PKG_SCRIPTS/type)
if [ -f $SYSTEM_PACKAGES_FILE -a -f $PKG_SCRIPTS/spkg-configure.m4 ]; then
PKG_SYSTEM_PACKAGES=$(echo $(${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE))
if [ -n "PKG_SYSTEM_PACKAGES" ]; then
case "$PKG_TYPE" in
$TYPE_PATTERN)
SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES"
if [ -f $PKG_SCRIPTS/spkg-configure.m4 ]; then
CONFIGURE_ARGS+="--with-system-$PKG_BASE=${WITH_SYSTEM_SPKG} "
fi
;;
esac
fi
fi
fi
done
echo "# Automatically generated by SAGE_ROOT/build/bin/write-dockerfile.sh"
echo "# the :comments: separate the generated file into sections"
echo "# to simplify writing scripts that customize this file"
RUN=RUN
case $SYSTEM in
debian*|ubuntu*)
cat <<EOF
ARG BASE_IMAGE=ubuntu:latest
FROM \${BASE_IMAGE}
EOF
UPDATE="apt-get update &&"
INSTALL="DEBIAN_FRONTEND=noninteractive apt-get install -qqq --no-install-recommends --yes"
CLEAN="&& apt-get clean"
;;
fedora*|redhat*|centos*)
cat <<EOF
ARG BASE_IMAGE=fedora:latest
FROM \${BASE_IMAGE}
EOF
INSTALL="yum install -y"
;;
arch*)
# https://hub.docker.com/_/archlinux/
cat <<EOF
ARG BASE_IMAGE=archlinux:latest
FROM \${BASE_IMAGE}
EOF
INSTALL="pacman -Syu --noconfirm"
;;
conda*)
cat <<EOF
ARG BASE_IMAGE=continuumio/miniconda3:latest
FROM \${BASE_IMAGE}
ARG USE_CONDARC=condarc.yml
ADD *condarc*.yml /tmp/
RUN echo \${CONDARC}; cd /tmp && conda config --stdin < \${USE_CONDARC}
RUN conda update -n base conda
RUN ln -sf /bin/bash /bin/sh
EOF
# On this image, /bin/sh -> /bin/dash;
# but some of the scripts in /opt/conda/etc/conda/activate.d
# from conda-forge (as of 2020-01-27) contain bash-isms:
# /bin/sh: 5: /opt/conda/etc/conda/activate.d/activate-binutils_linux-64.sh: Syntax error: "(" unexpected
# The command '/bin/sh -c . /opt/conda/etc/profile.d/conda.sh; conda activate base; ./bootstrap' returned a non-zero code
# We just change the link to /bin/bash.
INSTALL="conda install --update-all --quiet --yes"
EXISTS="2>/dev/null >/dev/null conda search -f"
#EXISTS="conda search -f"
CLEAN="&& conda info; conda list"
RUN="RUN . /opt/conda/etc/profile.d/conda.sh; conda activate base; " # to activate the conda env
;;
*)
echo "Not implemented: package installation for SYSTEM=$SYSTEM" >&2
exit 1
;;
esac
cat <<EOF
#:packages:
ENV PACKAGES="$SYSTEM_PACKAGES"
EOF
case "$IGNORE_MISSING_SYSTEM_PACKAGES" in
no)
cat <<EOF
RUN $UPDATE $INSTALL $SYSTEM_PACKAGES $CLEAN
EOF
;;
yes)
if [ -n "$EXISTS" ]; then
# Filter by existing packages, try to install these in one shot; fall back to one by one.
cat <<EOF
RUN $UPDATE EXISTING_PACKAGES=""; for pkg in \$PACKAGES; do echo -n .; if $EXISTS \$pkg; then EXISTING_PACKAGES="\$EXISTING_PACKAGES \$pkg"; echo -n "\$pkg"; fi; done; $INSTALL \$EXISTING_PACKAGES || (echo "Trying again one by one:"; for pkg in \$EXISTING_PACKAGES; do echo "Trying to install \$pkg"; $INSTALL \$pkg || echo "(ignoring error)"; done); : $CLEAN
EOF
else
# Try in one shot, fall back to one by one. Separate "RUN" commands
# for caching by docker.
cat <<EOF
RUN $UPDATE $INSTALL \${PACKAGES} || echo "(ignoring error)"
EOF
for pkg in $SYSTEM_PACKAGES; do
cat <<EOF
RUN $INSTALL $pkg || echo "(ignoring error)"
EOF
done
if [ -n "$CLEAN" ]; then
cat <<EOF
RUN : $CLEAN
EOF
fi
fi
;;
*)
echo "Argument IGNORE_MISSING_SYSTEM_PACKAGES must be yes or no"
;;
esac
cat <<EOF
#:bootstrapping:
RUN mkdir -p /sage
WORKDIR /sage
ADD Makefile VERSION.txt README.md bootstrap configure.ac sage ./
ADD src/doc/bootstrap src/doc/bootstrap
ADD m4 ./m4
ADD build ./build
ADD src/bin/sage-version.sh src/bin/sage-version.sh
$RUN ./bootstrap
#:configuring:
ADD src/ext src/ext
ADD src/bin src/bin
ADD src/Makefile.in src/Makefile.in
ARG EXTRA_CONFIGURE_ARGS=""
EOF
if [ ${WITH_SYSTEM_SPKG} = "force" ]; then
cat <<EOF
$RUN echo "****** Configuring: ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} *******"; ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} || (echo "********** configuring without forcing ***********"; cat config.log; ./configure --enable-build-as-root; cat config.log; exit 1)
EOF
else
cat <<EOF
$RUN echo "****** Configuring: ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} *******"; ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} || (cat config.log; exit 1)
EOF
fi
cat <<EOF
# We first compile base-toolchain because otherwise lots of packages are missing their dependency on 'patch'
ARG NUMPROC=8
ENV MAKE="make -j\${NUMPROC}"
ARG USE_MAKEFLAGS="-k"
#:toolchain:
$RUN make \${USE_MAKEFLAGS} base-toolchain
#:make:
# Avoid running the lengthy testsuite of the following.
$RUN make \${USE_MAKEFLAGS} cython
# By default, compile something tricky but that does not take too long. scipy uses BLAS.
ARG TARGETS="scipy"
$RUN SAGE_CHECK=yes SAGE_CHECK_PACKAGES="!r,!python3,!python2,!nose" make \${USE_MAKEFLAGS} \${TARGETS}
#:end:
EOF
15 changes: 13 additions & 2 deletions build/make/Makefile.in
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,17 @@ endif
# Directory to keep track of which packages are installed
INST = $(SAGE_SPKG_INST)

# Aliases for optional packages selected at configure time
# Aliases for mutually exclusive standard packages selected at configure time
TOOLCHAIN = @SAGE_TOOLCHAIN@
PYTHON = python@SAGE_PYTHON_VERSION@
MP_LIBRARY = @SAGE_MP_LIBRARY@
BLAS = @SAGE_BLAS@

# Because some .pc files are generated at configure time we can't write their
# names explicitly the Makefile, so we use the wildcard function here at make
# runtime
PCFILES = $(subst $(SAGE_SRC),$(SAGE_LOCAL),$(wildcard $(SAGE_SRC)/lib/pkgconfig/*.pc))

# Files to track installation of packages
BUILT_PACKAGES = @SAGE_BUILT_PACKAGES@
DUMMY_PACKAGES = @SAGE_DUMMY_PACKAGES@
Expand Down Expand Up @@ -62,10 +67,16 @@ STANDARD_PACKAGE_INSTS = \
$(foreach pkgname,$(STANDARD_PACKAGES),$(inst_$(pkgname)))

# All optional installed packages (triggers the auto-update)
OPTIONAL_INSTALLED_PACKAGES = @SAGE_OPTIONAL_INSTALLED_PACKAGES@
OPTIONAL_INSTALLED_PACKAGES = \
@SAGE_OPTIONAL_INSTALLED_PACKAGES@
OPTIONAL_INSTALLED_PACKAGE_INSTS = \
$(foreach pkgname,$(OPTIONAL_INSTALLED_PACKAGES),$(inst_$(pkgname)))

# All previously installed optional packages that are to be uninstalled
OPTIONAL_CLEANED_PACKAGES = \
@SAGE_OPTIONAL_CLEANED_PACKAGES@
OPTIONAL_CLEANED_PACKAGES_CLEANS = $(OPTIONAL_CLEANED_PACKAGES:%=%-clean)

# All packages which should be downloaded
SDIST_PACKAGES = @SAGE_SDIST_PACKAGES@

Expand Down
Loading

0 comments on commit bae1a82

Please sign in to comment.