build scripts update

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "test/data"]
 	path = test/data
 	url = https://github.com/mapnik/test-data.git
+[submodule "mason"]
+	path = mason
+	url = https://github.com/mapbox/mason.git

.travis.yml

@@ -1,22 +1,24 @@
 language: generic
 
+git:
+  submodules: false
+
 matrix:
   include:
     - os: linux
-      sudo: false
       compiler: clang
-      # note: only using ccache for CC is intentional here to
-      # workaround an odd bug in distutils that manifests when only `ccache` is used to link
-      # because distutils also has a bug whereby CC is used to compile instead of CXX, this works :)
-      env: JOBS=8 CXX="clang++-3.9 -Qunused-arguments" CC="ccache clang-3.9 -Qunused-arguments"
+      env: >-
+          CC="clang-3.9"
+          CXX="clang++-3.9"
       addons:
         apt:
-          sources: [ 'ubuntu-toolchain-r-test']
-          packages: [ 'libstdc++-5-dev', 'gdb', 'apport']
+          sources: [ ubuntu-toolchain-r-test ]
+          packages: [ clang-3.9, libstdc++-5-dev ]
     - os: osx
-      osx_image: xcode8.2
       compiler: clang
-      env: JOBS=4
+      env: >-
+          CC="clang"
+          CXX="clang++"
 
 cache:
   directories:
@@ -39,43 +41,80 @@ before_install:
    if [[ "${TRAVIS_OS_NAME}" == "osx" ]]; then
      rvm get head || true
    fi
- - source scripts/setup_mason.sh
- - export PYTHONUSERBASE=$(pwd)/mason_packages/.link
- - export PYTHONPATH=$(pwd)/mason_packages/.link/lib/python2.7/site-packages
- - export PATH=$(pwd)/mason_packages/.link/bin:${PYTHONUSERBASE}/bin:${PATH}
+ - declare -p HOME PATH "${!CCACHE@}" "${!TRAVIS@}"  # print variables by prefix
+
+ # An ancient bug in distutils (https://bugs.python.org/issue8027)
+ # prevents having multi-word CXX="ccache c++". Luckily distutils
+ # uses CC to compile C++ sources, and only uses CXX for linking,
+ # for which ccache is useless.
+ - export CC="ccache $CC"
+
  - export MASON_BUILD=true
+ - export MASON_ROOT=$PWD/mason_packages
+ - export PYTHONUSERBASE=$MASON_ROOT/.link
+ - export PATH=$MASON_ROOT/.link/bin:$PATH
  - export COMMIT_MESSAGE=$(git show -s --format=%B $TRAVIS_COMMIT | tr -d '\n')
- - |
-   if [[ $(uname -s) == 'Linux' ]]; then
-     export LDSHARED=$(python -c "import os;from distutils import sysconfig;print sysconfig.get_config_var('LDSHARED').replace('cc ','clang++-3.9 ')");
-     mason install clang++ 3.9.1
-     export PATH=$(mason prefix clang++ 3.9.1)/bin:${PATH}
-     which clang++
-   else
-     sudo easy_install pip;
-     export LDSHARED=$(python -c "import os;from distutils import sysconfig;print sysconfig.get_config_var('LDSHARED').replace('cc ','clang++ ')");
-   fi
+
+install:
+ - source ./bootstrap.sh
+ - install postgres 9.6.5
+ - install postgis 2.4.1
+ - install ccache 3.7.2
+ - ccache --version
+ - test "$TRAVIS_OS_NAME" != osx || sudo easy_install pip
+ - pip install --upgrade --user pip
  - pip install --upgrade --user nose
  - pip install --upgrade --user wheel
  - pip install --upgrade --user twine
  - pip install --upgrade --user setuptools
  - pip install --upgrade --user PyPDF2
- - python --version
-
-install:
- - mkdir -p ${PYTHONPATH}
- - python setup.py install --prefix ${PYTHONUSERBASE}
 
 before_script:
- # start postgres/postgis
- - source mason-config.env
- - ./mason_packages/.link/bin/postgres -k ${PGHOST} > postgres.log &
+ - touch SCRIPT_START
+ - python --version
+ - python -m site
+ - mkdir -p "$(python -m site --user-site)"
+
+ # distutils uses LDSHARED for linking, which it somehow constructs from CC
+ # with additional flags, but the resulting command is ultimately wrong.
+ #
+ #      LDSHARED[osx]='clang -bundle -undefined dynamic_lookup'
+ #      LDSHARED[linux]='x86_64-linux-gnu-gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -fno-strict-aliasing -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -Wdate-time -D_FORTIFY_SOURCE=2 -g -fstack-protector-strong -Wformat -Werror=format-security'
+ #
+ # On Linux there are some misplaced options that don't apply to linker.
+ # OSX command looks fine, but for some reason leads to the execution of
+ # 'clang++ clang -bundle -undefined dynamic_lookup ...' and fails.
+ #
+ #      LDCXXSHARED[osx]='clang++ -bundle -undefined dynamic_lookup'
+ #      LDCXXSHARED[linux]='c++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions'
+ #
+ # So let's just replace the value of LDSHARED with LDCXXSHARED, which
+ # apparently works on both OSX and Linux. Nevermind compiler 'c++',
+ # distutils substitutes that with CXX.
+ #
+ - LDSHARED=$(python -c 'from distutils import sysconfig as dusc; print(dusc.get_config_var("LDSHARED"))')
+ - LDCXXSHARED=$(python -c 'from distutils import sysconfig as dusc; print(dusc.get_config_var("LDCXXSHARED"))')
+ - printf 'default LDSHARED=%q\n' "$LDSHARED"
+ - printf 'default LDCXXSHARED=%q\n' "$LDCXXSHARED"
+ - export LDSHARED=${LDCXXSHARED}
 
 script:
+ - python setup.py install --user --prefix=
+   # why empty prefix is needed is explained here:
+   # https://stackoverflow.com/questions/4495120/combine-user-with-prefix-error-with-setup-py-install
+ - (( !TRAVIS_TEST_RESULT )) && source ./scripts/setup_postgres.sh
+ - (( !TRAVIS_TEST_RESULT )) && git submodule update --init --depth=20 test/
  - python test/run_tests.py
  - python test/visual.py -q
- # stop postgres
- - ./mason_packages/.link/bin/pg_ctl -w stop
+
+before_cache:
+ - ccache --version
+ - ccache --show-config
+ - ccache --show-stats
+ - ccache --zero-stats
+ - find $HOME/.ccache -mindepth 3 -newer SCRIPT_START | head | file --files-from -
+
+after_success:
  - |
    if [[ ${COMMIT_MESSAGE} =~ "[publish]" ]]; then
      python setup.py bdist_wheel
@@ -84,7 +123,7 @@ script:
         rename 's/linux_x86_64/any/;' $PRE_DISTS
      fi
      export DISTS='dist/*'
-     $(pwd)/mason_packages/.link/bin/twine upload -u $PYPI_USER -p $PYPI_PASSWORD $DISTS
+     "$MASON_ROOT/.link/bin/twine" upload -u $PYPI_USER -p $PYPI_PASSWORD $DISTS
    fi
 
 

bootstrap.sh

@@ -1,20 +1,19 @@
 #!/usr/bin/env bash
 
-set -eu
-set -o pipefail
-
-function install() {
-    MASON_PLATFORM_ID=$(mason env MASON_PLATFORM_ID)
-    if [[ ! -d ./mason_packages/${MASON_PLATFORM_ID}/${1}/ ]]; then
-        mason install $1 $2
-        mason link $1 $2
-    fi
-}
-
+BOOST_VERSION="1.66.0"
 ICU_VERSION="57.1"
+MAPNIK_VERSION="a0ea7db1a"
+
+function install_mason_deps() (
+    # subshell
+    set -eu
+    install_mapnik_deps
+    install mapnik ${MAPNIK_VERSION}
+)
 
-function install_mason_deps() {
-    install mapnik df0bbe4
+function install_mapnik_deps() (
+    # subshell
+    set -eu
     install jpeg_turbo 1.5.1
     install libpng 1.6.28
     install libtiff 4.0.7
@@ -27,61 +26,44 @@ function install_mason_deps() {
     install cairo 1.14.8
     install webp 0.6.0
     install libgdal 2.1.3
-    install boost 1.63.0
-    install boost_libsystem 1.63.0
-    install boost_libfilesystem 1.63.0
-    install boost_libprogram_options 1.63.0
-    install boost_libregex_icu57 1.63.0
+    install boost ${BOOST_VERSION}
+    install boost_libsystem ${BOOST_VERSION}
+    install boost_libfilesystem ${BOOST_VERSION}
+    install boost_libprogram_options ${BOOST_VERSION}
+    install boost_libregex_icu${ICU_VERSION%%.*} ${BOOST_VERSION}
     install freetype 2.7.1
     install harfbuzz 1.4.2-ft
     # deps needed by python-mapnik (not mapnik core)
-    install boost_libthread 1.63.0
-    install boost_libpython 1.63.0
-    install postgis 2.3.2-1
-}
+    install boost_libthread ${BOOST_VERSION}
+    install boost_libpython ${BOOST_VERSION}
+)
 
 function setup_runtime_settings() {
-    local MASON_LINKED_ABS=$(pwd)/mason_packages/.link
-    echo "export PROJ_LIB=${MASON_LINKED_ABS}/share/proj" > mason-config.env
-    echo "export ICU_DATA=${MASON_LINKED_ABS}/share/icu/${ICU_VERSION}" >> mason-config.env
-    echo "export GDAL_DATA=${MASON_LINKED_ABS}/share/gdal" >> mason-config.env
-    echo "export PATH=$(pwd)/mason_packages/.link/bin:${PATH}" >> mason-config.env
-    echo "export PGTEMP_DIR=$(pwd)/local-tmp" >> mason-config.env
-    echo "export PGDATA=$(pwd)/local-postgres" >> mason-config.env
-    echo "export PGHOST=$(pwd)/local-unix-socket" >> mason-config.env
-    echo "export PGPORT=1111" >> mason-config.env
-
-    source mason-config.env
-    rm -rf ${PGHOST}
-    mkdir -p ${PGHOST}
-    rm -rf ${PGDATA}
-    mkdir -p ${PGDATA}
-    rm -rf ${PGTEMP_DIR}
-    mkdir -p ${PGTEMP_DIR}
-    ./mason_packages/.link/bin/initdb
-    sleep 2
-    ./mason_packages/.link/bin/postgres -k ${PGHOST} > postgres.log &
-    sleep 2
-    ./mason_packages/.link/bin/createdb template_postgis -T postgres
-    ./mason_packages/.link/bin/psql template_postgis -c "CREATE TABLESPACE temp_disk LOCATION '${PGTEMP_DIR}';"
-    ./mason_packages/.link/bin/psql template_postgis -c "SET temp_tablespaces TO 'temp_disk';"
-    ./mason_packages/.link/bin/psql template_postgis -c "CREATE PROCEDURAL LANGUAGE 'plpythonu' HANDLER plpython_call_handler;"
-    ./mason_packages/.link/bin/psql template_postgis -c "CREATE EXTENSION postgis;"
-    ./mason_packages/.link/bin/psql template_postgis -c "SELECT PostGIS_Full_Version();"
-    ./mason_packages/.link/bin/pg_ctl -w stop
+    # PWD and ICU_VERSION are expanded here, but MASON_ROOT and PATH
+    # expansion must be deferred to the generated script, so that it
+    # can be used later
+    printf 'export %s=${MASON_ROOT:-%q}\n'          \
+            MASON_ROOT  "$PWD/mason_packages"       \
+        > ./mason-config.env
+    printf 'export %s=$%s\n'                                            \
+            MASON_BIN   "{MASON_ROOT}/.link/bin"                        \
+            PROJ_LIB    "{MASON_ROOT}/.link/share/proj"                 \
+            ICU_DATA    "{MASON_ROOT}/.link/share/icu/${ICU_VERSION}"   \
+            GDAL_DATA   "{MASON_ROOT}/.link/share/gdal"                 \
+            PATH    '{MASON_BIN}:${PATH}'                               \
+            PATH    '{PATH//":${MASON_BIN}:"/:}  # remove duplicates'   \
+        >> ./mason-config.env
+    source ./mason-config.env
 }
 
 function main() {
     source scripts/setup_mason.sh
-    setup_mason
-    install_mason_deps
-    setup_runtime_settings
+    [ $? = 0 ] && install_mason_deps
+    [ $? = 0 ] && setup_runtime_settings
+    [ $? = 0 ] || return
     echo "Ready, now run:"
     echo ""
     echo "    make test"
 }
 
 main
-
-set +eu
-set +o pipefail

scripts/setup_mason.sh

@@ -1,22 +1,51 @@
 #!/bin/bash
 
-set -eu
-set -o pipefail
-
 # we pin the mason version to avoid changes in mason breaking builds
-MASON_VERSION="3870d9c"
+MASON_ARCHIVE_VERSION="f00098c63"
+MASON_ARCHIVE_BASE="https://github.com/mapbox/mason/archive"
+
+function install() {
+    local package_prefix=
+    package_prefix=$(mason prefix $1 $2) || return
+    if [ ! -d "$package_prefix/" ]; then
+        mason install $1 $2 || return
+    fi
+    mason link $1 $2
+}
+
+function mason_packages() {
+    printf '%s\n' "${MASON_ROOT:-"$PWD/mason_packages"}"
+}
 
 function setup_mason() {
-    mkdir -p ./mason
-    curl -sSfL https://github.com/mapbox/mason/archive/${MASON_VERSION}.tar.gz | tar --gunzip --extract --strip-components=1 --exclude="*md" --exclude="test*" --directory=./mason
-    export MASON_HOME=$(pwd)/mason_packages/.link
-    export PATH=$(pwd)/mason:${PATH}
+    local cdup=
+    if cdup=$(git rev-parse --show-cdup 2>/dev/null); then
+        # we are inside *some* git repository (not necessarily python-mapnik)
+        if git submodule status "${cdup}mason" >/dev/null 2>&1; then
+            # there is submodule "mason" (assume we are in python-mapnik)
+            # update the submodule, bail out on failure
+            git submodule update --init "${cdup}mason" ||
+                return
+        else
+            # there is no submodule named "mason"
+            # proceed as if we were outside git repository
+            cdup=
+        fi
+    fi
+    if [ ! -d "${cdup}mason/" ]; then
+        # the directory doesn't exist, and we are either outside any git
+        # repository, or in a repository with no submodule named "mason"
+        mkdir -p "${cdup}mason/"
+        # download and unpack mason archive
+        curl -sSfL "${MASON_ARCHIVE_BASE}/${MASON_ARCHIVE_VERSION}.tar.gz" |
+            tar --extract --gunzip --directory="${cdup}mason/" \
+                --strip-components=1 --exclude="test" ||
+            return
+    fi
+    export PATH=$(cd "${cdup}mason" && pwd):${PATH}
     export CXX=${CXX:-clang++}
     export CC=${CC:-clang}
 }
 
 
 setup_mason
-
-set +eu
-set +o pipefail

scripts/setup_postgres.sh

@@ -0,0 +1,23 @@
+#! /bin/bash
+
+function setup_postgres_settings() {
+    export PGDATA=$PWD/local-postgres
+    export PGHOST=$PWD/local-unix-socket
+    export PGPORT=1111
+}
+
+function setup_postgres() {
+    rm -rf -- "$PGDATA" "$PGHOST"
+    mkdir -p -- "$PGDATA" "$PGHOST"
+
+    initdb -N
+    postgres -k "$PGHOST" -c fsync=off > postgres.log &
+    sleep 2
+    createdb template_postgis -T postgres
+    psql template_postgis -e -c "CREATE PROCEDURAL LANGUAGE 'plpythonu' HANDLER plpython_call_handler;"
+    psql template_postgis -e -c "CREATE EXTENSION postgis;"
+    psql template_postgis -e -c "SELECT PostGIS_Full_Version();"
+}
+
+setup_postgres_settings
+setup_postgres