Experimental multi-os testing

circle.yml

@@ -1,44 +1,21 @@
-#This file is generated by https://github.com/wcandillon/zorba-circleci-tpl
 machine:
-  post:
-    - sudo apt-get update -qq
-#    - sudo apt-get install -f
-    - sudo apt-get purge "libmagick*"
-    - sudo apt-get purge "libboost*"
-    - sudo apt-get purge "texlive*"
-    - sudo apt-get purge "flex"
-    - sudo apt-get purge "libfl-dev"
-    - sudo apt-get install distcc lib64z1 lib64z1-dev uuid-dev libxerces-c-dev graphviz libboost-filesystem-dev libgeos++-dev imagemagick libmagick++-dev libmagickwand-dev libfop-java libpdfbox-java libtidy-dev libjempbox-java libfontbox-java libarchive-dev libmysqlclient-dev openjdk-6-jdk libaccess-bridge-java-jni libxslt-dev libcurl4-openssl-dev libxerces-c-dev doxygen libedit-dev libkvutils-dev
-    - sudo locale-gen de_DE && sudo locale-gen de_DE.UTF-8 && sudo locale-gen fr_FR && sudo locale-gen fr_FR.UTF-8 && sudo update-locale
-    - wget http://launchpadlibrarian.net/160037457/bison_3.0.2.dfsg-2_amd64.deb
-    - wget http://launchpadlibrarian.net/160037458/libbison-dev_3.0.2.dfsg-2_amd64.deb
-    - sudo dpkg -i libbison-dev_3.0.2.dfsg-2_amd64.deb bison_3.0.2.dfsg-2_amd64.deb
-    - sudo sed -i.bak s/STARTDISTCC=\"false\"/STARTDISTCC=\"true\"/g /etc/default/distcc
-    - sudo /etc/init.d/distcc start
-    - wget https://launchpad.net/ubuntu/+source/flex/2.5.39-8/+build/6118519/+files/libfl-dev_2.5.39-8_amd64.deb
-    - wget https://launchpad.net/ubuntu/+source/flex/2.5.39-8/+build/6118519/+files/flex_2.5.39-8_amd64.deb
-    - sudo dpkg -i libfl-dev_2.5.39-8_amd64.deb flex_2.5.39-8_amd64.deb
   environment:
     ZORBA_SRC_DIR: /home/ubuntu/zorba
-    ZORBA_BUILD_DIR: /home/ubuntu/zorba/build
+    ZORBA_BUILD_DIR: /home/ubuntu/build
     ZORBA_MODULE_DIR: /home/ubuntu/zorba_modules
-    DISTCC_HOSTS: "ubuntu@node0 ubuntu@node1 ubuntu@node2 ubuntu@node3"
-    EXCLUDE_MODULES: "^zorba_info-extraction_module|conversion/currency-convert|zorba_couchbase_module|zorba_read-pdf_module|zorba_image_module|zorba_data-formatting_module"
+    EXCLUDE_MODULES: "^zorba_info-extraction_module|conversion/currency-convert|zorba_couchbase_module|zorba_read-pdf_module|zorba_image_module|zorba_data-formatting_module|zorba_nosqldb_module"
+  services:
+    - docker
 dependencies:
   override:
-    - echo "ZORBA_SRC_DIR ${ZORBA_SRC_DIR}"
-    - echo "ZORBA_BUILD_DIR ${ZORBA_BUILD_DIR}"
-    - echo "ZORBA_MODULE_DIR ${ZORBA_MODULE_DIR}"
     - mkdir -p "${ZORBA_MODULE_DIR}"
     - cmake -Doutdir="${ZORBA_MODULE_DIR}" -Dallmodules=1 -P "${ZORBA_SRC_DIR}/modules/DownloadModules.cmake"
     - mkdir -p "${ZORBA_BUILD_DIR}"
 test:
   pre:
-    - case $CIRCLE_NODE_INDEX in 0) cd "${ZORBA_BUILD_DIR}" && CC="distcc gcc" CXX="distcc c++" cmake -DZORBA_MODULES_DIR="${ZORBA_MODULE_DIR}" -DZORBA_XQUERYX:BOOL=1 -DZORBA_TEST_XQUERYX:BOOL=1 -DZORBA_WITH_BIG_INTEGER=1 -DCMAKE_BUILD_TYPE=Release -DZORBA_TEST_PLAN_SERIALIZATION:BOOL=1 -DZORBATEST_USE_MT_XQTS:BOOL=1 "${ZORBA_SRC_DIR}" && make -j8 && bash "${ZORBA_SRC_DIR}/import_tests.sh" && make fots-activate-sets && cd "${ZORBA_SRC_DIR}" && tar -cf build.tar build ;; esac:
-        parallel: true
-    - case $CIRCLE_NODE_INDEX in 1) cd "${ZORBA_SRC_DIR}" && scp -r ubuntu@node0:/home/ubuntu/zorba/build.tar . && tar -xvf build.tar && bash "${ZORBA_SRC_DIR}/import_tests.sh" && cd "${ZORBA_BUILD_DIR}" && make fots-activate-sets ;; 2) cd "${ZORBA_SRC_DIR}" && scp -r ubuntu@node0:/home/ubuntu/zorba/build.tar . && tar -xvf build.tar && bash "${ZORBA_SRC_DIR}/import_tests.sh" && cd "${ZORBA_BUILD_DIR}" && make fots-activate-sets ;; 3) cd "${ZORBA_SRC_DIR}" && scp -r ubuntu@node0:/home/ubuntu/zorba/build.tar . && tar -xvf build.tar && bash "${ZORBA_SRC_DIR}/import_tests.sh" && cd "${ZORBA_BUILD_DIR}" && make fots-activate-sets ;; esac:
+    - case $CIRCLE_NODE_INDEX in 0) docker run -v "/home/ubuntu:/zorba" "fcavalieri/zorba-test-precise" "cmake" && docker run -v "/home/ubuntu:/zorba" "fcavalieri/zorba-test-precise" "make" && docker run -v "/home/ubuntu:/zorba" "fcavalieri/zorba-test-precise" "enable-extra-tests" ;; 1) docker run -v "/home/ubuntu:/zorba" "fcavalieri/zorba-test-trusty" "cmake" && docker run -v "/home/ubuntu:/zorba" "fcavalieri/zorba-test-trusty" "make" && docker run -v "/home/ubuntu:/zorba" "fcavalieri/zorba-test-trusty" "enable-extra-tests" ;;  2) docker run -v "/home/ubuntu:/zorba" "fcavalieri/zorba-test-vivid" "cmake" && docker run -v "/home/ubuntu:/zorba" "fcavalieri/zorba-test-vivid" "make" && docker run -v "/home/ubuntu:/zorba" "fcavalieri/zorba-test-vivid" "enable-extra-tests" ;;  3) docker run -v "/home/ubuntu:/zorba" "fcavalieri/zorba-test-wily" "cmake" && docker run -v "/home/ubuntu:/zorba" "fcavalieri/zorba-test-wily" "make" && docker run -v "/home/ubuntu:/zorba" "fcavalieri/zorba-test-wily" "enable-extra-tests" ;; esac:
         parallel: true
   override:
-    - case $CIRCLE_NODE_INDEX in 0) cd "${ZORBA_BUILD_DIR}" && env QTTEST=42 QTTEST2=other QTTESTEMPTY='' ctest -j2 -T Test -I 1,2250 -E "${EXCLUDE_MODULES}" --output-on-failure ;; 1) cd "${ZORBA_BUILD_DIR}" && env QTTEST=42 QTTEST2=other QTTESTEMPTY='' ctest -j2 -T Test -I 2251,4500 -E "${EXCLUDE_MODULES}" --output-on-failure ;; 2) cd "${ZORBA_BUILD_DIR}" && env QTTEST=42 QTTEST2=other QTTESTEMPTY='' ctest -j2 -T Test -I 4501,6750 -E "${EXCLUDE_MODULES}" --output-on-failure ;; 3) cd "${ZORBA_BUILD_DIR}" && env QTTEST=42 QTTEST2=other QTTESTEMPTY='' ctest -j2 -T Test -I 6751 -E "${EXCLUDE_MODULES}" --output-on-failure ;; esac:
+    - case $CIRCLE_NODE_INDEX in 0) docker run -v "/home/ubuntu:/zorba" "fcavalieri/zorba-test-precise" "ctest" "-E" "${EXCLUDE_MODULES}" ;; 1) docker run -v "/home/ubuntu:/zorba" "fcavalieri/zorba-test-trusty" "ctest" "-E" "${EXCLUDE_MODULES}" ;; 2) docker run -v "/home/ubuntu:/zorba" "fcavalieri/zorba-test-vivid" "ctest" "-E" "${EXCLUDE_MODULES}" ;; 3) docker run -v "/home/ubuntu:/zorba" "fcavalieri/zorba-test-wily" "ctest" "-E" "${EXCLUDE_MODULES}" ;; esac:
         parallel: true
         timeout: 600

docker/zorba-test/build.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -e
+
+if [ "$#" -ne 1 ]
+then
+  echo "No version specified, use for instance: ./build.sh precise"
+  exit 1
+fi
+
+FOLDER=$1
+NAME=zorba-test-$FOLDER
+BUILD_FOLDER=build
+(
+  rm -rf $BUILD_FOLDER
+  mkdir $BUILD_FOLDER
+  cp docker-entrypoint.sh $BUILD_FOLDER/docker-entrypoint.sh
+  cp $FOLDER/Dockerfile $BUILD_FOLDER/Dockerfile
+  cd $BUILD_FOLDER
+
+  set +e
+  docker stop "$NAME" > /dev/null 2>&1
+  docker rm "$NAME" > /dev/null 2>&1
+  set -e
+
+  docker build -t "$NAME" .
+)

docker/zorba-test/docker-entrypoint.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+set -e
+
+function show_usage()
+{
+  echo "No action specified. Allowed actions are:"
+  echo "ssh                - to start an ssh server (requires running docker in detached mode (-d))"
+  echo "cmake              - to prepare for the build of zorba"
+  echo "make               - to build zorba"
+  echo "enable-extra-tests - to enable fots tests"
+  echo "ctest ...          - to run the test suite. Extra arguments are passed to ctest"
+}
+
+if [ "$#" -eq "0" ]
+then
+  show_usage
+  exit 1
+fi
+
+ACTION=$1
+case "$ACTION" in
+  ssh)
+    /usr/sbin/sshd -D
+    ;;
+  cmake)
+    rm -rf /zorba/build
+    mkdir /zorba/build
+    cd /zorba/build
+    cmake -DZORBA_MODULES_DIR="/zorba/zorba_modules" -DZORBA_XQUERYX:BOOL=1 -DZORBA_TEST_XQUERYX:BOOL=1 -DZORBA_WITH_BIG_INTEGER=1 -DCMAKE_BUILD_TYPE=Release -DZORBA_TEST_PLAN_SERIALIZATION:BOOL=1 -DZORBATEST_USE_MT_XQTS:BOOL=1 "/zorba/zorba"    
+    ;;
+  make)
+    cd /zorba/build
+    make -j8
+    ;;
+  enable-extra-tests)
+    cd /zorba/build
+    /zorba/zorba/import_tests.sh /zorba/zorba /zorba/build
+    make fots-activate-sets
+    ;;
+  ctest)
+    cd /zorba/build
+    env QTTEST=42 QTTEST2=other QTTESTEMPTY='' ctest -j2 -T Test --output-on-failure ${@:2}
+    ;;
+  help)
+    show_usage
+    exit 0
+    ;;
+  *)
+    show_usage
+    exit 1
+    ;;
+esac

docker/zorba-test/precise/Dockerfile

@@ -0,0 +1,73 @@
+FROM ubuntu:precise
+
+MAINTAINER Federico Cavalieri <f@28.io>
+
+RUN apt-get update -y
+RUN apt-get install python-software-properties -y
+RUN add-apt-repository ppa:fcavalieri/zorba -y
+RUN apt-get update -y
+
+#Prerequisites
+# - Utils
+RUN apt-get install git wget -y
+
+# - Build
+RUN apt-get install build-essential cmake -y
+
+# - Zorba Core
+RUN apt-get install libxml2-dev libicu-dev uuid-dev libxerces-c-dev libxslt-dev libcurl4-openssl-dev openjdk-7-jdk -y
+
+# - Documentation
+RUN apt-get install doxygen texlive -y
+
+# - Lexer and Parser
+RUN apt-get install m4 -y
+
+RUN wget http://launchpadlibrarian.net/160037457/bison_3.0.2.dfsg-2_amd64.deb
+RUN wget http://launchpadlibrarian.net/160037458/libbison-dev_3.0.2.dfsg-2_amd64.deb
+RUN dpkg -i libbison-dev_3.0.2.dfsg-2_amd64.deb bison_3.0.2.dfsg-2_amd64.deb
+RUN rm libbison-dev_3.0.2.dfsg-2_amd64.deb bison_3.0.2.dfsg-2_amd64.deb
+
+RUN wget https://launchpad.net/ubuntu/+source/flex/2.5.39-8/+build/6118519/+files/libfl-dev_2.5.39-8_amd64.deb
+RUN wget https://launchpad.net/ubuntu/+source/flex/2.5.39-8/+build/6118519/+files/flex_2.5.39-8_amd64.deb
+RUN dpkg -i libfl-dev_2.5.39-8_amd64.deb flex_2.5.39-8_amd64.deb
+RUN rm libfl-dev_2.5.39-8_amd64.deb flex_2.5.39-8_amd64.deb
+
+# - MT Testdriver:
+RUN apt-get install libboost-filesystem-dev libboost-system-dev -y
+
+# - External Modules:
+RUN apt-get install libfop-java graphviz-dev libtidy-dev libarchive-dev libmagick++-dev libgeos++-dev libsqlite3-dev libpdfbox-java libfontbox-java libjempbox-java xmlbeans -y
+
+#- NoSQL External Module: 
+RUN wget http://download.oracle.com/otn-pub/otn_software/nosql-database/kv-c-driver_3.3.5-0_amd64.deb 
+RUN dpkg -i kv-c-driver_3.3.5-0_amd64.deb
+RUN rm kv-c-driver_3.3.5-0_amd64.deb 
+
+# Couchbase External Module: 
+RUN wget -O/etc/apt/sources.list.d/couchbase.list http://packages.couchbase.com/ubuntu/couchbase-ubuntu1204.list
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A3FAA648D9223EDA
+RUN apt-get update -y
+RUN apt-get install libcouchbase-dev -y
+
+# - Email External Module
+RUN apt-get install libc-client2007e-dev libssl-dev libpam-krb5 libkrb5-dev libgssapi-krb5-2 libpam-krb5 libpam-modules libpam0g-dev -y
+
+# - Purge Legacy Java
+RUN apt-get purge openjdk-6-.* -y
+
+# - Fix Locales
+RUN locale-gen de_DE && locale-gen de_DE.UTF-8 && locale-gen fr_FR && locale-gen fr_FR.UTF-8 && update-locale
+
+#SSH
+RUN apt-get install openssh-server -y && \
+    mkdir -p /var/run/sshd && \
+    /bin/sh -c 'echo root:zorba | chpasswd'
+
+#Expose
+VOLUME /zorba
+EXPOSE 22
+
+#Entrypoint
+COPY docker-entrypoint.sh /docker-entrypoint.sh
+ENTRYPOINT ["/docker-entrypoint.sh"]

docker/zorba-test/publish.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+if [ "$#" -ne 1 ]
+then
+  echo "No version specified, use for instance: ./publish.sh precise"
+  exit 1
+fi
+
+NAME=zorba-test-$1
+FULL_NAME=fcavalieri/$NAME
+docker tag -f $NAME $FULL_NAME
+docker push $FULL_NAME

docker/zorba-test/trusty/Dockerfile

@@ -0,0 +1,61 @@
+FROM ubuntu:trusty
+
+MAINTAINER Federico Cavalieri <f@28.io>
+
+RUN apt-get update -y
+RUN apt-get install software-properties-common -y
+RUN add-apt-repository ppa:fcavalieri/zorba -y
+RUN apt-get update -y
+
+#Prerequisites
+# - Utils
+RUN apt-get install git wget -y
+
+# - Build
+RUN apt-get install build-essential cmake -y
+
+# - Zorba Core
+RUN apt-get install libxml2-dev libicu-dev uuid-dev libxerces-c-dev libxslt-dev libcurl4-openssl-dev openjdk-7-jdk -y
+
+# - Documentation
+RUN apt-get install doxygen texlive -y
+
+# - Lexer and Parser
+RUN apt-get install m4 bison libbison-dev flex libfl-dev -y
+
+# - MT Testdriver:
+RUN apt-get install libboost-filesystem-dev libboost-system-dev -y
+
+# - External Modules:
+RUN apt-get install libfop-java graphviz-dev libtidy-dev libarchive-dev libmagick++-dev libgeos++-dev libsqlite3-dev libpdfbox-java libfontbox-java libjempbox-java xmlbeans -y
+
+#- NoSQL External Module: 
+RUN wget http://download.oracle.com/otn-pub/otn_software/nosql-database/kv-c-driver_3.3.5-0_amd64.deb 
+RUN dpkg -i kv-c-driver_3.3.5-0_amd64.deb
+RUN rm kv-c-driver_3.3.5-0_amd64.deb 
+
+# Couchbase External Module: 
+RUN wget -O/etc/apt/sources.list.d/couchbase.list http://packages.couchbase.com/ubuntu/couchbase-ubuntu1404.list
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A3FAA648D9223EDA
+RUN apt-get update -y
+RUN apt-get install libcouchbase-dev -y
+
+# - Email External Module
+RUN apt-get install libc-client2007e-dev libssl-dev libkrb5-dev libpam0g-dev -y
+
+# - Fix Locales
+RUN locale-gen de_DE && locale-gen de_DE.UTF-8 && locale-gen fr_FR && locale-gen fr_FR.UTF-8 && update-locale
+
+#SSH
+RUN apt-get install openssh-server -y && \
+    mkdir -p /var/run/sshd && \
+    /bin/sh -c 'echo root:zorba | chpasswd'
+RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+
+#Expose
+VOLUME /zorba
+EXPOSE 22
+
+#Entrypoint
+COPY docker-entrypoint.sh /docker-entrypoint.sh
+ENTRYPOINT ["/docker-entrypoint.sh"]

docker/zorba-test/vivid/Dockerfile

@@ -0,0 +1,61 @@
+FROM ubuntu:vivid
+
+MAINTAINER Federico Cavalieri <f@28.io>
+
+RUN apt-get update -y
+RUN apt-get install software-properties-common -y
+RUN add-apt-repository ppa:fcavalieri/zorba -y
+RUN apt-get update -y
+
+#Prerequisites
+# - Utils
+RUN apt-get install git wget -y
+
+# - Build
+RUN apt-get install build-essential cmake -y
+
+# - Zorba Core
+RUN apt-get install libxml2-dev libicu-dev uuid-dev libxerces-c-dev libxslt-dev libcurl4-openssl-dev openjdk-8-jdk -y
+
+# - Documentation
+RUN apt-get install doxygen texlive -y
+
+# - Lexer and Parser
+RUN apt-get install m4 bison libbison-dev flex libfl-dev -y
+
+# - MT Testdriver:
+RUN apt-get install libboost-filesystem-dev libboost-system-dev -y
+
+# - External Modules:
+RUN apt-get install libfop-java graphviz-dev libtidy-dev libarchive-dev libmagick++-dev libgeos++-dev libsqlite3-dev libpdfbox-java libfontbox-java libjempbox-java xmlbeans -y
+
+#- NoSQL External Module: 
+RUN wget http://download.oracle.com/otn-pub/otn_software/nosql-database/kv-c-driver_3.3.5-0_amd64.deb 
+RUN dpkg -i kv-c-driver_3.3.5-0_amd64.deb
+RUN rm kv-c-driver_3.3.5-0_amd64.deb 
+
+# Couchbase External Module: 
+RUN wget -O/etc/apt/sources.list.d/couchbase.list http://packages.couchbase.com/ubuntu/couchbase-ubuntu1404.list
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A3FAA648D9223EDA
+RUN apt-get update -y
+RUN apt-get install libcouchbase-dev -y
+
+# - Email External Module
+RUN apt-get install libc-client2007e-dev libssl-dev libkrb5-dev libpam0g-dev -y
+
+# - Fix Locales
+RUN locale-gen de_DE && locale-gen de_DE.UTF-8 && locale-gen fr_FR && locale-gen fr_FR.UTF-8 && update-locale
+
+#SSH
+RUN apt-get install openssh-server -y && \
+    mkdir -p /var/run/sshd && \
+    /bin/sh -c 'echo root:zorba | chpasswd'
+RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+
+#Expose
+VOLUME /zorba
+EXPOSE 22
+
+#Entrypoint
+COPY docker-entrypoint.sh /docker-entrypoint.sh
+ENTRYPOINT ["/docker-entrypoint.sh"]